Forward backend options during streaming speech generation (#1389) (#1390)

Switch SQLite ANN connections when load() is given a new path

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
 
-    timeout-minutes: 60
+    timeout-minutes: 75
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
@@ -18,7 +18,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install Java
         uses: actions/setup-java@v5
@@ -29,7 +29,7 @@ jobs:
       - name: Install dependencies - Linux
         run: |
           sudo apt-get update
-          sudo apt-get install libportaudio2 libsndfile1
+          sudo apt-get install libportaudio2 libsndfile1 libegl1 libgles2
           sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
         if: matrix.os == 'ubuntu-latest'
 
@@ -40,7 +40,7 @@ jobs:
           echo "TIKA_STARTUP_SLEEP=30" >> $GITHUB_ENV
           echo "TIKA_STARTUP_MAX_RETRY=10" >> $GITHUB_ENV
           brew install portaudio
-          sudo xcode-select -s "/Applications/Xcode_16.app"
+          pip install torch==2.12.1 torchvision==0.27.1
         if: matrix.os == 'macos-latest'
 
       - name: Install dependencies - Windows
@@ -61,12 +61,16 @@ jobs:
           HF_HUB_ETAG_TIMEOUT: 100
           HF_HUB_DOWNLOAD_TIMEOUT: 100
           HF_XET_CHUNK_CACHE_SIZE_BYTES: 0
+          DOCLING_DEVICE: "cpu"
+          DOCLING_INFERENCE_COMPILE_TORCH_MODELS: false
 
       - uses: pre-commit/[email protected]
         if: matrix.os == 'ubuntu-latest'
 
       - name: Test Coverage
-        run: coveralls --service=github
+        run: |
+          coverage report --fail-under=100
+          coveralls --service=github
         if: matrix.os == 'ubuntu-latest'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -7,12 +7,19 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
-      - run: |
+          python-version: "3.11"
+
+      - name: Install txtai
+        run: |
           pip install -U pip wheel
           pip install .[all,dev]
-      - run: mkdocs gh-deploy --force
+
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --force
  

.github/workflows/minimal.yml

@@ -7,11 +7,16 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
-      - run: |
+          python-version: "3.11"
+
+      - name: Minimal install
+        run: |
           pip install -U pip wheel
-          pip install .
+          MINIMAL=1 pip install .
           python -c "import txtai"

.pylintrc

@@ -12,6 +12,8 @@ max-locals=40
 max-returns=10
 max-attributes=20
 min-public-methods=0
+max-public-methods=80
 
 [FORMAT]
 max-line-length=150
+max-module-lines=1500

Makefile

@@ -42,6 +42,7 @@ coverage:
 	coverage run -m unittest discover -v -k testdatabase -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testembeddings -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testgraph -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testlibrary -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testmodels -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testoptional -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testpipeline.testaudio -s ${TEST_DIR}

README.md

@@ -49,7 +49,7 @@ Summary of txtai features:
 - 🔋 Batteries included with defaults to get up and running fast
 - ☁️ Run local or scale out with container orchestration
 
-txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
 > [!NOTE]
 >
@@ -140,8 +140,9 @@ Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/ma
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 | [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
 
 #### Retrieval augmented generation
@@ -189,9 +190,9 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
-See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud).
+See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda), [lightweight minimal installation](https://neuml.github.io/txtai/install/#minimal-install) and how to [run with containers](https://neuml.github.io/txtai/cloud).
 
 ## Model guide
 
@@ -203,9 +204,9 @@ See the table below for the current recommended models. These models all allow c
 | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
 | [Embeddings](https://neuml.github.io/txtai/embeddings)                        | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
 | [Image Captions](https://neuml.github.io/txtai/pipeline/image/caption)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
-| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [DeBERTa v3 Zeroshot](https://hf.co/MoritzLaurer/deberta-v3-base-zeroshot-v2.0-c) |
 | [Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels)          | Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer)          |
-| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b)                 |
+| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [Gemma 4 31B](https://hf.co/google/gemma-4-31B) |
 | [Summarization](https://neuml.github.io/txtai/pipeline/text/summary)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
 | [Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
 | [Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
@@ -217,9 +218,8 @@ See the following links to learn more.
 
 - [Hugging Face Tasks](https://hf.co/tasks)
 - [Hugging Face Model Hub](https://hf.co/models)
-- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
-- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
-- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+- [Embeddings Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LLM Leaderboard](https://hf.co/spaces/lmarena-ai/arena-leaderboard)
 
 ## Powered by txtai
 
@@ -242,6 +242,7 @@ In addition to this list, there are also many other [open-source projects](https
 ![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further-ghdark.png#gh-dark-mode-only)
 
 - [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
 - [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)

docker/base/Dockerfile

@@ -1,5 +1,5 @@
 # Set base image
-ARG BASE_IMAGE=python:3.10-slim
+ARG BASE_IMAGE=python:3.11-slim
 FROM $BASE_IMAGE
 
 # Install GPU-enabled version of PyTorch if set
@@ -8,7 +8,7 @@ ARG GPU
 # Target CPU architecture
 ARG TARGETARCH
 
-# Set Python version (i.e. 3, 3.10)
+# Set Python version (i.e. 3, 3.11)
 ARG PYTHON_VERSION=3
 
 # List of txtai components to install
@@ -21,13 +21,13 @@ ENV LANG=C.UTF-8
 RUN \
     # Install required packages
     apt-get update && \
-    apt-get -y --no-install-recommends install libgomp1 libportaudio2 libsndfile1 git gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
+    apt-get -y --no-install-recommends install libgomp1 libportaudio2 libsndfile1 libegl1 libgles2 libvulkan1 git gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
     rm -rf /var/lib/apt/lists && \
     \
     # Install txtai project and dependencies
     ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
     python -m pip install --no-cache-dir -U pip wheel setuptools && \
-    if [ -z ${GPU} ] && { [ -z ${TARGETARCH} ] || [ ${TARGETARCH} = "amd64" ] ;}; then pip install --no-cache-dir torch==2.10.0+cpu torchvision==0.25.0+cpu -f https://download.pytorch.org/whl/torch -f https://download.pytorch.org/whl/torchvision; fi && \
+    if [ -z ${GPU} ]; then pip install --no-cache-dir torch==2.14.1+cpu torchvision==0.29.1+cpu -f https://download.pytorch.org/whl/torch -f https://download.pytorch.org/whl/torchvision; fi && \
     python -m pip install --no-cache-dir txtai${COMPONENTS} && \
     python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])" && \
     \

docker/minimal/Dockerfile

@@ -0,0 +1,20 @@
+# Set base image
+ARG BASE_IMAGE=python:3.11-slim
+FROM $BASE_IMAGE
+
+# Target CPU architecture
+ARG TARGETARCH
+
+# Set Python version (i.e. 3, 3.11)
+ARG PYTHON_VERSION=3
+
+# Locale environment variables
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN \
+    # Install minimal txtai project and dependencies
+    MINIMAL=1 python -m pip install --no-cache-dir txtai_minimal
+
+# Set default working directory
+WORKDIR /app

docs/agent/configuration.md

@@ -85,11 +85,19 @@ The following shortcut strings load tools directly. Passing a Tool instance is a
 
 | Tool        | Description                                               |
 |:------------|:----------------------------------------------------------|
+| bash        | Runs a shell command through subprocess                   |
+| defaults    | Loads all of these tools as the default toolkit           |
+| edit        | Edits a file in place and returns a diff                  |
+| glob        | Finds matching file patterns in a directory               |
+| grep        | Finds matching file content in a directory                |
 | http.*      | HTTP Path to a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/docs/getting-started/intro) server |
 | python      | Runs a Python action                                      |
-| *.md        | Loads a [`skill.md`](https://agentskills.io/specification) file
+| read        | Reads file or url content, supports text extraction       |
+| todowrite   | Generates a task list to organize complex tasks           |
 | websearch   | Runs a websearch using the built-in websearch tool        |
-| webview     | Extracts content from a web page                          |
+| webview     | Extracts content from a web page. Alias for `read` tool   |
+| write       | Writes content to file                                    |
+| *.md        | Loads a [`skill.md`](https://agentskills.io/specification) file |
 
 ## instructions
 

docs/agent/index.md

@@ -153,3 +153,4 @@ Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/ma
 | [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
 | [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
 | [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |

docs/api/configuration.md

@@ -33,7 +33,16 @@ Path to save and load the embeddings index. Each API instance can only access a
 writable: boolean
 ```
 
-Determines if the input embeddings index is writable (true) or read-only (false). This allows serving a read-only index.
+Sets this embeddings index as writable (true) or read-only (false). This allows serving a read-only index. Defaults to read-only.
+
+### reindex
+```yaml
+reindex: boolean
+```
+
+Enables re-indexing for this embeddings index (defaults to disabled).
+
+Re-indexing accepts new configuration and should only be enabled when the configuration comes from trusted sources.
 
 ### cloud
 [Cloud storage settings](../../embeddings/configuration/cloud) can be set under a `cloud` top level configuration group.

docs/api/mcp.md

@@ -5,9 +5,22 @@ The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction)
 The API can be configured to handle MCP requests. All enabled endpoints set in the API configuration are automatically added as MCP tools.
 
 ```yaml
-mcp: True
+mcp: boolean|dict
 ```
 
+When `mcp` is a boolean, default arguments are used. When `mcp` is a dictionary it supports the following options.
+
+```yaml
+mcp:
+  clientargs: http client options (dict)
+  mcpargs: FastApiMCP options (dict)
+```
+
+See the following links for details on the options.
+
+- [HTTP Client Arguments](https://www.python-httpx.org/api/#asyncclient)
+- [FastApiMCP Arguments](https://github.com/tadata-org/fastapi_mcp/blob/main/fastapi_mcp/server.py#L22)
+
 Once this configuration option is added, a new route is added to the application `/mcp`. 
 
 The [Model Context Protocol Inspector tool](https://www.npmjs.com/package/@modelcontextprotocol/inspector) is a quick way to explore how the MCP tools are exported through this interface.

docs/api/security.md

@@ -6,9 +6,9 @@ The default implementation of an API service runs via HTTP and is fully open. If
 
 The default API service command starts a Uvicorn server as a HTTP service on port 8000. To run a HTTPS service, consider the following options.
 
-- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.org/deployment/#running-behind-nginx) for more.
+- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.dev/deployment/#running-behind-nginx) for more.
 
-- [Uvicorn SSL Certificate](https://www.uvicorn.org/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
+- [Uvicorn SSL Certificate](https://www.uvicorn.dev/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
 
 ## Authorization
 
@@ -31,3 +31,44 @@ curl \
 It's important to note that HTTPS **must** be enabled using one of the methods mentioned above. Otherwise, tokens will be exchanged as clear text. 
 
 Authentication and Authorization can be fully customized. See the [dependencies](../customization#dependencies) section for more.
+
+## Safe Open
+
+Many components in txtai work with local files and have no restrictions on what files and urls can be accessed. The [textractor pipeline](../../pipeline/data/textractor) and [retrieve](../../workflow/task/retrieve) tasks both have a built-in option called `safeopen` which is enabled by default when running the API and applications. It enforces the following rules:
+
+- Local files must be in the safeopen directory (defaults to temp dir)
+- URLs must be public URLs (also has checks for redirects and DNS rebind attempts)
+
+The following workflow only allows accessing files stored in a specific directory along with public urls.
+
+```yaml
+workflow:
+  textract:
+    tasks:
+      - action: textractor
+```
+
+A number of of pipelines support reading local files. For these it's recommended to wrap those pipelines as workflows with a `retrieve` task. Note that pipelines can be created in-line and don't need to be exposed via the API.
+
+```yaml
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```
+
+Additionally, internal pipelines can be created but not exposed via the API.
+
+```yaml
+routes:
+  caption: False
+
+caption:
+
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```

docs/cloud.md

@@ -30,8 +30,8 @@ Examples build commands below.
 # Get Dockerfile
 wget https://raw.githubusercontent.com/neuml/txtai/master/docker/base/Dockerfile
 
-# Build Ubuntu 22.04 image running Python 3.10
-docker build -t txtai --build-arg BASE_IMAGE=ubuntu:22.04 --build-arg PYTHON_VERSION=3.10 .
+# Build Ubuntu 22.04 image running Python 3.11
+docker build -t txtai --build-arg BASE_IMAGE=ubuntu:22.04 --build-arg PYTHON_VERSION=3.11 .
 
 # Build image with GPU support
 docker build -t txtai --build-arg GPU=1 .

docs/embeddings/configuration/ann.md

@@ -4,7 +4,7 @@ Approximate Nearest Neighbor (ANN) index configuration for storing vector embedd
 
 ## backend
 ```yaml
-backend: faiss|hnsw|annoy|ggml|numpy|torch|pgvector|sqlite|custom
+backend: faiss|hnsw|annoy|ggml|milvus|numpy|rabitq|torch|turbovec|zvec|pgvector|sqlite|custom
 ```
 
 Sets the ANN backend. Defaults to `faiss`. Additional backends are available via the [ann](../../../install/#ann) extras package. Set custom backends via setting this parameter to the fully resolvable class string.
@@ -74,9 +74,14 @@ The [GGML](https://github.com/ggml-org/ggml) backend is a k-nearest neighbors ba
 
 [See this](https://github.com/ggml-org/ggml/blob/master/include/ggml.h#L379) for a list of quantization types.
 
-### numpy
+### milvus
 
-The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets that fit into memory.
+```yaml
+milvus:
+    m: number of HNSW links per element (int) - defaults to 50
+```
+
+### numpy
 
 ```yaml
 numpy:
@@ -84,12 +89,30 @@ numpy:
                  defaults to NumPy array storage
 ```
 
+The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets that fit into memory.
+
+### rabitq
+
+```yaml
+rabitq:
+    mode: index mode (ivf or hnsw) - defaults to "ivf"
+    nbits: number of quantization bits per dimension (int) - defaults to 1,
+           supports 1 to 9 in both modes, ivf mode also supports 32 which
+           keeps raw vectors for reranking
+    clusters: number of IVF clusters (int) - defaults to
+              max(1, min(round(4 * sqrt(embeddings count)), embeddings count))
+    nprobe: search probe setting for ivf mode (int) - defaults to
+            max(1, round(num_clusters/16))
+    m: M param for hnsw mode (int) - defaults to 16
+    efconstruction: ef_construction param for hnsw mode (int) - defaults to 200
+    efsearch: ef search param for hnsw mode (int) - defaults to None and not set
+    randomseed: random-seed param for hnsw mode (int) - defaults to 100
+```
+
+The [rabitq](https://github.com/VectorDB-NTU/RaBitQ-Library) backend is a quantized index powered by the [RaBitQ algorithm](https://doi.org/10.1145/3725413). It supports ivf and hnsw search modes. Vectors are stored with 1-bit quantization by default. More bits increase accuracy at the cost of memory. The index is stored as a single file and no vectors are kept outside of it. With nbits set to 32, the vectors are stored in the index for reranking. The ivf mode supports upserts and deletes. Deleted rows keep their storage. Rows added after the first index are assigned to the clusters trained when the index was built, so recall can drop if the new data differs from the original data. Rebuild the index to train new clusters. Note that hnsw indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+
 ### torch
 
-The Torch backend is a k-nearest neighbors backend like NumPy. It supports GPU-enabled operations. It also has support for quantization which enables fitting larger arrays into GPU memory.
-
-When quantization is enabled, vectors are _always_ stored in safetensors. _Note that macOS support for quantization is limited._
-
 ```yaml
 torch:
     safetensors: stores vectors using the safetensors format - defaults
@@ -99,6 +122,30 @@ torch:
         blocksize: quantization block size parameter
 ```
 
+The Torch backend is a k-nearest neighbors backend like NumPy. It supports GPU-enabled operations. It also has support for quantization which enables fitting larger arrays into GPU memory.
+
+When quantization is enabled, vectors are _always_ stored in safetensors. _Note that macOS support for quantization is limited._
+
+### turbovec
+
+```yaml
+turbovec:
+    bitwidth: number of bits to store each vector dimension as. Supports 2, 3 or 4.
+```
+
+The [turbovec](https://github.com/RyanCodrai/turbovec) backend is a k-nearest neighbors backend powered by the [TurboQuant algorithm](https://arxiv.org/abs/2504.19874).
+
+### zvec
+
+```yaml
+zvec:
+    efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+    m: number of HNSW links per element (int) - defaults to 50
+    efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+The [zvec](https://github.com/alibaba/zvec) backend is an embedded, path-based vector index.
+
 ### pgvector
 ```yaml
 pgvector:

docs/embeddings/configuration/database.md

@@ -50,14 +50,27 @@ Object encoding options are:
 functions: list
 ```
 
-List of functions with user-defined SQL functions, only used when [content](#content) is enabled. Each list element must be one of the following:
+List of functions with user-defined SQL functions. Each list element must be one of the following:
 
 - function
 - callable object
-- dict with fields for name, argcount and function
+- dict with fields for name, argcount, function and deterministic
 
 [An example can be found here](../../query#custom-sql-functions).
 
+## expressions
+```yaml
+expressions: list
+```
+
+List of expression shortcuts. Each list element must be a dict with the following fields.
+
+- `name`: name of the expression
+- `expression`: SQL expression, defaults to `name` when empty
+- `index`: if this expression should have a database index, defaults to False when not provided
+
+The expression can be a json data column, sql function or anything that can be run as a SQL snippet.
+
 ## query
 ```yaml
 query:

docs/embeddings/configuration/general.md

@@ -66,13 +66,18 @@ Sets the auto id generation method. When this is not set, an autogenerated numer
 columns:
     text: name of the text column
     object: name of the object column
+    store: limit json data fields to this list of columns
 ```
 
 Sets the `text` and `object` column names. Defaults to `text` and `object` if not provided.
 
+`store` sets a list of columns to store in the JSON data field. When this isn't provided, all columns are stored (default). When `store` is set to `None`, no JSON columns are stored. This is useful is a field is only needed at indexing time but not search time.
+
 ## format
 ```yaml
 format: json|pickle
 ```
 
 Sets the configuration storage format. Defaults to `json`.
+
+Note that `pickle` configuration is deprecated and will raise an error with the default settings. Its only kept for reading legacy indexes and requires setting the `ALLOW_PICKLE` environment variable. Only enable for local and/or trusted sources.

docs/embeddings/configuration/scoring.md

@@ -31,25 +31,30 @@ path: sparse vector model path
 vectormethod: vector embeddings method
 vectornormalize: enable vector embeddings normalization (boolean)
 gpu: boolean|int|string|device
-normalize: enable score normalization (boolean | float)
+normalize: enable score normalization (boolean|float|string|dict)
 batch: Sets the transform batch size
 encodebatch: Sets the encode batch size
 vectors: additional model init args
 encodeargs: additional encode() args
-backend: ivfsparse|pgsparse
+backend: ivfsparse|pgsparse|zvecsparse
 ```
 
 Sparse vector scoring options. The sparse scoring instance combines a sparse vector model with a sparse approximate nearest neighbor index (ANN). This method supports both vector normalization and score normalization.
 
 Vector normalization normalizes all vectors to have a magnitude of 1. By extension, all generated scores will be 0 to 1.
 
-Score normalization scales output scores by a scale factor. When `normalize` is set to `True` it uses a default scale factor. If `normalize` is an integer, then that is used as the scale factor. This scales the scores from 0 to 1.
+Score normalization scales the output between 0 and 1. This setting supports:
+
+- `True` for default scale normalization
+- `float` normalize using this as the scale factor
+- `"bayes"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
 
 #### ivfsparse
 ```yaml
 ivfsparse:
   sample: percent of data to use for model training (0.0 - 1.0)
-  nfeatures: top n features to use for model training (int)
+  nfeatures: top n features to use for model training, defaults to all features (int)
   nlist: desired number of clusters (int)
   nprobe: search probe setting (int)
   minpoints: minimum number of points for a cluster (int)
@@ -61,6 +66,16 @@ Inverted file (IVF) index with flat vector file storage and sparse array support
 
 Sparse ANN backed by Postgres. Supports same options as the [pgvector](../ann/#pgvector) ANN.
 
+#### zvecsparse
+```yaml
+zvecsparse:
+  efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+  m: number of HNSW links per element (int) - defaults to 50
+  efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+Sparse ANN backed by [zvec](https://github.com/alibaba/zvec). Stores sparse vectors in an embedded, path-based vector index. Requires the [ann](../../../install/#ann) extras package.
+
 ## terms
 ```yaml
 terms: boolean|dict
@@ -76,12 +91,34 @@ Supports a `dict` with the parameters `cachelimit` and `cutoff`.
 
 When `terms` is set to `True`, default parameters are used for the `cachelimit` and `cutoff`. Normally, these defaults are sufficient.
 
+### Wildcard queries
+
+Keyword queries containing `*` expand matching terms using SQL `LIKE`. Within these
+expressions, `%` matches any sequence and `_` matches one character. To match a literal
+percent sign, underscore or backslash, use `\%`, `\_` or `\\`, respectively.
+For example, `report\_*` matches `report_2026`, while `report_*` also matches `reportX2026`.
+Use a tokenizer that preserves these characters, such as `tokenizer: {whitespace: true}`,
+or pass pre-tokenized query terms. Queries without `*` retain exact-term matching.
+
 ## normalize
 ```yaml
-normalize: boolean
+normalize: boolean|str|dict
 ```
 
-Enables normalized scoring (ranging from 0 to 1). When enabled, statistics from the index will be used to calculate normalized scores.
+Enables normalized scoring (ranging from 0 to 1). This setting supports:
+
+- `True` for standard score normalization
+- `"bayes"` | `"bb25"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
+
+When standard normalization is enabled, statistics from the index are used to calculate normalized scores.
+When Bayesian/BB25 normalization is enabled, it uses positive-score candidates, dynamic `beta=median(scores)`, adaptive
+`alpha_eff=alpha/std(scores)` and a sigmoid transform (likelihood-only variant with flat prior) to map scores to `[0, 1]`.
+
+Bayesian normalization references:
+
+- [https://github.com/instructkr/bb25](https://github.com/instructkr/bb25)
+- [https://github.com/cognica-io/bayesian-bm25](https://github.com/cognica-io/bayesian-bm25)
 
 ## tokenizer
 ```yaml

docs/embeddings/configuration/vectors.md

@@ -12,7 +12,8 @@ Sets the path for a vectors model. When using a transformers/sentence-transforme
 
 ## method
 ```yaml
-method: transformers|sentence-transformers|llama.cpp|litellm|model2vec|external|words
+method: transformers|sentence-transformers|llama.cpp|litellm|model2vec|external|
+        words
 ```
 
 Embeddings method to use. If the method is not provided, it is inferred using the `path`.
@@ -24,11 +25,12 @@ Embeddings method to use. If the method is not provided, it is inferred using th
 Builds embeddings using a transformers model. While this can be any transformers model, it works best with
 [models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.
 
-`mean`, `cls` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method
-from `transformers` to `meanpooling`, `clspooling` or `latepooling` respectively.
+`mean`, `cls`, `last`, `late` and `max` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method from `transformers` to `meanpooling`, `clspooling`, `lastpooling`, `latepooling` or `maxpooling` respectively.
 
 Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.
 
+Supports loading ONNX models directly. [See this section for more](../../../examples/#model-training).
+
 ### sentence-transformers
 
 Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
@@ -65,7 +67,9 @@ Embeddings are created via an external model or API. Requires setting the [trans
 transform: function
 ```
 
-When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array or list of numpy arrays.
+When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array, list of numpy arrays or list of float arrays.
+
+Note that this method is deprecated and will raise an error with the default settings. Adding a custom `Vectors` implementation is a cleaner approach. The `ALLOW_RESOLVE_TRANSFORM` environment variable must be set to enable. Only enable for trusted and/or reviewed code.
 
 ## gpu
 ```yaml
@@ -104,7 +108,7 @@ quantize: int|boolean
 
 Enables scalar vector quantization at the specified precision. Supports 1-bit through 8-bit quantization. Scalar quantization transforms continuous floating point values to discrete unsigned integers. The `faiss`, `pgvector`, `numpy` and `torch` ANN backends support storing these vectors.
 
-This parameter supports booleans for backwards compatability. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
+This parameter supports booleans for backwards compatibility. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
 
 In addition to vector-level quantization, some ANN backends have the ability to quantize vectors at the storage layer. See the [ANN](../ann) configuration options for more.
 
@@ -140,6 +144,29 @@ vectors: dict
 
 Passes these additional parameters to the underlying vector model.
 
+### center
+```yaml
+center: boolean | dict
+```
+
+Centers normalized late-interaction token vectors and normalizes them again before MUVERA or LEMUR encoding. Centering is applied to
+both query and document vectors. `true` uses batch scope and `false` disables centering. An explicit setting always takes precedence.
+When omitted, a LEMUR artifact with a stored collection mean uses collection scope. Other LEMUR artifacts and late-interaction models
+default to batch scope only when they load more than one `torch.nn.Linear` layer; models with zero or one loaded linear layer keep their
+previous output.
+
+Dictionary configuration supports the following scopes:
+
+```yaml
+center:
+    scope: document | batch | collection
+```
+
+`document` subtracts the mean of each document or query independently. `batch` subtracts the mean of all real token rows in the current
+model batch. Explicit `collection` scope requires a caller-provided, one-dimensional mean vector through either `mean` (an array-like
+value) or `path` (a safetensors file containing a `center.mean` tensor), but not both. LEMUR artifacts can provide this mean
+automatically. Zero-padding rows are never centered and remain zero.
+
 ### muvera
 ```yaml
 muvera:
@@ -149,6 +176,21 @@ muvera:
 ```
 
 Settings to control the size of MUVERA fixed dimensional outputs. Default is 20 * 2^5 * 16 = 10,240 dimensions.
+Set `muvera` to `false` to disable it or `true` to use the default settings. `false` only makes sense when the multi-vector output
+is consumed directly, for example through `PoolingFactory`. Embeddings vectors require `muvera` or `lemur`.
+
+### lemur
+```yaml
+lemur:
+    path: local artifact directory or Hugging Face Hub path
+```
+
+Loads a trained [LEMUR](../../../pipeline/train/lemur) fixed dimensional encoder. LEMUR artifacts are corpus-specific and must be
+created with `LemurTrainer` before configuring an embeddings index. A bare string is treated as the artifact path. Query vectors are
+summed learned features and document vectors are ordinary least squares weights over the artifact's stored token sample. Each artifact
+contains `config.json` and `model.safetensors`; newly trained artifacts also store the collection token mean as `lemur.center`. Loading that mean automatically
+selects collection centering unless `center` is explicitly configured. Artifacts without `lemur.center` retain the previous default.
+The Safetensors file contains inference state only. See the trainer page for score filtering and Faiss exact/IVF search behavior.
 
 ### trust_remote_code
 ```yaml

docs/embeddings/indexing.md

@@ -23,6 +23,8 @@ As mentioned above, computed vectors are stored in an ANN. There are various ind
 
 Embeddings indexes can optionally [store content](../configuration/database#content). When this is enabled, the input content is saved in a database alongside the computed vectors. This enables filtering on additional fields and content retrieval.
 
+The columns used for text, object and JSON data storage are set via [column configuration](../configuration/general#columns).
+
 ## Index vs Upsert
 
 Data is loaded into an index with either an [index](../methods#txtai.embeddings.base.Embeddings.index) or [upsert](../methods#txtai.embeddings.base.Embeddings.upsert) call.

docs/embeddings/query.md

@@ -54,6 +54,8 @@ The number of candidates should be larger than the desired number of results whe
 - For a single query filter clause, the default is the query limit
 - With multiple filtering clauses, the default is 10x the query limit
 
+When a query orders results by score, its `offset` adds to the query limit.
+
 The index name is only applicable when [subindexes](../configuration/general/#indexes) are enabled. This specifies the index to use for the query.
 
 Weights sets the hybrid score weights when an index has both a sparse and dense index.
@@ -132,8 +134,11 @@ embeddings = Embeddings(content=True, objects=True)
 request = open("demo.gif", "rb")
 
 # Insert record
-embeddings.index([("txtai", {"text": "txtai executes machine-learning workflows.",
-                             "object": request.read()})])
+embeddings.index([(
+  "txtai",
+  {"text": "txtai executes machine-learning workflows.",
+   "object": request.read()}
+)])
 
 # Query txtai and get associated object
 query = "SELECT object FROM txtai WHERE similar('machine learning') LIMIT 1"
@@ -169,6 +174,26 @@ LIMIT 1
 """)
 ```
 
+## Expressions
+
+Expression shortcuts expand into more complex SQL snippets. This is useful for making SQL queries more concise. Indexing is also available on expressions as a performance improvement.
+
+The following example indexes a json extraction field (`filepath`) and the length of each field.
+
+```python
+# Create embeddings index
+embeddings = Embeddings(
+  path="sentence-transformers/nli-mpnet-base-v2",
+  content=True,
+  expressions=[
+    {"name": "filepath", "index": True},
+    {"name": "textlength", "expression": "length(text)", "index": True}
+  ]
+)
+
+embeddings.search("SELECT textlength, filepath FROM txtai LIMIT 1")
+```
+
 ## Query translation
 
 Natural language queries with filters can be converted to txtai-compatible SQL statements with query translation. For example:
@@ -192,7 +217,9 @@ When an embeddings database has both a sparse and dense index, both indexes will
 
 ```python
 embeddings.search("query", weights=0.5)
-embeddings.search("SELECT id, text, score FROM txtai WHERE similar('query', 0.5)")
+embeddings.search(
+  "SELECT id, text, score FROM txtai WHERE similar('query', 0.5)"
+)
 ```
 
 ## Graph search
@@ -203,11 +230,20 @@ Additionally, standard embeddings searches can be returned as graphs.
 
 ```python
 # Find all paths between id: 0 and id: 5 between 1 and 3 hops away
-embeddings.graph.search("""
+embeddings.search("""
 MATCH P=({id: 0})-[*1..3]->({id: 5})
 RETURN P
 """)
 
+# Find related nodes for query matches
+embeddings.search("""
+  MATCH P=(A)-[]->(B)
+  WHERE SIMILAR(A, "query")
+  RETURN B
+  ORDER BY A.score DESC
+  LIMIT 10
+""")
+
 # Standard embeddings search as graph
 embeddings.search("query", graph=True)
 ```
@@ -217,13 +253,28 @@ embeddings.search("query", graph=True)
 Subindexes can be queried as follows:
 
 ```python
+# Build index with subindexes
+embeddings = Embeddings(
+  content=True,
+  defaults=False,
+  indexes={
+    "keyword": {
+      "keyword": True
+    },
+    "dense":{
+      "dense": True
+    }
+  }
+)
+embeddings.index(stream())
+
 # Query with index parameter
-embeddings.search("query", index="subindex1")
+embeddings.search("query", index="keyword")
 
 # Specify with SQL
 embeddings.search("""
 SELECT id, text, score FROM txtai
-WHERE similar('query', 'subindex1')
+WHERE similar('query', 'keyword')
 """)
 ```
 

docs/examples.md

@@ -52,6 +52,7 @@ Autonomous agents, retrieval augmented generation (RAG), chat with your data, pi
 | [OpenCode as a txtai LLM](https://github.com/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) | Integrate OpenCode with the txtai ecosystem | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) |
 | [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify a list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
 | [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 
 ## Pipelines
 
@@ -98,6 +99,7 @@ Train, distill, fine-tune and export models.
 | [Train a QA model](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | Build and fine-tune question-answering models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
 | [Train a language model from scratch](https://github.com/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) | Build new language models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) |
 | [Distilling Knowledge into Tiny LLMs](https://github.com/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) [▶️](https://www.youtube.com/watch?v=Ol560ktgkf0) | Finetune tiny LLMs to enable inference using less resources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) |
+| [Progressive Distillation](https://github.com/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) | Incrementally compress knowledge from a series of teachers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) |
 | [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
 | [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
 

docs/further.md

@@ -4,6 +4,7 @@
 ![further](images/further-dark.png#only-dark)
 
 - [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
 - [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)

docs/index.md

@@ -51,7 +51,7 @@ Summary of txtai features:
 - 🔋 Batteries included with defaults to get up and running fast
 - ☁️ Run local or scale out with container orchestration
 
-txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
 !!! note
 

docs/install.md

@@ -9,7 +9,7 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
 ## Optional dependencies
 
@@ -130,7 +130,11 @@ Additional environment specific prerequisites are below.
 
 ### Linux
 
-The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library. The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.
+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library.
+
+The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.
+
+The LiteRT LLM pipeline requires libegl1, libgles2 and libvulkan1.
 
 ### macOS
 
@@ -146,7 +150,7 @@ The [txtai build workflow](https://github.com/neuml/txtai/blob/master/.github/wo
 
 ## CPU-only
 
-The default install adds PyTorch with GPU support. There are a number of dependencies that come with that. When running in a CPU-only environment or using Embeddings/LLM models without PyTorch (i.e. llama.cpp or API services), the CPU-only PyTorch package can be installed with txtai as follows.
+The default install adds PyTorch with GPU support. There are a number of dependencies that come with that. When running in a CPU-only environment, the CPU-only PyTorch package can be installed with txtai as follows.
 
 ```
 pip install txtai torch==[version]+cpu \
@@ -173,6 +177,22 @@ A [community-supported txtai package](https://anaconda.org/conda-forge/txtai) is
 conda install -c conda-forge txtai
 ```
 
+## Minimal install
+
+A lightweight zero dependency minimal install package is available. This is helpful if the default libraries are too heavy or there are no plans to use them.
+
+All the same extras mentioned above are available in this version of the package.
+
+```
+# Lightweight minimal install
+pip install txtai_minimal
+
+# Same as standard `txtai` install
+pip install txtai_minimal[default]
+```
+
+Note that in order to use the `Embeddings` or `LLM` interface without `torch` requires either `llama.cpp`, `litellm` or `litert`.
+
 ## Run with containers
 
 Docker images are available for txtai. [See this section](../cloud) for more information on container-based installs.

docs/models.md

@@ -8,9 +8,9 @@ See the table below for the current recommended models. These models all allow c
 | ---------------------------------------------------- | ------------------------------------------------------------------------ |
 | [Embeddings](../embeddings)                          | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
 | [Image Captions](./pipeline/image/caption.md)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
-| [Labels - Zero Shot](./pipeline/text/labels.md)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Zero Shot](./pipeline/text/labels.md)      | [DeBERTa v3 Zeroshot](https://hf.co/MoritzLaurer/deberta-v3-base-zeroshot-v2.0-c)                     |
 | [Labels - Fixed](./pipeline/text/labels.md)          | Fine-tune with [training pipeline](./pipeline/train/trainer.md)          |
-| [Large Language Model (LLM)](./pipeline/text/llm.md) | [gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b)                 |
+| [Large Language Model (LLM)](./pipeline/text/llm.md) | [Gemma 4 31B](https://hf.co/google/gemma-4-31B)                          |
 | [Summarization](./pipeline/text/summary.md)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
 | [Text-to-Speech](./pipeline/audio/texttospeech.md)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
 | [Transcription](./pipeline/audio/transcription.md)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
@@ -22,6 +22,5 @@ See the following links to learn more.
 
 - [Hugging Face Tasks](https://hf.co/tasks)
 - [Hugging Face Model Hub](https://hf.co/models)
-- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
-- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
-- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+- [Embeddings Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LLM Leaderboard](https://hf.co/spaces/lmarena-ai/arena-leaderboard)

docs/pipeline/data/filetohtml.md

@@ -17,6 +17,12 @@ Apache Tika requires [Java](https://en.wikipedia.org/wiki/Java_(programming_lang
 
 See [this link](https://github.com/DS4SD/docling?tab=readme-ov-file#features) for a list of supported document formats.
 
+## LiteParse
+
+[LiteParse](https://github.com/run-llama/liteparse) is a fast, helpful, and open-source document parser. It supports PDF out of the box with additional format support via third-party packages.
+
+See [this link](https://github.com/run-llama/liteparse#multi-format-input-support) for more.
+
 ## Example
 
 The following shows a simple example using this pipeline.

docs/pipeline/data/textractor.md

@@ -16,6 +16,8 @@ The [backend](../filetohtml/#txtai.pipeline.FileToHTML.__init__) parameter sets
 
 See the [FiletoHTML](../filetohtml) and [HTMLToMarkdown](../htmltomd) pipelines to learn more on the dependencies necessary for each of those pipelines.
 
+Note that the default parameters enable access to all local files and all URLs. The `safeopen` parameter limits this to only files in a specified directory (defaults to temp directory) and public URLs. When running the textractor pipeline through an Application or via the API, `safeopen` defaults to True. It can be disabled if desired via configuration.
+
 ## Example
 
 The following shows a simple example using this pipeline.

docs/pipeline/data/tokenizer.md

@@ -25,6 +25,16 @@ tokenizer("text to tokenize")
 # Tokenize using a regular expression
 tokenizer = Tokenizer(regexp=r"\w{5,}")
 tokenizer("text to tokenize")
+
+# Tokenize into trigrams like pg_trgm
+tokenizer = Tokenizer(ngrams={
+  "ngrams": 3, "lpad": "  ", "rpad": " ", "unique": True
+})
+tokenize("text to tokenize")
+
+# Tokenize into edge ngrams
+tokenizer = Tokenizer(ngrams={"nmin": 2, "nmax": 5, "edge": True})
+tokenizer("text to tokenize")
 ```
 
 ## Configuration-driven example

docs/pipeline/data/urlretrieve.md

@@ -0,0 +1,62 @@
+# URL Retrieve
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+The URL Retrieve pipeline retrieves content from a HTTP(s) URL.
+
+## Example
+
+The following shows a simple example using this pipeline.
+
+```python
+from txtai.pipeline import URLRetrieve
+
+# Create and run pipeline
+urlretrieve = URLRetrieve()
+urlretrieve("https://github.com/neuml/txtai")
+```
+
+## Configuration-driven example
+
+Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
+
+### config.yml
+```yaml
+# Create pipeline using lower case class name
+urlretrieve:
+
+# Run pipeline with workflow
+workflow:
+  retrieve:
+    tasks:
+      - action: urlretrieve
+```
+
+### Run with Workflows
+
+```python
+from txtai import Application
+
+# Create and run pipeline with workflow
+app = Application("config.yml")
+list(app.workflow("urlretrieve", ["https://github.com/neuml/txtai"]))
+```
+
+### Run with API
+
+```bash
+CONFIG=config.yml uvicorn "txtai.api:app" &
+
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"retrieve", "elements":["http://github.com/neuml/txtai"]}'
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.URLRetrieve.__init__
+### ::: txtai.pipeline.URLRetrieve.__call__

docs/pipeline/index.md

@@ -27,6 +27,7 @@ The `LLM` and `RAG` pipelines also have integrations for [llama.cpp](https://git
     - [Tabular](data/tabular)
     - [Text extraction](data/textractor)
     - [Tokenizer](data/tokenizer)
+    - [URLRetrieve](data/urlretrieve)
 - Image
     - [Caption](image/caption)
     - [Image Hash](image/imagehash)

docs/pipeline/llm/llm.md

@@ -63,9 +63,14 @@ The LLM pipeline automatically detects the underlying LLM framework. This can al
 - [Hugging Face Transformers](https://github.com/huggingface/transformers)
 - [llama.cpp](https://github.com/abetlen/llama-cpp-python)
 - [LLM APIs via LiteLLM](https://github.com/BerriAI/litellm)
+- [LiteRT-LM](https://ai.google.dev/edge/litert-lm)
 - [OpenCode server](https://github.com/anomalyco/opencode)
 
-`llama.cpp` models support both local and remote GGUF paths on the HF Hub. See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models. See the [OpenCode documentation](https://opencode.ai/docs/server/) for more on how to integrate the LLM pipeline with a running OpenCode instance.
+`llama.cpp` and `LiteRT-LM` support both local and remote file paths on the HF Hub.
+
+See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models.
+
+See the [OpenCode documentation](https://opencode.ai/docs/server/) for more on how to integrate the LLM pipeline with a running OpenCode instance.
 
 ```python
 from txtai import LLM
@@ -83,6 +88,15 @@ llm = LLM("unsloth/gpt-oss-20b-GGUF/gpt-oss-20b-Q4_K_M.gguf",
 llm = LLM("ollama/gpt-oss")
 llm = LLM("ollama/gpt-oss", method="litellm")
 
+# LiteRT-LM
+llm = LLM(
+  "litert-community/gemma-4-E2B-it-litert-lm/gemma-4-E2B-it.litertlm"
+)
+llm = LLM(
+  "litert-community/gemma-4-E2B-it-litert-lm/gemma-4-E2B-it.litertlm",
+  method="litert"
+)
+
 # Custom Ollama endpoint
 llm = LLM("ollama/gpt-oss", api_base="http://localhost:11434")
 
@@ -145,6 +159,7 @@ See the links below for more detailed examples.
 | [OpenCode as a txtai LLM](https://github.com/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) | Integrate OpenCode with the txtai ecosystem | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) |
 | [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify a list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
 | [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 
 ## Configuration-driven example
 

docs/pipeline/train/lemur.md

@@ -0,0 +1,98 @@
+# LemurTrainer
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+Trains a LEMUR fixed dimensional encoder for a late interaction model and corpus. Training is a separate pipeline step; the saved
+artifact is then loaded through the embeddings `vectors.lemur` configuration.
+
+`LemurTrainer` owns training-data construction, feature-model setup, epoch training and validation-based selection. The loaded LEMUR
+pooler remains responsible for query-feature and document-weight encoding.
+
+## Example
+
+```python
+from txtai.pipeline import LemurTrainer
+
+corpus = [
+    "First document",
+    "Second document",
+    "Third document",
+]
+
+trainer = LemurTrainer()
+trainer(
+    "colbert-ir/colbertv2.0",
+    corpus,
+    "lemur-model",
+    epochs=100,
+    validationsplit=0.1,
+)
+```
+
+`epochs` is required so the training mode is explicit. Use `epochs=100` for the quality-oriented MLP setting. This can take hours on
+a CPU. Use `epochs=0` to select deterministic random ELM features as a lower-cost fallback.
+
+The learn distribution defaults to `learncategory="query"`. Target documents are always encoded with the data encoder, so the
+default encodes selected corpus texts once as data and once as queries. Set `learncategory="data"` to reuse the data encodings, or
+pass a separate iterable of texts with `learn`.
+
+When `vectors.center` is omitted, the trainer encodes the corpus without centering, computes one mean over all corpus token rows and
+uses that mean to center both data and learn vectors before fitting. The collection mean is stored in the LEMUR artifact. An explicit
+`vectors.center` setting keeps its configured behavior and does not store an automatic collection mean.
+
+Set `corpussubsetsize` to a positive integer to sample that many raw corpus texts under `seed` before either encoding pass. The
+default is `None`, which uses every corpus text. `trainsubsetsize` and `learnsubsetsize` apply later, after token vectors have already
+been created. `corpussubsetsize` applies to `data`; a separate `learn` iterable remains caller-sized.
+
+For trained MLP features, set `validationsplit` to a fraction greater than zero and less than one to retain the epoch with the lowest
+held-out loss. The default is `0.0`, which preserves training-loss selection. The selected one-based epoch, loss and metric are
+available as `selectedepoch`, `selectedloss` and `selectionmetric` on the fitted or reloaded encoder.
+
+MLP training displays a `tqdm` progress bar on an interactive terminal, including percent complete and the current validation loss
+when `validationsplit` is enabled (or training loss otherwise). Progress output is disabled for non-interactive runs.
+
+The artifact contains `config.json` and `model.safetensors`. It stores the inference feature model, output-normalization statistics,
+token sample needed to encode documents added after training and, by default, the collection token mean. The training-only output
+readout is not saved.
+
+Load the artifact in an embeddings configuration.
+
+```yaml
+embeddings:
+    path: colbert-ir/colbertv2.0
+    vectors:
+        lemur:
+            path: lemur-model
+```
+
+## Search behavior
+
+Loading a LEMUR artifact with a stored collection mean automatically centers both query and document token vectors with that mean.
+This keeps training and search on the same representation and makes fixed vectors independent of indexing or query batch size. An
+explicit `vectors.center` setting takes precedence. Older artifacts without a stored mean use the standard late-pooling fallback:
+batch centering for models with multiple linear layers and no centering for models with zero or one linear layer.
+
+LEMUR approximates standardized MaxSim targets, so useful ranking scores can be negative. txtai's dense vector path L2-normalizes the
+fixed vectors and removes results with scores less than or equal to zero. This can change ordering and return fewer than the requested
+number of candidates compared with raw maximum inner-product search.
+
+The Faiss backend uses exact `IDMap,Flat` search through 5,000 rows and switches to an IVF index above that threshold. In the measured
+scifact run, default IVF reduced LEMUR NDCG@10 by 43% relative to exact search, compared with 25% for MUVERA. For LEMUR corpora above
+5,000 rows, either pin `faiss.components` to an exact index or tune the IVF settings for the corpus. The exact configuration is:
+
+```yaml
+embeddings:
+    path: colbert-ir/colbertv2.0
+    vectors:
+        lemur:
+            path: lemur-model
+    faiss:
+        components: IDMap,Flat
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.LemurTrainer.__call__

docs/usecases.md

@@ -48,8 +48,9 @@ Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/ma
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 | [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
 
 ### Retrieval augmented generation

docs/workflow/task/console.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: console
+  name:
+    tasks:
+      - task: console
 ```
 
 ## Methods

docs/workflow/task/export.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: export
+  name:
+    tasks:
+      - task: export
 ```
 
 ## Methods

docs/workflow/task/file.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: file
+  name:
+    tasks:
+      - task: file
 ```
 
 ## Methods

docs/workflow/task/image.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: image
+  name:
+    tasks:
+      - task: image
 ```
 
 ## Methods

docs/workflow/task/index.md

@@ -35,12 +35,26 @@ Tasks can also be created with configuration as part of a workflow.
 
 ```yaml
 workflow:
-  tasks:
-    - action: summary 
+  name:
+    tasks:
+      - action: summary
 ```
 
 ::: txtai.workflow.Task.__init__
 
+## Multi-action task inputs
+
+A multi-action `Task` requires reusable inputs. Passing a single-pass iterator, such as a generator or the output of a `StreamTask`, raises a `ValueError` before consuming inputs or running any actions. Automatically buffering an iterator could exhaust memory, so tasks do not convert these inputs to a list.
+
+Iterators passed to a `Workflow` remain supported: the workflow collects bounded batches before passing them to its tasks. For example:
+
+```python
+workflow = Workflow([Task([list, list])], batch=2)
+list(workflow(iter([1, 2, 3])))  # [(1, 1), (2, 2), (3, 3)]
+```
+
+This batching occurs at the workflow input; it does not batch iterator outputs produced by tasks inside that workflow. Single-action tasks continue to accept iterator inputs.
+
 ## Multi-action task concurrency
 
 The default processing mode is to run actions sequentially. Multiprocessing support is already built in at a number of levels. Any of the GPU models will maximize GPU utilization for example and even in CPU mode, concurrency is utilized. But there are still use cases for task action concurrency. For example, if the system has multiple GPUs, the task runs external sequential code, or the task has a large number of I/O tasks.
@@ -72,7 +86,7 @@ workflow = Workflow([Task(lambda x: [y * 3 for y in x], unpack=False, column=0)]
 list(workflow([(2, 8)]))
 ```
 
-For the example input tuple of (2, 2), the workflow will only select the first element (2) and run the task against that element. 
+For the example input tuple of (2, 8), the workflow will only select the first element (2) and run the task against that element.
 
 ```python
 workflow = Workflow([Task([lambda x: [y * 3 for y in x], 

docs/workflow/task/retrieve.md

@@ -22,9 +22,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: retrieve
-      directory: /tmp
+  name:
+    tasks:
+      - task: retrieve
+        directory: /tmp
 ```
 
 ## Methods

docs/workflow/task/service.md

@@ -22,9 +22,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: service
-      url: https://service.url/action
+  name:
+    tasks:
+      - task: service
+        url: https://service.url/action
 ```
 
 ## Methods

docs/workflow/task/storage.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: storage
+  name:
+    tasks:
+      - task: storage
 ```
 
 ## Methods

docs/workflow/task/template.md

@@ -23,9 +23,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: template
-      template: This is a {text} task
+  name:
+    tasks:
+      - task: template
+        template: This is a {text} task
 ```
 
 ## Methods

docs/workflow/task/url.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: url
+  name:
+    tasks:
+      - task: url
 ```
 
 ## Methods

examples/01_Introducing_txtai.ipynb

@@ -27,7 +27,7 @@
         "- 🔋 Batteries included with defaults to get up and running fast\n",
         "- ☁️ Run local or scale out with container orchestration\n",
         "\n",
-        "txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.\n",
+        "txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.\n",
         "\n",
         "> [NeuML](https://neuml.com) is the company behind txtai and we provide AI consulting services around our stack. [Schedule a meeting](https://cal.com/neuml/intro) or [send a message](mailto:[email protected]) to learn more.\n",
         ">\n",

examples/52_Build_RAG_pipelines_with_txtai.ipynb

@@ -125,7 +125,7 @@
             "## Install\n",
             "The easiest way to install is via pip and PyPI\n",
             "pip install txtai\n",
-            "Python 3.10+ is supported. Using a Python virtual environment is **recommended** .\n",
+            "Python 3.11+ is supported. Using a Python virtual environment is **recommended** .\n",
             "See the detailed install instructions for more information covering optional dependencies, environment specific prerequisites, installing from source, conda support and how to run with containers.\n",
             "\n",
             "\n",
@@ -525,8 +525,8 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "ANSWER: Python 3.10 and later versions (Python 3.10+) are supported.\n",
-            "CITATION: [{'id': '24', 'text': 'Python 3.10+ is supported. Using a Python virtual environment is recommended.'}]\n"
+            "ANSWER: Python 3.11 and later versions (Python 3.11+) are supported.\n",
+            "CITATION: [{'id': '24', 'text': 'Python 3.11+ is supported. Using a Python virtual environment is recommended.'}]\n"
           ]
         }
       ],

examples/73_Chunking_your_data_for_RAG.ipynb

@@ -100,12 +100,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'indexid': 0, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '**GitHub - neuml/txtai: 💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows**\\n\\n*💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows - neuml/txtai*\\n\\n\\n\\n**All-in-one embeddings database** \\ntxtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.\\n\\nEmbeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases.\\n\\nThis foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.\\n\\nBuild autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.\\n\\nSummary of txtai features:\\n\\n- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing\\n- 📄 Create embeddings for text, documents, audio, images and video\\n- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more\\n- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.\\n- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems\\n- ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js) , [Java](https://github.com/neuml/txtai.java) , [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go) .\\n- 🔋 Batteries included with defaults to get up and running fast\\n- ☁️ Run local or scale out with container orchestration\\ntxtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers) , [Sentence Transformers](https://github.'}\n",
+      "{'indexid': 0, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '**GitHub - neuml/txtai: 💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows**\\n\\n*💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows - neuml/txtai*\\n\\n\\n\\n**All-in-one embeddings database** \\ntxtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.\\n\\nEmbeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases.\\n\\nThis foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.\\n\\nBuild autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.\\n\\nSummary of txtai features:\\n\\n- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing\\n- 📄 Create embeddings for text, documents, audio, images and video\\n- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more\\n- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.\\n- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems\\n- ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js) , [Java](https://github.com/neuml/txtai.java) , [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go) .\\n- 🔋 Batteries included with defaults to get up and running fast\\n- ☁️ Run local or scale out with container orchestration\\ntxtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers) , [Sentence Transformers](https://github.'}\n",
       "{'indexid': 1, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi) . txtai is open-source under an Apache 2.0 license.\\n\\n*Interested in an easy and secure way to run hosted txtai applications? Then join the* [txtai.cloud](https://txtai.cloud) *preview to learn more.* \\n\\n## Why txtai?\\nNew vector databases, LLM frameworks and everything in between are sprouting up daily. Why build with txtai?\\n\\n- Up and running in minutes with [pip](https://neuml.github.io/txtai/install/) or [Docker](https://neuml.github.io/txtai/cloud/) \\n```\\n# Get started in a couple lines\\nimport txtai\\n\\nembeddings = txtai.Embeddings()\\nembeddings.index([\"Correct\", \"Not what we hoped\"])\\nembeddings.search(\"positive\", 1)\\n#[(0, 0.29862046241760254)]\\n```\\n\\n- Built-in API makes it easy to develop applications using your programming language of choice\\n```\\n# app.yml\\nembeddings:\\n path: sentence-transformers/all-MiniLM-L6-v2\\n```\\n\\n```\\nCONFIG=app.yml uvicorn \"txtai.api:app\"\\ncurl -X GET \"http://localhost:8000/search?query=positive\"\\n```\\n\\n- Run local - no need to ship data off to disparate remote services\\n- Work with micromodels all the way up to large language models (LLMs)\\n- Low footprint - install additional dependencies and scale up when needed\\n- [Learn by example](https://neuml.github.io/txtai/examples) - notebooks cover all available functionality\\n\\n## Use Cases\\nThe following sections introduce common txtai use cases. A comprehensive set of over 60 [example notebooks and applications](https://neuml.github.io/txtai/examples) are also available.\\n\\n\\n### Semantic Search\\nBuild semantic/similarity/vector/neural search applications.'}\n",
       "{'indexid': 2, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'Traditional search systems use keywords to find data. Semantic search has an understanding of natural language and identifies results that have the same meaning, not necessarily the same keywords.\\n\\nGet started with the following examples.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |Overview of the functionality provided by txtai||\\n|[Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |Embed images and text into the same space for search||\\n|[Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |Question matching with semantic search||\\n|[Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |Explore topics, data connectivity and run network analysis||\\n\\n### LLM Orchestration\\nAutonomous agents, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).\\n\\nSee below to learn more.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |Build model prompts and connect tasks together with workflows||\\n|[Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |Integrate llama.cpp, LiteLLM and custom generation frameworks||\\n|[Build knowledge graphs with LLMs](https://github.'}\n",
       "{'indexid': 3, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |Build knowledge graphs with LLM-driven entity extraction||\\n\\n#### Agents\\nAgents connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems.\\n\\ntxtai agents are built on top of the Transformers Agent framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM).\\n\\nSee the link below to learn more.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |Explore a rich dataset with Graph Analysis and Agents||\\n|[Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |Agents that iteratively solve problems as they see fit||\\n|[Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |Exploring how to improve social media engagement with AI||\\n\\n#### Retrieval augmented generation\\nRetrieval augmented generation (RAG) reduces the risk of LLM hallucinations by constraining the output with a knowledge base as context. RAG is commonly used to \"chat with your data\".\\n\\nA novel feature of txtai is that it can provide both an answer and source citation.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Build RAG pipelines with txtai](https://github.'}\n",
       "{'indexid': 4, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |Guide on retrieval augmented generation including how to create citations||\\n|[How RAG with txtai works](https://github.com/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) |Create RAG processes, API services and Docker instances||\\n|[Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |Graph path traversal to collect complex sets of data for advanced RAG||\\n|[Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |Full cycle speech to speech workflow with RAG||\\n\\n### Language Model Workflows\\nLanguage model workflows, also known as semantic workflows, connect language models together to build intelligent applications.\\n\\nWhile LLMs are powerful, there are plenty of smaller, more specialized models that work better and faster for specific tasks. This includes models for extractive question-answering, automatic summarization, text-to-speech, transcription and translation.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |Simple yet powerful constructs to efficiently process data||\\n|[Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |Run abstractive text summarization||\\n|[Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.'}\n",
-      "{'indexid': 5, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'ipynb) |Convert audio files to text||\\n|[Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |Streamline machine translation and language detection||\\n\\n## Installation\\nThe easiest way to install is via pip and PyPI\\n\\n```\\npip install txtai\\n```\\n\\nPython 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.\\n\\nSee the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies) , [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) , [installing from source](https://neuml.github.io/txtai/install/#install-from-source) , [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud) .\\n\\n\\n## Model guide\\nSee the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.\\n\\n|Component|Model(s)|\\n|---|---|\\n|[Embeddings](https://neuml.github.io/txtai/embeddings) |[all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) |\\n|[Image Captions](https://neuml.github.io/txtai/pipeline/image/caption) |[BLIP](https://hf.co/Salesforce/blip-image-captioning-base) |'}\n",
+      "{'indexid': 5, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'ipynb) |Convert audio files to text||\\n|[Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |Streamline machine translation and language detection||\\n\\n## Installation\\nThe easiest way to install is via pip and PyPI\\n\\n```\\npip install txtai\\n```\\n\\nPython 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.\\n\\nSee the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies) , [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) , [installing from source](https://neuml.github.io/txtai/install/#install-from-source) , [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud) .\\n\\n\\n## Model guide\\nSee the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.\\n\\n|Component|Model(s)|\\n|---|---|\\n|[Embeddings](https://neuml.github.io/txtai/embeddings) |[all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) |\\n|[Image Captions](https://neuml.github.io/txtai/pipeline/image/caption) |[BLIP](https://hf.co/Salesforce/blip-image-captioning-base) |'}\n",
       "{'indexid': 6, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '|[Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels) |[BART-Large-MNLI](https://hf.co/facebook/bart-large) |\\n|[Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels) |Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer) |\\n|[Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) |[Llama 3.1 Instruct](https://hf.co/meta-llama/Llama-3.1-8B-Instruct) |\\n|[Summarization](https://neuml.github.io/txtai/pipeline/text/summary) |[DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6) |\\n|[Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech) |[ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx) |\\n|[Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription) |[Whisper](https://hf.co/openai/whisper-base) |\\n|[Translation](https://neuml.github.io/txtai/pipeline/text/translation) |[OPUS Model Series](https://hf.co/Helsinki-NLP) |\\nModels can be loaded as either a path from the Hugging Face Hub or a local directory. Model paths are optional, defaults are loaded when not specified. For tasks with no recommended model, txtai uses the default models as shown in the Hugging Face Tasks guide.\\n\\nSee the following links to learn more.\\n\\n\\n## Powered by txtai\\nThe following applications are powered by txtai.'}\n",
       "{'indexid': 7, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': \"|Application|Description|\\n|---|---|\\n|[rag](https://github.com/neuml/rag) |Retrieval Augmented Generation (RAG) application|\\n|[ragdata](https://github.com/neuml/ragdata) |Build knowledge bases for RAG|\\n|[paperai](https://github.com/neuml/paperai) |Semantic search and workflows for medical/scientific papers|\\n|[annotateai](https://github.com/neuml/annotateai) |Automatically annotate papers with LLMs|\\nIn addition to this list, there are also many other [open-source projects](https://github.com/neuml/txtai/network/dependents) , [published research](https://scholar.google.com/scholar?q=txtai&hl=en&as_ylo=2022) and closed proprietary/commercial projects that have built on txtai in production.\\n\\n\\n## Further Reading\\n- [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg) \\n- [What's new in txtai 8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.\"}\n",
       "{'indexid': 8, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c) \\n- [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748) \\n\\n## Documentation\\n[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for embeddings, pipelines, workflows, API and a FAQ with common questions/issues is available.\\n\\n\\n## Contributing\\nFor those who would like to contribute to txtai, please see [this guide](https://github.com/neuml/.github/blob/master/CONTRIBUTING.md) .'}\n",

examples/85_Progressive_Distillation.ipynb

@@ -0,0 +1,337 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "40f124c0",
+   "metadata": {},
+   "source": [
+    "# Progressive Distillation\n",
+    "\n",
+    "Now that almost everyone has thought about or is actively integrating AI workflows into their projects, some might ask is this all worth the cost? Many think the current economics of the AI space don't scale and that there will be upward price movement. Others still might not be comfortable with sending their data to remote services for processing. Then there is the crowd that wants to deploy models in small spaces with limited compute.\n",
+    "\n",
+    "Are there ways we can deploy small models locally and run at a lower cost? Yes with [Knowledge Distillation](https://en.wikipedia.org/wiki/Knowledge_distillation). Knowledge distillation can get a bad rap due to it's questionable use in training some Large Language Models (LLMs). But it's a perfectly valid way to transfer performance from a larger model to a smaller one. Especially when both models are yours and/or open. \n",
+    "\n",
+    "This notebook will explore progressive distillation which is a technique to incrementally transfer knowledge from a series of larger teacher models into a smaller student."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4aad867",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "16e2c760",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3a23082",
+   "metadata": {},
+   "source": [
+    "# Setup the Training Pipeline\n",
+    "\n",
+    "The first step we need to do is setup up the training pipeline. We'll use the Hugging Face Training framework to build a series of models.\n",
+    "\n",
+    "The following code establishes a `train` method, `test` method and loads the classification training data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea7ee576",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+    "from txtai.pipeline import HFTrainer, Labels\n",
+    "\n",
+    "def train(teacher, student, distillation, **kwargs):\n",
+    "    trainer = HFTrainer()\n",
+    "\n",
+    "    model = AutoModelForSequenceClassification.from_pretrained(student, trust_remote_code=True)\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(student, trust_remote_code=True)\n",
+    "\n",
+    "    return trainer(\n",
+    "        (model, tokenizer),\n",
+    "        ds[\"train\"],\n",
+    "        columns=(\"sentence\", \"label\"),\n",
+    "        maxlength=maxlength,\n",
+    "        teacher=teacher,\n",
+    "        distillation=distillation,\n",
+    "        **kwargs\n",
+    "    )\n",
+    "\n",
+    "def test(model):\n",
+    "    labels = Labels(model, dynamic=False, trust_remote_code=True)\n",
+    "\n",
+    "    # Determine accuracy on test set\n",
+    "    results = [row[\"label\"] == labels(row[\"sentence\"], max_length=maxlength)[0][0] for row in ds[\"validation\"]]\n",
+    "    print(sum(results) / len(ds[\"validation\"]))\n",
+    "\n",
+    "# Hugging Face dataset\n",
+    "ds = load_dataset(\"nyu-mll/glue\", \"sst2\")\n",
+    "maxlength = 128"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "732fad7d",
+   "metadata": {},
+   "source": [
+    "# Progressive Distillation\n",
+    "\n",
+    "We're going to build a [bert-hash-femto](https://huggingface.co/NeuML/bert-hash-femto) classifier which is a extremely small 250K parameter model. This model was pretrained using the same recipe as [BERT](https://arxiv.org/abs/1810.04805). The paper [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962) established that small models perform better with Knowledge Distillation tasks when they are pretrained. [This article](https://huggingface.co/blog/NeuML/bert-hash-embeddings) is also a good source for more information on the topic.\n",
+    "\n",
+    "The next series of steps will do the following:\n",
+    "\n",
+    "- Train a [BERT Mini](https://huggingface.co/google/bert_uncased_L-4_H-256_A-4) model using a [BERT Large](https://huggingface.co/assemblyai/bert-large-uncased-sst2) teacher\n",
+    "- Train a [BERT Tiny](https://huggingface.co/google/bert_uncased_L-2_H-128_A-2) model using the BERT Mini teacher above\n",
+    "- Train a [BERT Hash Femto](https://huggingface.co/NeuML/bert-hash-femto) model using the BERT Tiny teacher above\n",
+    "\n",
+    "This path goes from a 11M parameter model -> 4.4M parameter model -> 250K parameter model\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "43baccd6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertForSequenceClassification LOAD REPORT\u001b[0m from: google/bert_uncased_L-4_H-256_A-4\n",
+      "Key                                        | Status     | \n",
+      "-------------------------------------------+------------+-\n",
+      "cls.predictions.transform.dense.weight     | UNEXPECTED | \n",
+      "cls.predictions.decoder.bias               | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.bias   | UNEXPECTED | \n",
+      "cls.predictions.decoder.weight             | UNEXPECTED | \n",
+      "cls.seq_relationship.bias                  | UNEXPECTED | \n",
+      "cls.seq_relationship.weight                | UNEXPECTED | \n",
+      "cls.predictions.transform.dense.bias       | UNEXPECTED | \n",
+      "cls.predictions.bias                       | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.weight | UNEXPECTED | \n",
+      "classifier.weight                          | MISSING    | \n",
+      "classifier.bias                            | MISSING    | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.875\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"assemblyai/bert-large-uncased-sst2\",\n",
+    "    \"google/bert_uncased_L-4_H-256_A-4\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=1e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-mini-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9ba3158e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertForSequenceClassification LOAD REPORT\u001b[0m from: google/bert_uncased_L-2_H-128_A-2\n",
+      "Key                                        | Status     | \n",
+      "-------------------------------------------+------------+-\n",
+      "cls.predictions.transform.dense.weight     | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.bias   | UNEXPECTED | \n",
+      "cls.seq_relationship.weight                | UNEXPECTED | \n",
+      "cls.seq_relationship.bias                  | UNEXPECTED | \n",
+      "cls.predictions.transform.dense.bias       | UNEXPECTED | \n",
+      "cls.predictions.bias                       | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.weight | UNEXPECTED | \n",
+      "classifier.weight                          | MISSING    | \n",
+      "classifier.bias                            | MISSING    | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.8325688073394495\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"bert-mini-sst2\",\n",
+    "    \"google/bert_uncased_L-2_H-128_A-2\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=1e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-tiny-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "e98d950e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertHashForSequenceClassification LOAD REPORT\u001b[0m from: neuml/bert-hash-femto\n",
+      "Key                      | Status  | \n",
+      "-------------------------+---------+-\n",
+      "bert.pooler.dense.bias   | MISSING | \n",
+      "bert.pooler.dense.weight | MISSING | \n",
+      "classifier.bias          | MISSING | \n",
+      "classifier.weight        | MISSING | \n",
+      "\n",
+      "Notes:\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.8084862385321101\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"bert-tiny-sst2\",\n",
+    "    \"neuml/bert-hash-femto\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=3e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-femto-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f1c94e7",
+   "metadata": {},
+   "source": [
+    "Let's look at the performance. The 11M parameter model registered an accuracy score of `0.8750` on the SST2 dev set. Then the 4M parameter model scored `0.8326` and finally the tiny femto model scored `0.8085`. As we can see each score got progressively worse but each model has less capability. Let's train a femto model directly to compare."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "b89d8292",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertHashForSequenceClassification LOAD REPORT\u001b[0m from: neuml/bert-hash-femto\n",
+      "Key                      | Status  | \n",
+      "-------------------------+---------+-\n",
+      "bert.pooler.dense.bias   | MISSING | \n",
+      "bert.pooler.dense.weight | MISSING | \n",
+      "classifier.bias          | MISSING | \n",
+      "classifier.weight        | MISSING | \n",
+      "\n",
+      "Notes:\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.801605504587156\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    None,\n",
+    "    \"neuml/bert-hash-femto\",\n",
+    "    None,\n",
+    "    learning_rate=3e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-femto-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4864b8d2",
+   "metadata": {},
+   "source": [
+    "This scored `0.8016` a sizable shift down from the progressively distilled model. Now keep in mind the femto model only has `250K` parameters but we gave it a sizable accuracy boost within it's capabilities."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e4e04226",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This example showed how progressive distillation can boost overall model performance, especially for tiny models. Incrementally compressing knowledge into a series of smaller subsets enabled the final model to learn more efficiently than directly training the model on the dataset without distillation. Put progressive distillation in the toolkit when working with tiny models!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.20"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

mkdocs.yml

@@ -92,6 +92,7 @@ nav:
             - Tabular: pipeline/data/tabular.md
             - Textractor: pipeline/data/textractor.md
             - Tokenizer: pipeline/data/tokenizer.md
+            - URL Retrieve: pipeline/data/urlretrieve.md
         - Image:
             - Caption: pipeline/image/caption.md
             - Image Hash: pipeline/image/imagehash.md
@@ -108,6 +109,7 @@ nav:
             - Translation: pipeline/text/translation.md
         - Train:
             - HF ONNX: pipeline/train/hfonnx.md
+            - LEMUR Trainer: pipeline/train/lemur.md
             - ML ONNX: pipeline/train/mlonnx.md
             - Trainer: pipeline/train/trainer.md
     - Workflow:

setup.py

@@ -1,4 +1,6 @@
-# pylint: disable = C0111
+# pylint: disable = C0103,C0111
+import os
+
 from setuptools import find_packages, setup
 
 with open("README.md", "r", encoding="utf-8") as f:
@@ -6,14 +8,38 @@ with open("README.md", "r", encoding="utf-8") as f:
     DESCRIPTION = "".join([line for line in f if "gh-dark-mode-only" not in line])
 
 # Required dependencies
-install = ["faiss-cpu>=1.7.1.post2", "msgpack>=1.0.7", "torch>=2.4", "transformers>=4.56.2"]
+install = []
 
-# Required dependencies that are also base transformers dependencies
-install += ["huggingface-hub>=0.34.0", "numpy>=1.18.4", "pyyaml>=5.3", "regex>=2022.8.17", "safetensors>=0.4.5"]
+# Default dependencies
+default = [
+    "faiss-cpu>=1.7.1.post2",
+    "huggingface-hub>=0.34.0",
+    "msgpack>=1.0.7",
+    "numpy>=1.18.4",
+    "regex>=2022.8.17",
+    "pyyaml>=5.3",
+    "safetensors>=0.4.5",
+    "torch>=2.4",
+    "tqdm>=4.66.3",
+    "transformers>=5.9.0",
+]
 
 # Optional dependencies
 extras = {}
 
+# Package name
+package = "txtai"
+
+if os.getenv("MINIMAL"):
+    # Add default extra
+    extras["default"] = default
+
+    # Rename this package to have minimal suffix
+    package += "_minimal"
+else:
+    # Default dependencies are required with standard install
+    install += default
+
 # Development dependencies - not included in "all" install
 extras["dev"] = [
     "black",
@@ -27,24 +53,28 @@ extras["dev"] = [
     "pylint",
 ]
 
-extras["agent"] = ["jinja2>=3.1.6", "mcpadapt>=0.1.0", "smolagents>=1.23"]
+extras["agent"] = ["beautifulsoup4>=4.9.3", "jinja2>=3.1.6", "mcp<2.0", "mcpadapt>=0.1.0", "smolagents>=1.23"]
 
 extras["ann"] = [
     "annoy>=1.16.3",
     "bitsandbytes>=0.42.0",
-    "ggml-py>=0.9.4",
+    "ggml-python>=0.0.41",
     "hnswlib>=0.5.0",
+    "milvus-lite>=3.1.1",
     "pgvector>=0.4.1",
+    "rabitqlib>=0.5.1",
     "scikit-learn>=0.23.1",
     "scipy>=1.4.1",
     "sqlalchemy>=2.0.20",
     "sqlite-vec>=0.1.1",
+    "turbovec>=0.7.0",
+    "zvec>=0.5.1",
 ]
 
 extras["api"] = [
     "aiohttp>=3.8.1",
     "fastapi>=0.94.0",
-    "fastapi-mcp>=0.2.0",
+    "fastapi-mcp>=0.4.0",
     "httpx>=0.28.1",
     "pillow>=7.1.2",
     "python-multipart>=0.0.7",
@@ -55,7 +85,7 @@ extras["cloud"] = ["apache-libcloud>=3.3.1", "fasteners>=0.14.1"]
 
 extras["console"] = ["rich>=12.0.1"]
 
-extras["database"] = ["duckdb>=0.7.1", "pillow>=7.1.2", "sqlalchemy>=2.0.20"]
+extras["database"] = ["duckdb>=0.8.0", "pillow>=7.1.2", "sqlalchemy>=2.0.20"]
 
 extras["graph"] = ["grand-cypher>=0.6.0", "grand-graph>=0.6.0", "networkx>=2.7.1", "sqlalchemy>=2.0.20"]
 
@@ -71,13 +101,21 @@ extras["pipeline-audio"] = [
     "webrtcvad-wheels>=2.0.14",
 ]
 
-extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "chonkie>=1.0.2", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]
+extras["pipeline-data"] = [
+    "beautifulsoup4>=4.9.3",
+    "chonkie>=1.0.2",
+    "docling>=2.8.2",
+    "liteparse>=2.1.1",
+    "nltk>=3.5",
+    "pandas>=1.1.0",
+    "tika>=1.24",
+]
 
 extras["pipeline-image"] = ["imagehash>=4.2.1", "pillow>=7.1.2", "timm>=0.4.12"]
 
-extras["pipeline-llm"] = ["httpx>=0.28.1", "litellm>=1.37.16", "llama-cpp-python>=0.2.75"]
+extras["pipeline-llm"] = ["httpx>=0.28.1", "litert-lm-api>=0.11.0", "litellm>=1.37.16", "llama-cpp-python>=0.2.75"]
 
-extras["pipeline-text"] = ["gliner>=0.2.23", "sentencepiece>=0.1.91", "staticvectors>=0.2.0"]
+extras["pipeline-text"] = ["gliner-py>=0.2.27", "sentencepiece>=0.1.91", "staticvectors>=0.2.0"]
 
 extras["pipeline-train"] = [
     "accelerate>=0.26.0",
@@ -101,14 +139,17 @@ extras["pipeline"] = (
 extras["scoring"] = ["sqlalchemy>=2.0.20"]
 
 extras["vectors"] = [
+    "ai-edge-litert>=2.1.5",
     "litellm>=1.37.16",
     "llama-cpp-python>=0.2.75",
     "model2vec>=0.3.0",
+    "onnxruntime>=1.11.0",
     "scikit-learn>=0.23.1",
     "scipy>=1.4.1",
     "sentence-transformers>=5.0.0",
     "skops>=0.9.0",
     "staticvectors>=0.2.0",
+    "tokenizers>=0.22.2",
 ]
 
 extras["workflow"] = [
@@ -139,8 +180,8 @@ extras["all"] = (
 )
 
 setup(
-    name="txtai",
-    version="9.5.0",
+    name=package,
+    version="9.15.0",
     author="NeuML",
     description="All-in-one open-source AI framework for semantic search, LLM orchestration and language model workflows",
     long_description=DESCRIPTION,
@@ -155,7 +196,7 @@ setup(
     packages=find_packages(where="src/python"),
     package_dir={"": "src/python"},
     keywords="search embedding machine-learning nlp",
-    python_requires=">=3.10",
+    python_requires=">=3.11",
     install_requires=install,
     extras_require=extras,
     classifiers=[

src/python/txtai/agent/base.py

@@ -6,7 +6,7 @@ import os
 
 from collections import deque
 
-from jinja2 import Template
+from jinja2.sandbox import SandboxedEnvironment
 
 from .factory import ProcessFactory
 
@@ -48,7 +48,21 @@ class Agent:
         # Agent memory
         self.memory = {}
         self.window = memory
-        self.template = template
+
+        # Create template
+        self.template = SandboxedEnvironment().from_string(
+            template
+            if template
+            else """{{ text }}
+{% if memory %}
+Use the following conversation history to help answer the question above.
+
+{{ memory }}
+
+If the history is irrelevant, forget it and use other tools to answer the question.
+{% endif %}
+"""
+        )
 
     def __call__(self, text, maxlength=8192, stream=False, session=None, reset=False, **kwargs):
         """
@@ -95,20 +109,6 @@ class Agent:
             formatted instructions
         """
 
-        template = (
-            self.template
-            if self.template
-            else """{{ text }}
-{% if memory %}
-Use the following conversation history to help answer the question above.
-
-{{ memory }}
-
-If the history is irrelevant, forget it and use other tools to answer the question.
-{% endif %}
-"""
-        )
-
         # pylint: disable=E1133
         memory = []
         if self.memory.get(session):
@@ -119,7 +119,7 @@ If the history is irrelevant, forget it and use other tools to answer the questi
             memory = "\n\n".join(memory)
 
         # Command template with memory
-        return Template(template).render(text=text, memory=memory)
+        return self.template.render(text=text, memory=memory)
 
     def instructions(self, config):
         """

src/python/txtai/agent/tool/__init__.py

@@ -5,3 +5,10 @@ Tool imports
 from .embeddings import EmbeddingsTool
 from .factory import ToolFactory
 from .function import FunctionTool
+from .bash import BashTool
+from .edit import EditTool
+from .glob import GlobTool
+from .grep import GrepTool
+from .read import ReadTool
+from .skill import SkillTool
+from .todo import TodoWriteTool

src/python/txtai/agent/tool/bash.py

@@ -0,0 +1,67 @@
+"""
+Bash imports
+"""
+
+import subprocess
+
+from smolagents import Tool
+
+
+class BashTool(Tool):
+    """
+    The BashTool runs a command through a subprocess. This tool only allows a small subset of commands.
+    More can be added through configuration.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self, allowed=None, timeout=30):
+        """
+        Creates a BashTool.
+
+        Args:
+            allowed: list of allowed commands to run, has limited set of defaults which are a best effort not a sandbox
+            timeout: maximum number of seconds to wait for a command to complete, defaults to 30, None disables the timeout
+        """
+
+        # Tool parameters
+        self.name = "bash"
+        self.description = "Implementation of a bash shell subprocess tool. Runs a shell command and returns the output."
+        self.inputs = {
+            "command": {"type": "array", "description": "Command to run. Follows Python subprocess.open pattern for command as a list of arguments."}
+        }
+        self.output_type = "any"
+
+        # Default list of allowed commands
+        self.allowed = allowed if allowed is not None else ["cat", "cut", "diff", "grep", "head", "ls", "tail"]
+
+        # Maximum time to wait for a command, None waits indefinitely
+        self.timeout = timeout
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, command):
+        """
+        Runs a shell command as a subprocess.
+
+        stdin is closed and a timeout is applied. Otherwise an allowed command called with no file arguments
+        (i.e. `cat`) reads from stdin and blocks the agent indefinitely.
+
+        Args:
+            command: command arguments as a list
+
+        Returns:
+            command output
+        """
+
+        output = None
+        if command and command[0] in self.allowed:
+            try:
+                output = subprocess.run(
+                    command, capture_output=True, text=True, encoding="utf-8", check=False, stdin=subprocess.DEVNULL, timeout=self.timeout
+                ).stdout
+            except subprocess.TimeoutExpired:
+                output = f"Command timed out after {self.timeout} seconds"
+
+        return output

src/python/txtai/agent/tool/edit.py

@@ -0,0 +1,59 @@
+"""
+Edit imports
+"""
+
+import difflib
+
+from smolagents import Tool
+
+
+class EditTool(Tool):
+    """
+    The EditTool modifies content in a file.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates an EditTool.
+        """
+
+        # Tool parameters
+        self.name = "edit"
+        self.description = "Implementation of a file edit tool. Makes in-place edits to content in a file."
+        self.inputs = {
+            "path": {"type": "string", "description": "Path of file to edit"},
+            "search": {"type": "string", "description": "String to replace"},
+            "replace": {"type": "string", "description": "Replacement string"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path, search, replace):
+        """
+        Modifies content in a file. Returns a diff of the changes, if any.
+
+        Args:
+            path: file to edit
+            search: string to replace
+            replace: replacement
+
+        Returns:
+            diff
+        """
+
+        content = None
+        with open(path, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        # Make edits
+        modified = content.replace(search, replace)
+        if modified != content:
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(modified)
+
+        # Return diff of file edits
+        return "".join(difflib.unified_diff(content.splitlines(True), modified.splitlines(True), path, path))

src/python/txtai/agent/tool/factory.py

@@ -9,13 +9,20 @@ from types import FunctionType, MethodType
 import mcpadapt.core
 
 from mcpadapt.smolagents_adapter import SmolAgentsAdapter
-from smolagents import PythonInterpreterTool, Tool, tool as CreateTool, VisitWebpageTool, WebSearchTool
+from smolagents import PythonInterpreterTool, Tool, tool as CreateTool, UserInputTool, WebSearchTool
 from transformers.utils import chat_template_utils, TypeHintParsingException
 
 from ...embeddings import Embeddings
+from .bash import BashTool
+from .edit import EditTool
 from .embeddings import EmbeddingsTool
 from .function import FunctionTool
+from .glob import GlobTool
+from .grep import GrepTool
+from .read import ReadTool
 from .skill import SkillTool
+from .todo import TodoWriteTool
+from .write import WriteTool
 
 
 class ToolFactory:
@@ -24,7 +31,21 @@ class ToolFactory:
     """
 
     # Default toolkit
-    DEFAULTS = {"python": PythonInterpreterTool(), "websearch": WebSearchTool(), "webview": VisitWebpageTool()}
+    DEFAULTS = {
+        "bash": BashTool,
+        "edit": EditTool,
+        "glob": GlobTool,
+        "grep": GrepTool,
+        "python": PythonInterpreterTool,
+        "question": UserInputTool,
+        "read": ReadTool,
+        "todowrite": TodoWriteTool,
+        "websearch": WebSearchTool,
+        "write": WriteTool,
+    }
+
+    # Backwards compatible mappings
+    DEFAULTS["webview"] = DEFAULTS["read"]
 
     @staticmethod
     def create(config):
@@ -60,7 +81,12 @@ class ToolFactory:
 
             # Get default tool, if applicable
             elif isinstance(tool, str) and tool in ToolFactory.DEFAULTS:
-                tool = ToolFactory.DEFAULTS[tool]
+                tool = ToolFactory.default(tool)
+
+            # Get ALL default tools, if applicable
+            elif isinstance(tool, str) and tool == "defaults":
+                tools.extend(ToolFactory.default(name) for name in ToolFactory.DEFAULTS if name != "webview")
+                tool = None
 
             # Support importing MCP tool collections
             elif isinstance(tool, str) and tool.startswith("http"):
@@ -96,6 +122,20 @@ class ToolFactory:
         except (TypeHintParsingException, TypeError):
             return ToolFactory.fromdocs(target, config if config else {})
 
+    @staticmethod
+    def default(name):
+        """
+        Creates a default tool by alias name.
+
+        Args:
+            name: default tool alias name
+
+        Returns:
+            Tool
+        """
+
+        return ToolFactory.DEFAULTS[name]()
+
     @staticmethod
     def fromdocs(target, config):
         """

src/python/txtai/agent/tool/function.py

@@ -7,8 +7,7 @@ from smolagents import Tool
 
 class FunctionTool(Tool):
     """
-    Creates a FunctionTool. A FunctionTool takes descriptive configuration and injects it along with a target function
-    into an LLM prompt.
+    A FunctionTool takes descriptive configuration and injects it along with a target function into an LLM prompt.
     """
 
     # pylint: disable=W0231

src/python/txtai/agent/tool/glob.py

@@ -0,0 +1,49 @@
+"""
+Glob imports
+"""
+
+import glob as g
+import os
+
+from smolagents import Tool
+
+
+class GlobTool(Tool):
+    """
+    The GlobTool finds matching file patterns.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a GlobTool.
+        """
+
+        # Tool parameters
+        self.name = "glob"
+        self.description = "Implementation of a glob tool. Finds files that match glob patterns."
+        self.inputs = {
+            "files": {"type": "string", "description": "File or file glob pattern to search"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, files):
+        """
+        Lists files matching a glob pattern
+
+        Args:
+            files: files glob pattern
+
+        Returns:
+            list of matching files
+        """
+
+        # Format file pattern
+        files = os.path.join(files, "*") if os.path.isdir(files) else files
+
+        # Iterate over each found file
+        return g.glob(files, recursive=True)

src/python/txtai/agent/tool/grep.py

@@ -0,0 +1,75 @@
+"""
+Grep imports
+"""
+
+import glob
+import os
+import re
+
+from smolagents import Tool
+
+
+class GrepTool(Tool):
+    """
+    The GrepTool "greps" for file matches.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a GrepTool.
+        """
+
+        # Tool parameters
+        self.name = "grep"
+        self.description = (
+            "Implementation of a grep tool. Searches a list of files for matches. "
+            "Returns a dictionary with the file path as the key and list of matches as the value."
+        )
+        self.inputs = {
+            "search": {"type": "string", "description": "Search grep pattern"},
+            "files": {"type": "string", "description": "File or file glob pattern to search"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, search, files):
+        """
+        Greps a set of files for a pattern.
+
+        Args:
+            search: search pattern
+            files: file or file glob pattern to search
+
+        Returns:
+            {path: [matches]}
+        """
+
+        # Compile the search pattern
+        pattern = re.compile(search)
+
+        # Format file pattern
+        files = os.path.join(files, "*") if os.path.isdir(files) else files
+
+        # Iterate over each found file
+        results = {}
+        for path in glob.glob(files, recursive=True):
+            if os.path.isfile(path):
+                try:
+                    with open(path, "r", encoding="utf-8") as f:
+                        # Search for matches line-by-line
+                        matches = []
+                        for line in f:
+                            if pattern.search(line):
+                                matches.append(line)
+
+                        if matches:
+                            results[path] = matches
+
+                except (UnicodeDecodeError, FileNotFoundError):
+                    pass
+
+        return results

src/python/txtai/agent/tool/read.py

@@ -0,0 +1,59 @@
+"""
+Read imports
+"""
+
+import re
+
+from smolagents import Tool
+
+from ...pipeline import Textractor
+
+
+class ReadTool(Tool):
+    """
+    The ReadTool retrieves file or url content. This tool automatically extracts text content from
+    binary files using the Textractor pipeline.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self, maxlength=40000):
+        """
+        Creates a ReadTool.
+
+        Args:
+            maxlength: Truncate content above this maxlength
+        """
+
+        # Tool parameters
+        self.name = "read"
+        self.description = (
+            "Implementation of a file read tool. Returns file content. Also supports reading web content. "
+            "Use this tool to browse webpages in addition to reading files."
+        )
+        self.inputs = {"path": {"type": "string", "description": "File path or url"}}
+        self.output_type = "any"
+
+        # Create textractor instance
+        self.textractor = Textractor()
+        self.maxlength = maxlength
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path):
+        """
+        Reads content from path.
+
+        Args:
+            path: file path or url
+
+        Returns:
+            content
+        """
+
+        content = self.textractor(path)
+        content = re.sub(r"\n{3,}", "\n\n", content)
+
+        # Truncate content to max length
+        return content[: self.maxlength]

src/python/txtai/agent/tool/skill.py

@@ -2,14 +2,17 @@
 Skill imports
 """
 
-import yaml
-
 from smolagents import Tool
 
+# Core library imports
+from ...util import Library
+
+yaml = Library().yaml()
+
 
 class SkillTool(Tool):
     """
-    Creates a SkillTool. A SkillTool loads a skill.md file.
+    A SkillTool loads a skill.md file.
     """
 
     # pylint: disable=W0231

src/python/txtai/agent/tool/todo.py

@@ -0,0 +1,40 @@
+"""
+Todo imports
+"""
+
+from smolagents import Tool
+
+
+class TodoWriteTool(Tool):
+    """
+    The TodoWriteTool plans for task execution.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a TodoWriteTool.
+        """
+
+        # Tool parameters
+        self.name = "todowrite"
+        self.description = (
+            "Implementation of a todo write tool. Generates a structured task list to help organize complex tasks. "
+            "Only use this tool for complex tasks with multiple steps. Skip for simple tasks that can be done easily."
+        )
+        self.inputs = {"plan": {"type": "string", "description": "The task plan"}}
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, plan):
+        """
+        Returns the plan.
+
+        Returns:
+            plan
+        """
+
+        return plan

src/python/txtai/agent/tool/write.py

@@ -0,0 +1,42 @@
+"""
+Write imports
+"""
+
+from smolagents import Tool
+
+
+class WriteTool(Tool):
+    """
+    The WriteTool writes file content.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a WriteTool.
+        """
+
+        # Tool parameters
+        self.name = "write"
+        self.description = "Implementation of a file write tool. Writes content to file."
+        self.inputs = {
+            "path": {"type": "string", "description": "Output file path"},
+            "content": {"type": "string", "description": "File content to write"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path, content):
+        """
+        Writes content to path.
+
+        Args:
+            path: output file path
+            content: content to write
+        """
+
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(content)

src/python/txtai/ann/dense/__init__.py

@@ -6,6 +6,10 @@ from .annoy import Annoy
 from .factory import ANNFactory
 from .faiss import Faiss
 from .hnsw import HNSW
+from .milvus import Milvus
 from .numpy import NumPy
 from .pgvector import PGVector
+from .rabitq import RabitQ
 from .torch import Torch
+from .turbovec import TurboVec
+from .zvec import Zvec

src/python/txtai/ann/dense/factory.py

@@ -4,14 +4,19 @@ Factory module
 
 from ...util import Resolver
 
+from ..base import ANN
 from .annoy import Annoy
-from .faiss import Faiss
+from .faiss import Faiss, FAISS
 from .ggml import GGML
 from .hnsw import HNSW
+from .milvus import Milvus
 from .numpy import NumPy
 from .pgvector import PGVector
+from .rabitq import RabitQ
 from .sqlite import SQLite
 from .torch import Torch
+from .turbovec import TurboVec
+from .zvec import Zvec
 
 
 class ANNFactory:
@@ -31,27 +36,36 @@ class ANNFactory:
             ANN
         """
 
+        # pylint: disable=R0912
         # ANN instance
         ann = None
-        backend = config.get("backend", "faiss")
+        backend = config.get("backend", "faiss" if FAISS else "numpy")
 
         # Create ANN instance
         if backend == "annoy":
             ann = Annoy(config)
         elif backend == "faiss":
             ann = Faiss(config)
-        elif backend == "hnsw":
-            ann = HNSW(config)
         elif backend == "ggml":
             ann = GGML(config)
+        elif backend == "hnsw":
+            ann = HNSW(config)
+        elif backend == "milvus":
+            ann = Milvus(config)
         elif backend == "numpy":
             ann = NumPy(config)
         elif backend == "pgvector":
             ann = PGVector(config)
+        elif backend == "rabitq":
+            ann = RabitQ(config)
         elif backend == "sqlite":
             ann = SQLite(config)
         elif backend == "torch":
             ann = Torch(config)
+        elif backend == "turbovec":
+            ann = TurboVec(config)
+        elif backend == "zvec":
+            ann = Zvec(config)
         else:
             ann = ANNFactory.resolve(backend, config)
 
@@ -74,6 +88,6 @@ class ANNFactory:
         """
 
         try:
-            return Resolver()(backend)(config)
+            return Resolver()(backend, ANN)(config)
         except Exception as e:
             raise ImportError(f"Unable to resolve ann backend: '{backend}'") from e

src/python/txtai/ann/dense/faiss.py

@@ -14,13 +14,22 @@ if platform.system() == "Darwin" or os.name == "nt":
     # Workaround for a Faiss issue with OMP: Error #15. See txtai FAQ for more.
     os.environ["KMP_DUPLICATE_LIB_OK"] = os.environ.get("KMP_DUPLICATE_LIB_OK", "TRUE")
 
-import numpy as np
+# Conditional import
+try:
+    from faiss import index_factory, IO_FLAG_MMAP, METRIC_INNER_PRODUCT, read_index, write_index
+    from faiss import index_binary_factory, read_index_binary, write_index_binary, IndexBinaryIDMap
 
-from faiss import index_factory, IO_FLAG_MMAP, METRIC_INNER_PRODUCT, read_index, write_index
-from faiss import index_binary_factory, read_index_binary, write_index_binary, IndexBinaryIDMap
+    FAISS = True
+except ImportError:
+    FAISS = False
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class Faiss(ANN):
     """
@@ -30,6 +39,9 @@ class Faiss(ANN):
     def __init__(self, config):
         super().__init__(config)
 
+        if not FAISS:
+            raise ImportError("Faiss is not available - install faiss-cpu to enable")
+
         # Scalar quantization
         quantize = self.config.get("quantize")
         self.qbits = quantize if quantize and isinstance(quantize, int) and not isinstance(quantize, bool) else None
@@ -130,7 +142,7 @@ class Faiss(ANN):
 
         # Derive quantization. Prefer backend-specific setting. Fallback to root-level parameter.
         quantize = self.setting("quantize", self.config.get("quantize"))
-        quantize = 8 if isinstance(quantize, bool) else quantize
+        quantize = 8 if isinstance(quantize, bool) and quantize else quantize
 
         # Get storage setting
         storage = f"SQ{quantize}" if quantize else "Flat"
@@ -217,6 +229,7 @@ class Faiss(ANN):
         count = self.count()
 
         default = 6 if count <= 5000 else round(self.cells(count) / 16)
+        default = min(default, getattr(self.backend, "nlist", default))
         return self.setting("nprobe", default)
 
     def scores(self, scores):

src/python/txtai/ann/dense/ggml.py

@@ -5,8 +5,6 @@ GGML module
 import ctypes
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     import ggml
@@ -18,6 +16,11 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class GGML(ANN):
     """
@@ -152,7 +155,7 @@ class GGMLTensors:
         gguf = ggml.gguf_init_from_file(path.encode("utf-8"), params)
 
         # Load tensors from GGUF file
-        self.loadtensors(context)
+        self.loadtensors(gguf, context)
 
         # Create graph operation
         self.graph, self.output = self.creategraph()
@@ -215,11 +218,12 @@ class GGMLTensors:
             ids: ids to delete
         """
 
-        shape = utils.get_shape(self.data)
+        rows = self.rows()
 
-        # Filter any index greater than size of array
-        ids = [x for x in ids if x < shape[1]]
-        self.deletes.extend(ids)
+        # Set index ids as deleted, ignoring ids outside the array and ids already marked deleted
+        for x in ids:
+            if x < rows and x not in self.deletes:
+                self.deletes.append(x)
 
     def search(self, queries):
         """
@@ -247,7 +251,7 @@ class GGMLTensors:
             ggml.ggml_backend_graph_compute(self.backend, self.graph)
 
             # Get size of embeddings data
-            size = utils.get_shape(self.data)[1]
+            size = self.rows()
 
             # Get and return results
             results = np.zeros((batch.shape[0], size), dtype=np.float32)
@@ -268,7 +272,7 @@ class GGMLTensors:
             count
         """
 
-        return utils.get_shape(self.data)[1] - len(self.deletes) if self.data else 0
+        return self.rows() - len(self.deletes) if self.data else 0
 
     def save(self, path):
         """
@@ -284,9 +288,12 @@ class GGMLTensors:
         # Init and save data tensor
         gguf = ggml.gguf_init_empty()
 
-        # Add the data tensor
-        ggml.ggml_set_name(self.data, b"data")
-        ggml.gguf_add_tensor(gguf, self.data)
+        # Add the data tensor. GGUF can't store a tensor with zero rows, only save the dimensions for an empty index.
+        if self.rows():
+            ggml.ggml_set_name(self.data, b"data")
+            ggml.gguf_add_tensor(gguf, self.data)
+        else:
+            ggml.gguf_set_val_u32(gguf, b"dimensions", utils.get_shape(self.data)[0])
 
         # Optionally create and add the deletes tensor
         if self.deletes:
@@ -409,11 +416,12 @@ class GGMLTensors:
         # Copy embeddings data
         self.copy(data, self.data, tensortype, 0)
 
-    def loadtensors(self, context):
+    def loadtensors(self, gguf, context):
         """
         Loads existing tensors from context.
 
         Args:
+            gguf: gguf context
             context: ggml context
         """
 
@@ -432,6 +440,10 @@ class GGMLTensors:
 
             # Copy tensor data to backend
             ggml.ggml_backend_tensor_set(self.data, ggml.ggml_get_data(data), 0, ggml.ggml_nbytes(data))
+        else:
+            # Empty index, create tensors using the saved dimensions
+            dimensions = ggml.gguf_get_val_u32(gguf, ggml.gguf_find_key(gguf, b"dimensions"))
+            self.createtensors(np.zeros((0, dimensions), dtype=np.float32))
 
         # Load deletes tensor
         data = ggml.ggml_get_tensor(context, b"deletes")
@@ -461,8 +473,7 @@ class GGMLTensors:
         queries = ggml.ggml_new_tensor_2d(context, ggml.GGML_TYPE_F32, data.shape[1], self.querysize)
 
         # Embeddings data with space for both existing and new data
-        shape = utils.get_shape(self.data)
-        merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + shape[1])
+        merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + self.rows())
 
         # Create new buffer
         buffer = ggml.ggml_backend_alloc_ctx_tensors(context, backend)
@@ -508,7 +519,11 @@ class GGMLTensors:
 
         # Read tensor type
         tensortype = self.quantize
-        tensortype = "Q8_0" if isinstance(tensortype, bool) else f"Q{int(tensortype)}_0" if isinstance(tensortype, int) else tensortype
+        tensortype = (
+            "Q8_0"
+            if isinstance(tensortype, bool) and tensortype
+            else f"Q{int(tensortype)}_0" if isinstance(tensortype, int) and not isinstance(tensortype, bool) else tensortype
+        )
         tensortype = tensortype.upper() if tensortype else "F32"
 
         # Validate tensor type
@@ -568,3 +583,13 @@ class GGMLTensors:
         """
 
         return [queries[x : x + self.querysize] for x in range(0, len(queries), self.querysize)]
+
+    def rows(self):
+        """
+        Number of rows in the data tensor. Tensors with a single row or no rows report a 1-D shape, so the shape can't be used.
+
+        Returns:
+            row count
+        """
+
+        return ggml.ggml_nrows(self.data)

src/python/txtai/ann/dense/hnsw.py

@@ -2,8 +2,6 @@
 HNSW module
 """
 
-import numpy as np
-
 # Conditional import
 try:
     # pylint: disable=E0611
@@ -15,6 +13,11 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class HNSW(ANN):
     """
@@ -32,6 +35,9 @@ class HNSW(ANN):
         self.backend = Index(dim=self.config["dimensions"], space=self.config["metric"])
         self.backend.load_index(path)
 
+        # Default the delete counter for configurations that don't have it
+        self.config["deletes"] = self.config.get("deletes", 0)
+
     def index(self, embeddings):
         # Inner product is equal to cosine similarity on normalized vectors
         self.config["metric"] = "ip"
@@ -82,8 +88,8 @@ class HNSW(ANN):
         if ef:
             self.backend.set_ef(ef)
 
-        # Run the query
-        ids, distances = self.backend.knn_query(queries, k=limit)
+        # Run the query, clamp k to the current element count as hnswlib errors when k exceeds it
+        ids, distances = self.backend.knn_query(queries, k=min(limit, self.count()))
 
         # Map results to [(id, score)]
         results = []

src/python/txtai/ann/dense/milvus.py

@@ -0,0 +1,144 @@
+"""
+milvus-lite module
+"""
+
+import os
+import shutil
+import tempfile
+
+# Conditional import
+try:
+    import milvus_lite
+
+    MILVUS = True
+except ImportError:
+    MILVUS = False
+
+from ...archive import ArchiveFactory
+from ..base import ANN
+
+
+class Milvus(ANN):
+    """
+    Builds an ANN index using the milvus-lite library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not MILVUS:
+            raise ImportError('milvus-lite is not available - install "ann" extra to enable')
+
+        self.collection = None
+        self.directory = None
+        self.path = None
+
+    def load(self, path):
+        self.close()
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        try:
+            archive = ArchiveFactory.create(self.directory)
+            archive.load(path, "tar")
+            self.open()
+        except Exception:
+            self.close()
+            raise
+
+    def index(self, embeddings):
+        self.close()
+
+        # Lookup index settings
+        m = self.setting("m", 50)
+
+        # Create collection
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        # Use milvus-lite's native embedded API directly and keep this integration thin
+        self.backend = milvus_lite.MilvusLite(self.path)
+        schema = milvus_lite.CollectionSchema(
+            fields=[
+                milvus_lite.FieldSchema("id", milvus_lite.DataType.INT64, is_primary=True),
+                milvus_lite.FieldSchema("embedding", milvus_lite.DataType.FLOAT_VECTOR, dim=self.config["dimensions"]),
+            ]
+        )
+        self.collection = self.backend.create_collection("txtai", schema)
+        self.collection.create_index("embedding", {"index_type": "HNSW", "metric_type": "IP", "params": {"M": m}})
+        self.collection.load()
+
+        # Add items - position in embeddings is used as the id
+        self.insert(embeddings, 0)
+
+        # Add id offset and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"m": m, "milvus": milvus_lite.__version__})
+
+    def append(self, embeddings):
+        self.insert(embeddings, self.config["offset"])
+
+        # Update id offset and index metadata
+        self.config["offset"] += embeddings.shape[0]
+        self.metadata()
+
+    def delete(self, ids):
+        # Check length, ids can be a NumPy array
+        if len(ids):
+            self.collection.delete([int(x) for x in ids])
+
+    def search(self, queries, limit):
+        matches = self.collection.search(queries.tolist(), top_k=limit, metric_type="IP", anns_field="embedding")
+        return [[(int(match["id"]), float(match["distance"])) for match in results] for results in matches]
+
+    def count(self):
+        return self.collection.num_entities
+
+    def save(self, path):
+        self.collection.flush()
+        self.backend.close()
+        self.collection = None
+        self.backend = None
+
+        archive = ArchiveFactory.create(self.directory)
+        archive.save(path, "tar")
+        self.open()
+
+    def close(self):
+        # Release milvus-lite collection handles and the directory lock
+        if self.backend:
+            self.backend.close()
+
+        self.collection = None
+        super().close()
+
+        if self.directory and os.path.exists(self.directory):
+            shutil.rmtree(self.directory)
+
+        self.directory = None
+        self.path = None
+
+    def insert(self, embeddings, offset):
+        """
+        Inserts embeddings starting at offset.
+
+        Args:
+            embeddings: embeddings array
+            offset: starting id
+        """
+
+        if embeddings.shape[0]:
+            for start in range(0, embeddings.shape[0], 1024):
+                batch = embeddings[start : start + 1024]
+                self.collection.insert([{"id": offset + start + uid, "embedding": embedding.tolist()} for uid, embedding in enumerate(batch)])
+
+    def open(self):
+        """
+        Opens the milvus-lite database and collection
+        """
+
+        self.backend = milvus_lite.MilvusLite(self.path)
+        self.collection = self.backend.get_collection("txtai")
+
+        # Reopened collections with an index start released, load to enable search
+        self.collection.load()

src/python/txtai/ann/dense/numpy.py

@@ -2,15 +2,17 @@
 NumPy module
 """
 
-import numpy as np
-
-from safetensors import safe_open
-from safetensors.numpy import save_file
-
 from ...serialize import SerializeFactory
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+safetensors = library.safetensors()
+
 
 class NumPy(ANN):
     """
@@ -58,16 +60,20 @@ class NumPy(ANN):
         self.metadata()
 
     def delete(self, ids):
-        # Filter any index greater than size of array
-        ids = [x for x in ids if x < self.backend.shape[0]]
+        # Ignore IDs outside the array; negative IDs must not wrap to live rows.
+        ids = [x for x in ids if 0 <= x < self.backend.shape[0]]
 
-        # Clear specified ids
-        self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1])))
+        # Clear specified ids, zeros must match the array data type (e.g. uint8 for quantized data)
+        self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1]), dtype=self.backend.dtype))
 
     def search(self, queries, limit):
         if self.qbits:
             # Calculate hamming score for integer vectors
             scores = self.hammingscore(queries)
+
+            # Zero out deleted (all-zero) rows. Unlike dot product, hamming scores them nonzero,
+            # so they'd otherwise pass the caller's score > 0 filter and resurface in results.
+            scores[:, self.all(self.backend == 0, axis=1)] = 0
         else:
             # Dot product on normalized vectors is equal to cosine similarity
             scores = self.dot(self.tensor(queries), self.backend.T)
@@ -157,7 +163,7 @@ class NumPy(ANN):
         """
 
         # Merge metadata and tensors into single dictionary
-        with safe_open(path, framework="np") as f:
+        with safetensors.safe_open(path, framework="np") as f:
             return {**(f.metadata() if f.metadata() else {}), **{k: f.get_tensor(k) for k in f.keys()}}
 
     def savesafetensors(self, data, path, metadata=None):
@@ -170,7 +176,7 @@ class NumPy(ANN):
             metadata: additional metadata to save
         """
 
-        save_file(data, path, metadata)
+        safetensors.numpy.save_file(data, path, metadata)
 
     def hammingscore(self, queries):
         """

src/python/txtai/ann/dense/pgvector.py

@@ -4,8 +4,6 @@ PGVector module
 
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     from pgvector.sqlalchemy import BIT, HALFVEC, VECTOR
@@ -20,8 +18,12 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
-# pylint: disable=R0904
 class PGVector(ANN):
     """
     Builds an ANN index backed by a Postgres database.
@@ -69,7 +71,8 @@ class PGVector(ANN):
         self.metadata()
 
     def delete(self, ids):
-        self.database.execute(delete(self.table).where(self.table.c["indexid"].in_(ids)))
+        # Bind ids as int, NumPy integers don't match the indexid column
+        self.database.execute(delete(self.table).where(self.table.c["indexid"].in_([int(x) for x in ids])))
 
     def search(self, queries, limit):
         results = []

src/python/txtai/ann/dense/rabitq.py

@@ -0,0 +1,176 @@
+"""
+RabitQ module
+"""
+
+import math
+
+# Conditional import
+try:
+    from rabitqlib import HnswIndex, IvfIndex
+    from sklearn.cluster import KMeans
+
+    RABITQ = True
+except ImportError:
+    RABITQ = False
+
+from ..base import ANN
+
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+
+# Sentinel for unfilled top-k slots in native search results
+SENTINEL = 0xFFFFFFFF
+
+
+class RabitQ(ANN):
+    """
+    Builds an ANN index using the rabitqlib library (RaBitQ quantization).
+
+    Only the native index is stored, as a single file. No vectors are kept outside of it. ivf indexes support append
+    and delete natively. The native id of a row is its index id. hnsw indexes can't be extended or have rows removed,
+    so append and delete are not supported in that mode.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not RABITQ:
+            raise ImportError('rabitqlib is not available - install "ann" extra to enable')
+
+        # Number of rows removed from the native index (ivf mode only)
+        self.config["deletes"] = self.config.get("deletes", 0)
+
+    def load(self, path):
+        self.close()
+
+        # Invalid modes raise ValueError before native code runs
+        self.backend = IvfIndex.load(path) if self.mode() == "ivf" else HnswIndex.load(path)
+
+    def index(self, embeddings):
+        self.close()
+
+        # Mode is validated before any state is stored
+        mode = self.mode()
+
+        vectors = np.ascontiguousarray(embeddings, dtype=np.float32)
+        rows = vectors.shape[0]
+
+        # Quantization bits per dimension, validated by rabitqlib
+        nbits = self.setting("nbits", 1)
+
+        # Default clusters to 4 * sqrt(N), bounded to [1, N] so KMeans always fits
+        clusters = self.setting("clusters", None)
+        clusters = max(1, min(clusters if clusters else round(4 * math.sqrt(rows)), rows))
+
+        cluster = KMeans(n_clusters=clusters, random_state=0, n_init=1).fit(vectors)
+        centroids = np.ascontiguousarray(cluster.cluster_centers_, dtype=np.float32)
+        clusterids = np.ascontiguousarray(cluster.labels_, dtype=np.uint32)
+
+        dim = self.config["dimensions"]
+        if mode == "ivf":
+            backend = IvfIndex(dim=dim, max_elements=rows, num_clusters=clusters, nbits=nbits, metric="ip")
+            settings = {"mode": mode, "nbits": nbits, "clusters": clusters, "nprobe": self.setting("nprobe", self.nprobe(clusters))}
+        else:
+            backend = HnswIndex(
+                dim=dim,
+                max_elements=rows,
+                M=self.setting("m", 16),
+                ef_construction=self.setting("efconstruction", 200),
+                nbits=nbits,
+                metric="ip",
+                random_seed=self.setting("randomseed", 100),
+            )
+            settings = {
+                "mode": mode,
+                "nbits": nbits,
+                "m": self.setting("m", 16),
+                "efconstruction": self.setting("efconstruction", 200),
+                "efsearch": self.setting("efsearch", None),
+                "randomseed": self.setting("randomseed", 100),
+            }
+
+        backend.build(vectors, centroids, clusterids)
+        self.backend = backend
+
+        # Add id offset and index build metadata
+        self.config["offset"] = rows
+        self.config["deletes"] = 0
+        self.metadata(settings)
+
+    def append(self, embeddings):
+        if self.mode() != "ivf":
+            super().append(embeddings)
+
+        # Native ids continue from the last row ever added
+        embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+        self.backend.add(embeddings)
+
+        # Update id offset and index metadata
+        self.config["offset"] = self.backend.max_elements
+        self.metadata()
+
+    def delete(self, ids):
+        if self.mode() != "ivf":
+            super().delete(ids)
+
+        # Native remove hides rows from search. Unknown ids are silently ignored.
+        ids = [uid for uid in ids if 0 <= uid < self.backend.max_elements]
+        if ids:
+            self.config["deletes"] += self.backend.remove(np.array(ids, dtype=np.int64))
+
+    def search(self, queries, limit):
+        if not self.count():
+            return [[] for _ in queries]
+
+        queries = np.ascontiguousarray(queries, dtype=np.float32)
+
+        # The native index errors when k exceeds its row count
+        k = min(limit, self.count())
+
+        if self.mode() == "ivf":
+            nprobe = self.setting("nprobe", self.nprobe(self.backend.num_clusters))
+            ids, distances = self.backend.search(queries, k, nprobe)
+        else:
+            ids, distances = self.backend.search(queries, k, self.setting("efsearch", None) or 0)
+
+        # Map results to [(id, score)], dropping unfilled sentinel slots
+        return [
+            [(int(uid), float(1.0 - dist)) for uid, dist in zip(uids, dists) if uid != SENTINEL]
+            for uids, dists in zip(ids.tolist(), distances.tolist())
+        ]
+
+    def count(self):
+        return self.backend.max_elements - self.config["deletes"] if self.backend is not None else 0
+
+    def save(self, path):
+        self.backend.save(path)
+
+    def mode(self):
+        """
+        Returns the configured index mode.
+
+        Returns:
+            index mode, only "ivf" or "hnsw" are accepted
+        """
+
+        mode = self.setting("mode", "ivf")
+        if mode not in ("ivf", "hnsw"):
+            raise ValueError(f'Invalid RabitQ mode: {mode}. Supported modes are "ivf" and "hnsw".')
+
+        return mode
+
+    def nprobe(self, clusters):
+        """
+        Returns the default number of clusters to probe at search time.
+
+        Args:
+            clusters: number of IVF clusters
+
+        Returns:
+            default nprobe
+        """
+
+        return max(1, round(clusters / 16))

src/python/txtai/ann/dense/sqlite.py

@@ -32,9 +32,11 @@ class SQLite(ANN):
 
         # Quantization setting
         self.quantize = self.setting("quantize")
-        self.quantize = 8 if isinstance(self.quantize, bool) else int(self.quantize) if self.quantize else None
+        self.quantize = 8 if isinstance(self.quantize, bool) and self.quantize else int(self.quantize) if self.quantize else None
 
     def load(self, path):
+        if self.path != path:
+            self.close()
         self.path = path
 
     def index(self, embeddings):
@@ -55,7 +57,8 @@ class SQLite(ANN):
         self.metadata()
 
     def delete(self, ids):
-        self.database().executemany(self.deletesql(), [(x,) for x in ids])
+        # Bind ids as int, sqlite3 binds NumPy integers as blobs that match the wrong rows
+        self.database().executemany(self.deletesql(), [(int(x),) for x in ids])
 
     def search(self, queries, limit):
         results = []
@@ -73,6 +76,9 @@ class SQLite(ANN):
         return self.cursor.fetchone()[0]
 
     def save(self, path):
+        # Loading defers opening the connection until the first database operation.
+        self.database()
+
         # Temporary database
         if not self.path:
             # Save temporary database
@@ -193,10 +199,13 @@ class SQLite(ANN):
             connection.execute(self.tablesql())
 
             # The backup call will hang if there are uncommitted changes, need to copy over
-            # with iterdump (which is much slower)
-            for sql in self.connection.iterdump():
-                if self.tosql('insert into "{table}"') in sql.lower():
-                    connection.execute(sql)
+            # the rows (which is much slower). Quantized vectors must be inserted with their type.
+            embedding = "vec_bit(?)" if self.quantize == 1 else "vec_int8(?)" if self.quantize == 8 else "?"
+            rows = self.connection.execute(self.tosql("SELECT indexid, embedding FROM {table}"))
+            connection.executemany(self.tosql(f"INSERT INTO {{table}}(indexid, embedding) VALUES (?, {embedding})"), rows)
+
+            # Commit copied rows, otherwise they are rolled back when the connection is closed
+            connection.commit()
         else:
             # Database is up to date, can do a more efficient copy with SQLite C API
             self.connection.backup(connection)
@@ -254,7 +263,12 @@ class SQLite(ANN):
             SELECT
         """
 
-        return self.tosql(("SELECT indexid, 1 - distance FROM {table} " f"WHERE embedding MATCH {self.embeddingsql()} AND k = ? ORDER BY distance"))
+        # BIT distance counts differing bits; normalize it before converting to similarity.
+        # SQLite binary dimensions are the number of bits, not the packed byte count.
+        distance = f"distance / {float(self.config['dimensions'])}" if self.quantize == 1 else "distance"
+        return self.tosql(
+            f"SELECT indexid, 1 - ({distance}) FROM {{table}} " f"WHERE embedding MATCH {self.embeddingsql()} AND k = ? ORDER BY distance"
+        )
 
     def countsql(self):
         """

src/python/txtai/ann/dense/torch.py

@@ -2,9 +2,6 @@
 PyTorch module
 """
 
-import numpy as np
-import torch
-
 try:
     from bitsandbytes import matmul_4bit
     from bitsandbytes.functional import (
@@ -23,6 +20,13 @@ except ImportError:
 
 from .numpy import NumPy
 
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+torch = library.torch()
+
 
 class Torch(NumPy):
     """

src/python/txtai/ann/dense/turbovec.py

@@ -0,0 +1,82 @@
+"""
+TurboVec module
+"""
+
+# Conditional import
+try:
+    from turbovec import IdMapIndex
+
+    TURBOVEC = True
+except ImportError:
+    TURBOVEC = False
+
+from ..base import ANN
+
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
+
+class TurboVec(ANN):
+    """
+    Builds an ANN index using the turbovec library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not TURBOVEC:
+            raise ImportError('turbovec is not available - install "ann" extra to enable')
+
+    def load(self, path):
+        # Load index
+        self.backend = IdMapIndex.load(path)
+
+    def index(self, embeddings):
+        # Lookup index settings
+        bitwidth = self.setting("bitwidth", 4)
+
+        # Create index
+        self.backend = IdMapIndex(dim=self.config["dimensions"], bit_width=bitwidth)
+
+        # Add items - position in embeddings is used as the id
+        self.backend.add_with_ids(embeddings, np.arange(embeddings.shape[0], dtype=np.uint64))
+
+        # Add id offset, delete counter and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"bitwidth": bitwidth})
+
+    def append(self, embeddings):
+        new = embeddings.shape[0]
+
+        self.backend.add_with_ids(embeddings, np.arange(self.config["offset"], self.config["offset"] + new, dtype=np.uint64))
+
+        # Update id offset and index metadata
+        self.config["offset"] += new
+        self.metadata()
+
+    def delete(self, ids):
+        # Mark elements as deleted to omit from search results
+        for uid in ids:
+            self.backend.remove(uid)
+
+    def search(self, queries, limit):
+        # Run the query
+        scores, ids = self.backend.search(queries=queries, k=limit)
+
+        # Map results to [(id, score)]
+        results = []
+        for x, score in enumerate(scores):
+
+            # Build (id, score) tuples, convert to ints and floats
+            results.append(list(zip(ids[x].tolist(), score.tolist())))
+
+        return results
+
+    def count(self):
+        return len(self.backend)
+
+    def save(self, path):
+        # Write index
+        self.backend.write(path)

src/python/txtai/ann/dense/zvec.py

@@ -0,0 +1,166 @@
+"""
+zvec module
+"""
+
+import os
+import shutil
+import tempfile
+
+# Conditional import
+try:
+    import zvec
+
+    ZVEC = True
+except ImportError:
+    ZVEC = False
+
+from ...archive import ArchiveFactory
+from ..base import ANN
+
+
+class Zvec(ANN):
+    """
+    Builds an ANN index using the zvec library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not ZVEC:
+            raise ImportError('zvec is not available - install "ann" extra to enable')
+
+        self.directory = None
+        self.path = None
+
+    def load(self, path):
+        self.close()
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        try:
+            archive = ArchiveFactory.create(self.directory)
+            archive.load(path, "tar")
+            self.backend = zvec.open(self.path)
+        except Exception:
+            self.close()
+            raise
+
+    def index(self, embeddings):
+        self.close()
+
+        # Lookup index settings
+        efconstruction = self.setting("efconstruction", 200)
+        m = self.setting("m", 50)
+
+        # Create collection
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+        schema = zvec.CollectionSchema(
+            name="txtai",
+            vectors=zvec.VectorSchema(
+                "embedding",
+                self.datatype(),
+                self.config["dimensions"],
+                index_param=zvec.HnswIndexParam(metric_type=zvec.MetricType.IP, m=m, ef_construction=efconstruction),
+            ),
+        )
+        self.backend = zvec.create_and_open(path=self.path, schema=schema)
+
+        # Add items - position in embeddings is used as the id
+        self.insert(embeddings, 0)
+
+        # Build HNSW index
+        self.backend.optimize()
+
+        # Add id offset and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"efconstruction": efconstruction, "m": m, "zvec": zvec.__version__})
+
+    def append(self, embeddings):
+        self.insert(embeddings, self.config["offset"])
+
+        # Add new items to HNSW index
+        self.backend.optimize()
+
+        # Update id offset and index metadata
+        self.config["offset"] += embeddings.shape[0]
+        self.metadata()
+
+    def delete(self, ids):
+        # Check length, ids can be a NumPy array
+        if len(ids):
+            self.backend.delete([str(uid) for uid in ids])
+
+    def search(self, queries, limit):
+        # Lookup search settings
+        param = zvec.HnswQueryParam(ef=self.setting("efsearch", 300))
+
+        results = []
+        for query in queries:
+            matches = self.backend.query(
+                zvec.Query(field_name="embedding", vector=self.prepare(query), param=param),
+                topk=limit,
+            )
+            results.append([(int(match.id), float(match.score)) for match in matches])
+
+        return results
+
+    def count(self):
+        return self.backend.stats.doc_count
+
+    def save(self, path):
+        self.backend.flush()
+        self.backend = None
+
+        archive = ArchiveFactory.create(self.directory)
+        archive.save(path, "tar")
+        self.backend = zvec.open(self.path)
+
+    def close(self):
+        # Parent logic releases the collection and its file locks
+        super().close()
+
+        if self.directory and os.path.exists(self.directory):
+            shutil.rmtree(self.directory)
+
+        self.directory = None
+        self.path = None
+
+    def insert(self, embeddings, offset):
+        """
+        Inserts embeddings starting at offset.
+
+        Args:
+            embeddings: embeddings array
+            offset: starting id
+        """
+
+        if embeddings.shape[0]:
+            for start in range(0, embeddings.shape[0], 1024):
+                batch = embeddings[start : start + 1024]
+                self.backend.insert(
+                    [zvec.Doc(id=str(offset + start + uid), vectors={"embedding": self.prepare(embedding)}) for uid, embedding in enumerate(batch)]
+                )
+
+    def datatype(self):
+        """
+        Gets the vector data type.
+
+        Returns:
+            zvec vector data type
+        """
+
+        return zvec.DataType.VECTOR_FP32
+
+    def prepare(self, data):
+        """
+        Prepares data for the vector field.
+
+        Args:
+            data: input data
+
+        Returns:
+            data formatted for zvec
+        """
+
+        return data.tolist()

src/python/txtai/ann/sparse/__init__.py

@@ -5,3 +5,4 @@ Sparse ANN imports
 from .factory import SparseANNFactory
 from .ivfsparse import IVFSparse
 from .pgsparse import PGSparse
+from .zvecsparse import ZvecSparse

src/python/txtai/ann/sparse/factory.py

@@ -4,8 +4,10 @@ Factory module
 
 from ...util import Resolver
 
+from ..base import ANN
 from .ivfsparse import IVFSparse
 from .pgsparse import PGSparse
+from .zvecsparse import ZvecSparse
 
 
 class SparseANNFactory:
@@ -34,6 +36,8 @@ class SparseANNFactory:
             ann = IVFSparse(config)
         elif backend == "pgsparse":
             ann = PGSparse(config)
+        elif backend == "zvecsparse":
+            ann = ZvecSparse(config)
         else:
             ann = SparseANNFactory.resolve(backend, config)
 
@@ -56,6 +60,6 @@ class SparseANNFactory:
         """
 
         try:
-            return Resolver()(backend)(config)
+            return Resolver()(backend, ANN)(config)
         except Exception as e:
             raise ImportError(f"Unable to resolve sparse ann backend: '{backend}'") from e

src/python/txtai/ann/sparse/ivfsparse.py

@@ -7,8 +7,6 @@ import os
 
 from multiprocessing.pool import ThreadPool
 
-import numpy as np
-
 # Conditional import
 try:
     from scipy.sparse import csr_matrix, vstack
@@ -22,9 +20,12 @@ except ImportError:
     IVFSPARSE = False
 
 from ...serialize import SerializeFactory
-from ...util import SparseArray
+from ...util import Library, SparseArray
 from ..base import ANN
 
+# Core library imports
+np = Library().numpy()
+
 
 class IVFSparse(ANN):
     """
@@ -114,8 +115,12 @@ class IVFSparse(ANN):
         self.metadata()
 
     def delete(self, ids):
-        # Set index ids as deleted
-        self.deletes.extend(ids)
+        # Set index ids as deleted, ignoring ids that were never indexed and ids already marked deleted
+        # Ids are stored as int, the deletes list is serialized with msgpack
+        size = self.size()
+        for x in ids:
+            if x < size and x not in self.deletes:
+                self.deletes.append(int(x))
 
     def search(self, queries, limit):
         results = []
@@ -203,10 +208,10 @@ class IVFSparse(ANN):
             cluster centroids
         """
 
-        # Select top n most important features that contribute to L2 vector norm
-        indices = np.argsort(-norm(train, axis=0))[: self.setting("nfeatures", 25)]
-        data = train[:, indices]
-        data = train
+        # Select top n most important features that contribute to L2 vector norm.
+        # nfeatures is opt-in: the full feature space is used when it isn't set.
+        nfeatures = self.setting("nfeatures")
+        data = train[:, np.argsort(-norm(train, axis=0))[:nfeatures]] if nfeatures else train
 
         # Cluster data using k-means
         kmeans = MiniBatchKMeans(n_clusters=clusters, random_state=0, n_init=5).fit(data)
@@ -269,9 +274,13 @@ class IVFSparse(ANN):
             scores[:, deletes] = 0
 
         # Get top n matching indices and scores
-        indices = np.argpartition(-scores, limit if limit < scores.shape[0] else scores.shape[0] - 1)[:, :limit]
+        indices = np.argpartition(-scores, limit if limit < scores.shape[1] else scores.shape[1] - 1)[:, :limit]
         scores = np.take_along_axis(scores, indices, axis=1)
 
+        # argpartition doesn't order within the top n - sort each row by score descending
+        order = np.argsort(-scores, axis=1)
+        indices, scores = np.take_along_axis(indices, order, axis=1), np.take_along_axis(scores, order, axis=1)
+
         return indices, scores
 
     def scan(self, query, limit, blockids):

src/python/txtai/ann/sparse/pgsparse.py

@@ -4,8 +4,6 @@ PGSparse module
 
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     from pgvector import SparseVector
@@ -17,6 +15,11 @@ except ImportError:
 
 from ..dense import PGVector
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class PGSparse(PGVector):
     """

src/python/txtai/ann/sparse/zvecsparse.py

@@ -0,0 +1,35 @@
+"""
+ZvecSparse module
+"""
+
+# Conditional import
+try:
+    import zvec
+except ImportError:
+    pass
+
+from ..dense import Zvec
+
+
+class ZvecSparse(Zvec):
+    """
+    Builds a Sparse ANN index using the zvec library.
+    """
+
+    def search(self, queries, limit):
+        # Lookup search settings
+        param = zvec.HnswQueryParam(ef=self.setting("efsearch", 300))
+
+        results = []
+        for query in queries:
+            # zvec requires at least one non-zero value per query
+            matches = self.backend.query(zvec.Query(field_name="embedding", vector=self.prepare(query), param=param), topk=limit) if query.nnz else []
+            results.append([(int(match.id), float(match.score)) for match in matches])
+
+        return results
+
+    def datatype(self):
+        return zvec.DataType.SPARSE_VECTOR_FP32
+
+    def prepare(self, data):
+        return {int(index): float(value) for index, value in zip(data.indices, data.data)}

src/python/txtai/api/application.py

@@ -8,10 +8,11 @@ import sys
 
 from fastapi import APIRouter, Depends, FastAPI
 from fastapi_mcp import FastApiMCP
-from httpx import AsyncClient
+from httpx import AsyncClient, ASGITransport
 
 from .authorization import Authorization
 from .base import API
+from .extension import Extension
 from .factory import APIFactory
 
 from ..app import Application
@@ -95,15 +96,15 @@ def lifespan(application):
 
     # Conditionally add routes based on configuration
     for name, router in routers.items():
-        if name in config:
+        if name in config and enabled(config, name):
             application.include_router(router)
 
     # Special case for embeddings clusters
-    if "cluster" in config and "embeddings" not in config:
+    if "cluster" in config and "embeddings" not in config and enabled(config, "embeddings"):
         application.include_router(routers["embeddings"])
 
     # Special case to add similarity instance for embeddings
-    if "embeddings" in config and "similarity" not in config:
+    if "embeddings" in config and "similarity" not in config and enabled(config, "similarity"):
         application.include_router(routers["similarity"])
 
     # Execute extensions if present
@@ -111,17 +112,56 @@ def lifespan(application):
     if extensions:
         for extension in extensions.split(","):
             # Create instance and execute extension
-            extension = APIFactory.get(extension.strip())()
+            extension = APIFactory.get(extension.strip(), Extension)()
             extension(application)
 
     # Add Model Context Protocol (MCP) service, if applicable
-    if config.get("mcp"):
-        mcp = FastApiMCP(application, http_client=AsyncClient(timeout=100))
-        mcp.mount()
+    createmcp(application, config)
 
     yield
 
 
+def enabled(config, name):
+    """
+    Check configuration to determine if a route is enabled or disabled
+
+    Args:
+        config: configuration
+        name: route name
+
+    Returns:
+        True if a route is enabled, False otherwise
+    """
+
+    return config.get("routes", {}).get(name, True)
+
+
+def createmcp(application, config):
+    """
+    Create a MCP service if necessary.
+
+    Args:
+        application: FastAPI Application
+        config: configuration
+    """
+
+    mcp = config.get("mcp")
+    if mcp:
+        # HTTP Client arguments
+        defaults = {"base_url": "http://apiserver", "timeout": 100}
+        clientargs = mcp.get("clientargs", {}) if isinstance(mcp, dict) else {}
+        clientargs = {**defaults, **clientargs}
+
+        # Create HTTP client with custom options
+        client = AsyncClient(transport=ASGITransport(app=application, raise_app_exceptions=False), **clientargs)
+
+        # MCP service arguments
+        mcpargs = mcp.get("mcpargs", {}) if isinstance(mcp, dict) else {}
+
+        mcp = FastApiMCP(application, http_client=client, **mcpargs)
+        mcp.mount()
+
+
 def start():
     """
     Runs application lifespan handler.

src/python/txtai/api/authorization.py

@@ -3,9 +3,10 @@ Authorization module
 """
 
 import hashlib
-import os
+import hmac
 
-from fastapi import Header, HTTPException
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 
 
 class Authorization:
@@ -13,17 +14,17 @@ class Authorization:
     Basic token authorization.
     """
 
-    def __init__(self, token=None):
+    def __init__(self, token):
         """
         Creates a new Authorization instance.
 
         Args:
-            token: SHA-256 hash of token to check
+            token: SHA-256 hash of the valid authorization token
         """
 
-        self.token = token if token else os.environ.get("TOKEN")
+        self.token = token
 
-    def __call__(self, authorization: str = Header(default=None)):
+    def __call__(self, authorization: HTTPAuthorizationCredentials = Depends(HTTPBearer())):
         """
         Validates authorization header is present and equal to current token.
 
@@ -31,23 +32,19 @@ class Authorization:
             authorization: authorization header
         """
 
-        if not authorization or self.token != self.digest(authorization):
+        # Authorization header is parsed into {scheme="Bearer", credentials="<token>"}
+        if not hmac.compare_digest(self.token, self.digest(authorization.credentials)):
             raise HTTPException(status_code=401, detail="Invalid Authorization Token")
 
-    def digest(self, authorization):
+    def digest(self, token):
         """
         Computes a SHA-256 hash for input authorization token.
 
         Args:
-            authorization: authorization header
+            token: authorization token
 
         Returns:
             SHA-256 hash of authorization token
         """
 
-        # Replace Bearer prefix
-        prefix = "Bearer "
-        token = authorization[len(prefix) :] if authorization.startswith(prefix) else authorization
-
-        # Compute SHA-256 hash
         return hashlib.sha256(token.encode("utf-8")).hexdigest()

src/python/txtai/api/base.py

@@ -32,7 +32,7 @@ class API(Application):
         weights = self.weights(request.query_params.get("weights") if request and hasattr(request, "query_params") else weights)
         index = request.query_params.get("index") if request and hasattr(request, "query_params") else index
         parameters = request.query_params.get("parameters") if request and hasattr(request, "query_params") else parameters
-        graph = request.query_params.get("graph") if request and hasattr(request, "query_params") else graph
+        graph = self.graph(request.query_params.get("graph") if request and hasattr(request, "query_params") else graph)
 
         # Decode parameters
         parameters = json.loads(parameters) if parameters and isinstance(parameters, str) else parameters
@@ -157,3 +157,17 @@ class API(Application):
         """
 
         return float(weights) if weights else weights
+
+    def graph(self, graph):
+        """
+        Parses the graph parameter from the request.
+
+        Args:
+            graph: graph parameter
+
+        Returns:
+            graph as a bool
+        """
+
+        # Query parameters are strings - accept the same values the batchsearch endpoint coerces
+        return graph.strip().lower() in ("1", "true", "yes", "on") if isinstance(graph, str) else bool(graph)

src/python/txtai/api/cluster.py

@@ -56,17 +56,19 @@ class Cluster:
         """
 
         # Build URL
-        action = f"search?query={urllib.parse.quote_plus(query)}"
+        params = {"query": query}
         if limit:
-            action += f"&limit={limit}"
-        if weights:
-            action += f"&weights={weights}"
+            params["limit"] = limit
+        if weights is not None:
+            params["weights"] = weights
         if index:
-            action += f"&index={index}"
+            params["index"] = index
         if parameters:
-            action += f"&parameters={json.dumps(parameters) if isinstance(parameters, dict) else parameters}"
+            params["parameters"] = json.dumps(parameters) if isinstance(parameters, dict) else parameters
         if graph is not None:
-            action += f"&graph={graph}"
+            params["graph"] = graph
+
+        action = f"search?{urllib.parse.urlencode(params)}"
 
         # Run query and flatten results into single results list
         results = []
@@ -100,7 +102,7 @@ class Cluster:
         params = {"queries": queries}
         if limit:
             params["limit"] = limit
-        if weights:
+        if weights is not None:
             params["weights"] = weights
         if index:
             params["index"] = index
@@ -196,11 +198,11 @@ class Cluster:
         shards = [[] for _ in range(len(self.shards))]
         for document in documents:
             uid = document.get("id") if isinstance(document, dict) else document
-            if uid and isinstance(uid, str):
-                # Quick int hash of string to help derive shard id
+            if isinstance(uid, str):
+                # Quick int hash of string to help derive shard id (handles empty strings too)
                 uid = zlib.adler32(uid.encode("utf-8"))
-            elif uid is None:
-                # Get random shard id when uid isn't set
+            elif not isinstance(uid, int):
+                # Get random shard id when uid isn't a usable int (None, a tuple document, etc.)
                 uid = random.randint(0, len(shards) - 1)
 
             shards[uid % len(self.shards)].append(document)

src/python/txtai/api/factory.py

@@ -4,6 +4,8 @@ API factory module
 
 from ..util import Resolver
 
+from .base import API
+
 
 class APIFactory:
     """
@@ -11,18 +13,19 @@ class APIFactory:
     """
 
     @staticmethod
-    def get(api):
+    def get(path, base=None):
         """
-        Gets a new instance of api class.
+        Gets a new instance of a class.
 
         Args:
-            api: API instance class
+            path: class path
+            base: optional required base class
 
         Returns:
-            API
+            Class
         """
 
-        return Resolver()(api)
+        return Resolver()(path, base)
 
     @staticmethod
     def create(config, api):
@@ -37,4 +40,4 @@ class APIFactory:
             API instance
         """
 
-        return APIFactory.get(api)(config)
+        return APIFactory.get(api, API)(config)

src/python/txtai/api/route.py

@@ -2,9 +2,12 @@
 Route module
 """
 
-from fastapi.routing import APIRoute, get_request_handler
+import json
 
-from .responses import ResponseFactory
+from fastapi.responses import StreamingResponse
+from fastapi.routing import APIRoute
+
+from .responses import ResponseFactory, MessagePackResponse
 
 
 class EncodingAPIRoute(APIRoute):
@@ -20,22 +23,26 @@ class EncodingAPIRoute(APIRoute):
             route handler function
         """
 
+        # Get handle to the original route handler
+        original = super().get_route_handler()
+
         async def handler(request):
-            route = get_request_handler(
-                dependant=self.dependant,
-                body_field=self.body_field,
-                status_code=self.status_code,
-                response_class=ResponseFactory.create(request),
-                response_field=getattr(self, "secure_cloned_response_field", self.response_field),
-                response_model_include=self.response_model_include,
-                response_model_exclude=self.response_model_exclude,
-                response_model_by_alias=self.response_model_by_alias,
-                response_model_exclude_unset=self.response_model_exclude_unset,
-                response_model_exclude_defaults=self.response_model_exclude_defaults,
-                response_model_exclude_none=self.response_model_exclude_none,
-                dependency_overrides_provider=self.dependency_overrides_provider,
+            # Target response class
+            target = ResponseFactory.create(request)
+            request.state.response_class = target
+
+            # Get response
+            response = await original(request)
+
+            # Force MessagePackResponse when it's requested and response type doesn't match
+            return (
+                target(
+                    content=json.loads(response.body),
+                    status_code=response.status_code,
+                    headers={k: v for k, v in response.headers.items() if k.lower() not in ["content-length", "content-type"]},
+                )
+                if not isinstance(response, (target, StreamingResponse)) and target == MessagePackResponse
+                else response
             )
 
-            return await route(request)
-
         return handler

src/python/txtai/api/routers/embeddings.py

@@ -14,7 +14,7 @@ from .. import application
 from ..responses import ResponseFactory
 from ..route import EncodingAPIRoute
 
-from ...app import ReadOnlyError
+from ...app import ReadOnlyError, ReindexDisabledError
 from ...graph import Graph
 
 router = APIRouter(route_class=EncodingAPIRoute)
@@ -198,7 +198,7 @@ def reindex(config: dict = Body(...), function: str = Body(default=None)):
 
     try:
         application.get().reindex(config, function)
-    except ReadOnlyError as e:
+    except (ReadOnlyError, ReindexDisabledError) as e:
         raise HTTPException(status_code=403, detail=e.args[0]) from e
 
 

src/python/txtai/api/routers/openai.py

@@ -44,8 +44,8 @@ def chat(
     # Build keyword arguments
     kwargs = {key: value for key, value in [("stream", stream), ("maxlength", max_completion_tokens)] if value}
 
-    # Get first message
-    message = messages[0]["content"]
+    # Get latest message
+    message = messages[-1]["content"]
 
     # Agent
     if model in application.get().agents:
@@ -165,9 +165,9 @@ class ChatResponse:
         return {
             "id": str(uuid.uuid4()),
             "object": "chat.completion",
-            "created": int(time.time() * 1000),
+            "created": int(time.time()),
             "model": model,
-            "choices": [{"id": 0, "message": {"role": "assistant", "content": result}, "finish_reason": "stop"}],
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": result}, "finish_reason": "stop"}],
         }
 
 
@@ -177,15 +177,38 @@ class StreamingChatResponse:
     """
 
     def __call__(self, model, result):
+        # All chunks of a response share the same id and creation time
+        uid, created = str(uuid.uuid4()), int(time.time())
+
         for chunk in result:
-            yield "data: " + json.dumps(
-                {
-                    "id": str(uuid.uuid4()),
-                    "object": "chat.completion.chunk",
-                    "created": int(time.time() * 1000),
-                    "model": model,
-                    "choices": [{"id": 0, "delta": {"content": chunk}}],
-                }
-            ) + "\n\n"
+            yield self.chunk(uid, created, model, {"content": chunk}, None)
+
+        # Final chunk signals the end of the response
+        yield self.chunk(uid, created, model, {}, "stop")
 
         yield "data: [DONE]\n\n"
+
+    def chunk(self, uid, created, model, delta, reason):
+        """
+        Builds a server-sent event for a single chat completion chunk.
+
+        Args:
+            uid: response id
+            created: response creation time in seconds
+            model: model name
+            delta: chunk content
+            reason: finish reason, None until the final chunk
+
+        Returns:
+            server-sent event
+        """
+
+        data = {
+            "id": uid,
+            "object": "chat.completion.chunk",
+            "created": created,
+            "model": model,
+            "choices": [{"index": 0, "delta": delta, "finish_reason": reason}],
+        }
+
+        return "data: " + json.dumps(data) + "\n\n"

src/python/txtai/app/__init__.py

@@ -2,4 +2,4 @@
 App imports
 """
 
-from .base import Application, ReadOnlyError
+from .base import Application, ReadOnlyError, ReindexDisabledError

src/python/txtai/app/base.py

@@ -7,15 +7,17 @@ import os
 from multiprocessing.pool import ThreadPool
 from threading import RLock
 
-import yaml
-
 from ..agent import Agent
 from ..embeddings import Documents, Embeddings
 from ..pipeline import PipelineFactory
 from ..workflow import WorkflowFactory
 
+# Core library imports
+from ..util import Library
+
+yaml = Library().yaml()
+
 
-# pylint: disable=R0904
 class Application:
     """
     Builds YAML-configured txtai applications.
@@ -139,6 +141,10 @@ class Application:
                     config["embeddings"] = None
                     config["similarity"] = self.pipelines["similarity"]
 
+                elif pipeline in ("textractor", "urlretrieve"):
+                    # Default to safeopen enabled
+                    config["safeopen"] = config.get("safeopen", True)
+
                 self.pipelines[pipeline] = PipelineFactory.create(config, pipeline)
 
     def createworkflows(self):
@@ -252,7 +258,7 @@ class Application:
         """
 
         # Check for task shorthand syntax
-        task = {"action": task} if isinstance(task, (str, list)) else task
+        task = {"action": task} if isinstance(task, (str, list)) else task.copy()
 
         if "action" in task:
             action = task["action"]
@@ -284,6 +290,10 @@ class Application:
             # Save resolved action(s)
             task["action"] = actions[0] if not isinstance(action, list) else actions
 
+        if "task" in task and task["task"] == "retrieve":
+            # Default to safeopen enabled
+            task["safeopen"] = task.get("safeopen", True)
+
         # Resolve initializer
         if "initialize" in task and isinstance(task["initialize"], str):
             task["initialize"] = self.function(task["initialize"])
@@ -343,6 +353,9 @@ class Application:
             resolved function
         """
 
+        # Function configuration
+        config = {}
+
         # Check if function is a pipeline
         if function in self.pipelines:
             return self.pipelines[function]
@@ -351,8 +364,12 @@ class Application:
         if function in self.workflows:
             return self.workflows[function]
 
+        if function in ("textractor", "urlretrieve"):
+            # Default to safeopen enabled
+            config["safeopen"] = config.get("safeopen", True)
+
         # Attempt to resolve action as a callable function
-        return PipelineFactory.create({}, function)
+        return PipelineFactory.create(config, function)
 
     def search(self, query, limit=10, weights=None, index=None, parameters=None, graph=False):
         """
@@ -559,6 +576,10 @@ class Application:
         if not self.config.get("writable"):
             raise ReadOnlyError("Attempting to reindex a read-only index (writable != True)")
 
+        # Raise error if not re-indexable
+        if not self.config.get("reindex"):
+            raise ReindexDisabledError("Attempting to reindex with reindexing disabled (reindex != True)")
+
         if self.embeddings:
             with self.lock:
                 # Resolve function, if necessary
@@ -819,3 +840,9 @@ class ReadOnlyError(Exception):
     """
     Error raised when trying to modify a read-only index
     """
+
+
+class ReindexDisabledError(Exception):
+    """
+    Error raised when trying to reindex with reindexing disabled
+    """

src/python/txtai/archive/compress.py

@@ -46,6 +46,6 @@ class Compress:
 
         directory = os.path.abspath(directory)
         path = os.path.abspath(path)
-        prefix = os.path.commonprefix([directory, path])
+        prefix = os.path.commonpath([directory, path])
 
         return prefix == directory

src/python/txtai/cloud/factory.py

@@ -4,6 +4,7 @@ Factory module
 
 from ..util import Resolver
 
+from .base import Cloud
 from .hub import HuggingFaceHub
 from .storage import ObjectStorage, LIBCLOUD
 
@@ -58,7 +59,7 @@ class CloudFactory:
         """
 
         try:
-            return Resolver()(backend)(config)
+            return Resolver()(backend, Cloud)(config)
 
         except Exception as e:
             # Failure message

src/python/txtai/cloud/hub.py

@@ -5,12 +5,13 @@ Hugging Face Hub module
 import os
 import tempfile
 
-import huggingface_hub
-
-from huggingface_hub.utils import RepositoryNotFoundError
-
 from .base import Cloud
 
+# Core library imports
+from ..util import Library
+
+huggingface_hub = Library().huggingface_hub()
+
 
 class HuggingFaceHub(Cloud):
     """
@@ -30,11 +31,11 @@ class HuggingFaceHub(Cloud):
             # Otherwise return repository metadata
             return huggingface_hub.model_info(repo_id=self.config["container"], revision=self.config.get("revision"), token=self.config.get("token"))
 
-        except RepositoryNotFoundError:
+        except huggingface_hub.utils.RepositoryNotFoundError:
             return None
 
     def load(self, path=None):
-        # Download archvie file and return local path
+        # Download archive file and return local path
         if self.isarchive(path):
             return huggingface_hub.hf_hub_download(
                 repo_id=self.config["container"],

src/python/txtai/console/base.py

@@ -154,6 +154,11 @@ class Console(Cmd):
         else:
             results = self.app.search(query, limit=self.vlimit)
 
+        # No matches, nothing to render
+        if not results:
+            self.console.print("No results")
+            return
+
         columns, table = {}, Table(box=box.SQUARE, style="#03a9f4")
 
         # Build column list

src/python/txtai/data/base.py

@@ -24,7 +24,7 @@ class Data:
         self.columns = columns
         self.maxlength = maxlength
 
-    def __call__(self, train, validation, workers):
+    def __call__(self, train, validation, workers, batch):
         """
         Tokenizes training and validation data and returns processed datasets.
 
@@ -32,14 +32,18 @@ class Data:
             train: training data
             validation: validation data
             workers: number of concurrent tokenizers when processing datasets, only main process used when set to None
+            batch: tokenize batch size
 
         Returns:
             (train, validation)
         """
 
-        return (self.prepare(train, self.process, workers), self.prepare(validation, self.process, workers) if validation else None)
+        return (
+            self.prepare(train, self.process, workers, batch),
+            self.prepare(validation, self.process, workers, batch) if validation is not None else None,
+        )
 
-    def prepare(self, data, fn, workers):
+    def prepare(self, data, fn, workers, batch):
         """
         Prepares and tokenizes data for training.
 
@@ -47,14 +51,16 @@ class Data:
             data: input data
             fn: tokenize processing function to apply
             workers: number of concurrent tokenizers when processing datasets, only main process used when set to None
+            batch: tokenize batch size
 
         Returns:
             tokens
         """
 
-        if hasattr(data, "map"):
+        # Check for column_names, pandas DataFrames also have a map method
+        if hasattr(data, "column_names"):
             # Hugging Face dataset
-            tokens = data.map(fn, batched=True, num_proc=workers, remove_columns=data.column_names)
+            tokens = data.map(fn, batched=True, batch_size=batch, num_proc=workers, remove_columns=data.column_names)
         else:
             # Re-orient data into columns for efficient batch tokenization
             columns = {}
@@ -91,11 +97,11 @@ class Data:
         column = self.columns[-1]
 
         # Return length of labels if it's an array
-        length = self.length(data[column][0] if hasattr(data, "columns") else data[0][column])
+        length = self.length(next(iter(data[column])) if hasattr(data, "columns") else data[0][column])
         if length:
             return length
 
-        if hasattr(data, "map"):
+        if hasattr(data, "column_names"):
             # Hugging Face dataset
             labels = sorted(data.unique(self.columns[-1]))
         elif hasattr(data, "columns"):

src/python/txtai/data/questions.py

@@ -72,14 +72,20 @@ class Questions(Data):
                 while sequences[end] != (1 if self.rpad else 0):
                     end -= 1
 
-                # Map start character and end character to matching token index
-                while start < len(offset) and offset[start][0] <= startchar:
-                    start += 1
-                tokenized["start_positions"].append(start - 1)
+                # Label with CLS token if the answer isn't fully in this span. Long contexts are split into
+                # overlapping spans with stride and most spans don't contain the answer.
+                if not (offset[start][0] <= startchar and offset[end][1] >= endchar):
+                    tokenized["start_positions"].append(clstoken)
+                    tokenized["end_positions"].append(clstoken)
+                else:
+                    # Map start character and end character to matching token index
+                    while start < len(offset) and offset[start][0] <= startchar:
+                        start += 1
+                    tokenized["start_positions"].append(start - 1)
 
-                while offset[end][1] >= endchar:
-                    end -= 1
-                tokenized["end_positions"].append(end + 1)
+                    while offset[end][1] >= endchar:
+                        end -= 1
+                    tokenized["end_positions"].append(end + 1)
 
         return tokenized
 

src/python/txtai/data/sequences.py

@@ -38,9 +38,8 @@ class Sequences(Data):
         source = [self.prefix + x if self.prefix else x for x in data[source]]
         inputs = self.tokenizer(source, max_length=self.maxlength, padding=False, truncation=True)
 
-        # Tokenize target
-        with self.tokenizer.as_target_tokenizer():
-            targets = self.tokenizer(data[target], max_length=self.maxlength, padding=False, truncation=True)
+        # Tokenize targets
+        targets = self.tokenizer(data[target], max_length=self.maxlength, padding=False, truncation=True)
 
         # Combine inputs
         inputs["labels"] = targets["input_ids"]

src/python/txtai/data/texts.py

@@ -104,7 +104,7 @@ class Texts(Data):
             nextlength = len(rows[x + 1]) if x < len(rows) - 1 else 0
 
             # New row
-            if (length + nextlength) >= self.maxlength:
+            if (length + nextlength) > self.maxlength:
                 for column in columns:
                     results[column].append(list(chain(*inputs[column][index : x + 1])))
 

src/python/txtai/data/tokens.py

@@ -2,10 +2,13 @@
 Tokens module
 """
 
-import torch
+# Core library imports
+from ..util import Library
+
+Dataset = Library().dataset()
 
 
-class Tokens(torch.utils.data.Dataset):
+class Tokens(Dataset):
     """
     Default dataset used to hold tokenized data.
     """