Forward backend options during streaming speech generation (#1389) (#1390)

Switch SQLite ANN connections when load() is given a new path

.github/workflows/build.yml

@@ -10,15 +10,15 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
 
-    timeout-minutes: 60
+    timeout-minutes: 75
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Install Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install Java
         uses: actions/setup-java@v5
@@ -29,20 +29,18 @@ jobs:
       - name: Install dependencies - Linux
         run: |
           sudo apt-get update
-          sudo apt-get install libportaudio2 libsndfile1
+          sudo apt-get install libportaudio2 libsndfile1 libegl1 libgles2
           sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install dependencies - macOS
         run: |
-          echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV
           echo "PYTORCH_MPS_DISABLE=1" >> $GITHUB_ENV
           echo "LLAMA_NO_METAL=1" >> $GITHUB_ENV
           echo "TIKA_STARTUP_SLEEP=30" >> $GITHUB_ENV
           echo "TIKA_STARTUP_MAX_RETRY=10" >> $GITHUB_ENV
-          echo "KMP_DUPLICATE_LIB_OK=TRUE" >> $GITHUB_ENV
           brew install portaudio
-          sudo xcode-select -s "/Applications/Xcode_16.app"
+          pip install torch==2.12.1 torchvision==0.27.1
         if: matrix.os == 'macos-latest'
 
       - name: Install dependencies - Windows
@@ -63,12 +61,16 @@ jobs:
           HF_HUB_ETAG_TIMEOUT: 100
           HF_HUB_DOWNLOAD_TIMEOUT: 100
           HF_XET_CHUNK_CACHE_SIZE_BYTES: 0
+          DOCLING_DEVICE: "cpu"
+          DOCLING_INFERENCE_COMPILE_TORCH_MODELS: false
 
       - uses: pre-commit/[email protected]
         if: matrix.os == 'ubuntu-latest'
 
       - name: Test Coverage
-        run: coveralls --service=github
+        run: |
+          coverage report --fail-under=100
+          coveralls --service=github
         if: matrix.os == 'ubuntu-latest'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -7,12 +7,19 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v6
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
-      - run: |
+          python-version: "3.11"
+
+      - name: Install txtai
+        run: |
           pip install -U pip wheel
           pip install .[all,dev]
-      - run: mkdocs gh-deploy --force
+
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --force
  

.github/workflows/minimal.yml

@@ -0,0 +1,22 @@
+# GitHub Actions minimal build
+name: minimal
+
+on: ["push", "pull_request"]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Minimal install
+        run: |
+          pip install -U pip wheel
+          MINIMAL=1 pip install .
+          python -c "import txtai"

.pylintrc

@@ -12,6 +12,8 @@ max-locals=40
 max-returns=10
 max-attributes=20
 min-public-methods=0
+max-public-methods=80
 
 [FORMAT]
 max-line-length=150
+max-module-lines=1500

Makefile

@@ -42,6 +42,7 @@ coverage:
 	coverage run -m unittest discover -v -k testdatabase -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testembeddings -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testgraph -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testlibrary -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testmodels -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testoptional -s ${TEST_DIR}
 	coverage run -m unittest discover -v -k testpipeline.testaudio -s ${TEST_DIR}

README.md

@@ -49,7 +49,7 @@ Summary of txtai features:
 - 🔋 Batteries included with defaults to get up and running fast
 - ☁️ Run local or scale out with container orchestration
 
-txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
 > [!NOTE]
 >
@@ -134,14 +134,15 @@ Agents connect embeddings, pipelines, workflows and other agents together to aut
 
 ![agent](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/agent.png)
 
-txtai agents are built on top of the [smolagents](https://github.com/huggingface/smolagents) framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM).
+txtai agents are built on top of the [smolagents](https://github.com/huggingface/smolagents) framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM). Agent prompting with [`agents.md`](https://github.com/agentsmd/agents.md) and [`skill.md`](https://agentskills.io/specification) are also supported.
 
 Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/agent_quickstart.py). Additional examples are listed below.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 | [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
 
 #### Retrieval augmented generation
@@ -155,7 +156,7 @@ Check out this [RAG Quickstart Example](https://github.com/neuml/txtai/blob/mast
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
 | [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
 | [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
 | [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |
@@ -189,9 +190,9 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
-See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud).
+See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda), [lightweight minimal installation](https://neuml.github.io/txtai/install/#minimal-install) and how to [run with containers](https://neuml.github.io/txtai/cloud).
 
 ## Model guide
 
@@ -203,9 +204,9 @@ See the table below for the current recommended models. These models all allow c
 | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
 | [Embeddings](https://neuml.github.io/txtai/embeddings)                        | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
 | [Image Captions](https://neuml.github.io/txtai/pipeline/image/caption)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
-| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [DeBERTa v3 Zeroshot](https://hf.co/MoritzLaurer/deberta-v3-base-zeroshot-v2.0-c) |
 | [Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels)          | Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer)          |
-| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b)                 |
+| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [Gemma 4 31B](https://hf.co/google/gemma-4-31B) |
 | [Summarization](https://neuml.github.io/txtai/pipeline/text/summary)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
 | [Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
 | [Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
@@ -217,9 +218,8 @@ See the following links to learn more.
 
 - [Hugging Face Tasks](https://hf.co/tasks)
 - [Hugging Face Model Hub](https://hf.co/models)
-- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
-- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
-- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+- [Embeddings Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LLM Leaderboard](https://hf.co/spaces/lmarena-ai/arena-leaderboard)
 
 ## Powered by txtai
 
@@ -230,7 +230,7 @@ The following applications are powered by txtai.
 | Application  | Description  |
 |:------------ |:-------------|
 | [rag](https://github.com/neuml/rag) | Retrieval Augmented Generation (RAG) application |
-| [ragdata](https://github.com/neuml/ragdata) | Build knowledge bases for RAG |
+| [ncoder](https://github.com/neuml/ncoder) | Open-Source AI coding agent |
 | [paperai](https://github.com/neuml/paperai) | AI for medical and scientific papers |
 | [annotateai](https://github.com/neuml/annotateai) | Automatically annotate papers with LLMs |
 
@@ -242,6 +242,7 @@ In addition to this list, there are also many other [open-source projects](https
 ![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further-ghdark.png#gh-dark-mode-only)
 
 - [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
 - [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)

docker/base/Dockerfile

@@ -1,5 +1,5 @@
 # Set base image
-ARG BASE_IMAGE=python:3.10-slim
+ARG BASE_IMAGE=python:3.11-slim
 FROM $BASE_IMAGE
 
 # Install GPU-enabled version of PyTorch if set
@@ -8,7 +8,7 @@ ARG GPU
 # Target CPU architecture
 ARG TARGETARCH
 
-# Set Python version (i.e. 3, 3.10)
+# Set Python version (i.e. 3, 3.11)
 ARG PYTHON_VERSION=3
 
 # List of txtai components to install
@@ -21,13 +21,13 @@ ENV LANG=C.UTF-8
 RUN \
     # Install required packages
     apt-get update && \
-    apt-get -y --no-install-recommends install libgomp1 libportaudio2 libsndfile1 git gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
+    apt-get -y --no-install-recommends install libgomp1 libportaudio2 libsndfile1 libegl1 libgles2 libvulkan1 git gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
     rm -rf /var/lib/apt/lists && \
     \
     # Install txtai project and dependencies
     ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
     python -m pip install --no-cache-dir -U pip wheel setuptools && \
-    if [ -z ${GPU} ] && { [ -z ${TARGETARCH} ] || [ ${TARGETARCH} = "amd64" ] ;}; then pip install --no-cache-dir torch==2.9.1+cpu torchvision==0.24.1+cpu -f https://download.pytorch.org/whl/torch -f https://download.pytorch.org/whl/torchvision; fi && \
+    if [ -z ${GPU} ]; then pip install --no-cache-dir torch==2.14.1+cpu torchvision==0.29.1+cpu -f https://download.pytorch.org/whl/torch -f https://download.pytorch.org/whl/torchvision; fi && \
     python -m pip install --no-cache-dir txtai${COMPONENTS} && \
     python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])" && \
     \

docker/minimal/Dockerfile

@@ -0,0 +1,20 @@
+# Set base image
+ARG BASE_IMAGE=python:3.11-slim
+FROM $BASE_IMAGE
+
+# Target CPU architecture
+ARG TARGETARCH
+
+# Set Python version (i.e. 3, 3.11)
+ARG PYTHON_VERSION=3
+
+# Locale environment variables
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN \
+    # Install minimal txtai project and dependencies
+    MINIMAL=1 python -m pip install --no-cache-dir txtai_minimal
+
+# Set default working directory
+WORKDIR /app

docs/agent/configuration.md

@@ -81,14 +81,52 @@ Embeddings indexes have built-in support. Provide the following dictionary confi
 
 ### tool
 
-A tool instance can be provided. Additionally, the following strings load tools directly.
+The following shortcut strings load tools directly. Passing a Tool instance is also supported.
 
 | Tool        | Description                                               |
 |:------------|:----------------------------------------------------------|
-| http.*      | HTTP Path to a Model Context Protocol (MCP) server        |
+| bash        | Runs a shell command through subprocess                   |
+| defaults    | Loads all of these tools as the default toolkit           |
+| edit        | Edits a file in place and returns a diff                  |
+| glob        | Finds matching file patterns in a directory               |
+| grep        | Finds matching file content in a directory                |
+| http.*      | HTTP Path to a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/docs/getting-started/intro) server |
 | python      | Runs a Python action                                      |
+| read        | Reads file or url content, supports text extraction       |
+| todowrite   | Generates a task list to organize complex tasks           |
 | websearch   | Runs a websearch using the built-in websearch tool        |
-| webview     | Extracts content from a web page                          |
+| webview     | Extracts content from a web page. Alias for `read` tool   |
+| write       | Writes content to file                                    |
+| *.md        | Loads a [`skill.md`](https://agentskills.io/specification) file |
+
+## instructions
+
+```yaml
+instructions: string|path
+```
+
+Supports loading an `agents.md` file. Can be provided directly as a string or as a path to a file.
+
+[Read more about agents.md here](https://github.com/agentsmd/agents.md)
+
+## template
+
+```yaml
+template: string
+```
+
+Customize the prompt template used by this agent. Supports Jinja templates. Uses a default template when this parameter is not provided.
+Must include `{{ text }}` and `{{ memory }}` placeholders.
+
+## memory
+
+```yaml
+memory: int
+```
+
+Keeps a rolling window of `memory` inputs and outputs. These are added to future prompts and serve as "agent memory".
+
+Supports storing memory by `session` to enable multiple conversation threads. Defaults to shared memory when not set. See the [method documentation](../methods#txtai.agent.base.Agent.__call__) for more information.
 
 ## method
 

docs/agent/index.md

@@ -5,6 +5,8 @@
 An agent automatically creates workflows to answer multi-faceted user requests. Agents iteratively prompt and/or interface with tools to
 step through a process and ultimately come to an answer for a request.
 
+Agent prompting with [`agents.md`](https://github.com/agentsmd/agents.md) and [`skill.md`](https://agentskills.io/specification) are also supported. [Read the configuration](./configuration/#tool) for more on how to setup those up.
+
 Agents excel at complex tasks where multiple tools and/or methods are required. They incorporate a level of randomness similar to different
 people working on the same task. When the request is simple and/or there is a rule-based process, other methods such as RAG and Workflows
 should be explored.
@@ -149,3 +151,6 @@ Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/ma
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
 | [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
 | [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
+| [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |

docs/api/configuration.md

@@ -33,7 +33,16 @@ Path to save and load the embeddings index. Each API instance can only access a
 writable: boolean
 ```
 
-Determines if the input embeddings index is writable (true) or read-only (false). This allows serving a read-only index.
+Sets this embeddings index as writable (true) or read-only (false). This allows serving a read-only index. Defaults to read-only.
+
+### reindex
+```yaml
+reindex: boolean
+```
+
+Enables re-indexing for this embeddings index (defaults to disabled).
+
+Re-indexing accepts new configuration and should only be enabled when the configuration comes from trusted sources.
 
 ### cloud
 [Cloud storage settings](../../embeddings/configuration/cloud) can be set under a `cloud` top level configuration group.

docs/api/mcp.md

@@ -5,9 +5,22 @@ The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction)
 The API can be configured to handle MCP requests. All enabled endpoints set in the API configuration are automatically added as MCP tools.
 
 ```yaml
-mcp: True
+mcp: boolean|dict
 ```
 
+When `mcp` is a boolean, default arguments are used. When `mcp` is a dictionary it supports the following options.
+
+```yaml
+mcp:
+  clientargs: http client options (dict)
+  mcpargs: FastApiMCP options (dict)
+```
+
+See the following links for details on the options.
+
+- [HTTP Client Arguments](https://www.python-httpx.org/api/#asyncclient)
+- [FastApiMCP Arguments](https://github.com/tadata-org/fastapi_mcp/blob/main/fastapi_mcp/server.py#L22)
+
 Once this configuration option is added, a new route is added to the application `/mcp`. 
 
 The [Model Context Protocol Inspector tool](https://www.npmjs.com/package/@modelcontextprotocol/inspector) is a quick way to explore how the MCP tools are exported through this interface.

docs/api/security.md

@@ -6,9 +6,9 @@ The default implementation of an API service runs via HTTP and is fully open. If
 
 The default API service command starts a Uvicorn server as a HTTP service on port 8000. To run a HTTPS service, consider the following options.
 
-- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.org/deployment/#running-behind-nginx) for more.
+- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.dev/deployment/#running-behind-nginx) for more.
 
-- [Uvicorn SSL Certificate](https://www.uvicorn.org/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
+- [Uvicorn SSL Certificate](https://www.uvicorn.dev/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
 
 ## Authorization
 
@@ -31,3 +31,44 @@ curl \
 It's important to note that HTTPS **must** be enabled using one of the methods mentioned above. Otherwise, tokens will be exchanged as clear text. 
 
 Authentication and Authorization can be fully customized. See the [dependencies](../customization#dependencies) section for more.
+
+## Safe Open
+
+Many components in txtai work with local files and have no restrictions on what files and urls can be accessed. The [textractor pipeline](../../pipeline/data/textractor) and [retrieve](../../workflow/task/retrieve) tasks both have a built-in option called `safeopen` which is enabled by default when running the API and applications. It enforces the following rules:
+
+- Local files must be in the safeopen directory (defaults to temp dir)
+- URLs must be public URLs (also has checks for redirects and DNS rebind attempts)
+
+The following workflow only allows accessing files stored in a specific directory along with public urls.
+
+```yaml
+workflow:
+  textract:
+    tasks:
+      - action: textractor
+```
+
+A number of of pipelines support reading local files. For these it's recommended to wrap those pipelines as workflows with a `retrieve` task. Note that pipelines can be created in-line and don't need to be exposed via the API.
+
+```yaml
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```
+
+Additionally, internal pipelines can be created but not exposed via the API.
+
+```yaml
+routes:
+  caption: False
+
+caption:
+
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```

docs/cloud.md

@@ -30,8 +30,8 @@ Examples build commands below.
 # Get Dockerfile
 wget https://raw.githubusercontent.com/neuml/txtai/master/docker/base/Dockerfile
 
-# Build Ubuntu 22.04 image running Python 3.10
-docker build -t txtai --build-arg BASE_IMAGE=ubuntu:22.04 --build-arg PYTHON_VERSION=3.10 .
+# Build Ubuntu 22.04 image running Python 3.11
+docker build -t txtai --build-arg BASE_IMAGE=ubuntu:22.04 --build-arg PYTHON_VERSION=3.11 .
 
 # Build image with GPU support
 docker build -t txtai --build-arg GPU=1 .

docs/embeddings/configuration/ann.md

@@ -4,7 +4,7 @@ Approximate Nearest Neighbor (ANN) index configuration for storing vector embedd
 
 ## backend
 ```yaml
-backend: faiss|hnsw|annoy|ggml|numpy|torch|pgvector|sqlite|custom
+backend: faiss|hnsw|annoy|ggml|milvus|numpy|rabitq|torch|turbovec|zvec|pgvector|sqlite|custom
 ```
 
 Sets the ANN backend. Defaults to `faiss`. Additional backends are available via the [ann](../../../install/#ann) extras package. Set custom backends via setting this parameter to the fully resolvable class string.
@@ -74,9 +74,14 @@ The [GGML](https://github.com/ggml-org/ggml) backend is a k-nearest neighbors ba
 
 [See this](https://github.com/ggml-org/ggml/blob/master/include/ggml.h#L379) for a list of quantization types.
 
-### numpy
+### milvus
 
-The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets that fit into memory.
+```yaml
+milvus:
+    m: number of HNSW links per element (int) - defaults to 50
+```
+
+### numpy
 
 ```yaml
 numpy:
@@ -84,12 +89,30 @@ numpy:
                  defaults to NumPy array storage
 ```
 
+The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets that fit into memory.
+
+### rabitq
+
+```yaml
+rabitq:
+    mode: index mode (ivf or hnsw) - defaults to "ivf"
+    nbits: number of quantization bits per dimension (int) - defaults to 1,
+           supports 1 to 9 in both modes, ivf mode also supports 32 which
+           keeps raw vectors for reranking
+    clusters: number of IVF clusters (int) - defaults to
+              max(1, min(round(4 * sqrt(embeddings count)), embeddings count))
+    nprobe: search probe setting for ivf mode (int) - defaults to
+            max(1, round(num_clusters/16))
+    m: M param for hnsw mode (int) - defaults to 16
+    efconstruction: ef_construction param for hnsw mode (int) - defaults to 200
+    efsearch: ef search param for hnsw mode (int) - defaults to None and not set
+    randomseed: random-seed param for hnsw mode (int) - defaults to 100
+```
+
+The [rabitq](https://github.com/VectorDB-NTU/RaBitQ-Library) backend is a quantized index powered by the [RaBitQ algorithm](https://doi.org/10.1145/3725413). It supports ivf and hnsw search modes. Vectors are stored with 1-bit quantization by default. More bits increase accuracy at the cost of memory. The index is stored as a single file and no vectors are kept outside of it. With nbits set to 32, the vectors are stored in the index for reranking. The ivf mode supports upserts and deletes. Deleted rows keep their storage. Rows added after the first index are assigned to the clusters trained when the index was built, so recall can drop if the new data differs from the original data. Rebuild the index to train new clusters. Note that hnsw indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+
 ### torch
 
-The Torch backend is a k-nearest neighbors backend like NumPy. It supports GPU-enabled operations. It also has support for quantization which enables fitting larger arrays into GPU memory.
-
-When quantization is enabled, vectors are _always_ stored in safetensors. _Note that macOS support for quantization is limited._
-
 ```yaml
 torch:
     safetensors: stores vectors using the safetensors format - defaults
@@ -99,6 +122,30 @@ torch:
         blocksize: quantization block size parameter
 ```
 
+The Torch backend is a k-nearest neighbors backend like NumPy. It supports GPU-enabled operations. It also has support for quantization which enables fitting larger arrays into GPU memory.
+
+When quantization is enabled, vectors are _always_ stored in safetensors. _Note that macOS support for quantization is limited._
+
+### turbovec
+
+```yaml
+turbovec:
+    bitwidth: number of bits to store each vector dimension as. Supports 2, 3 or 4.
+```
+
+The [turbovec](https://github.com/RyanCodrai/turbovec) backend is a k-nearest neighbors backend powered by the [TurboQuant algorithm](https://arxiv.org/abs/2504.19874).
+
+### zvec
+
+```yaml
+zvec:
+    efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+    m: number of HNSW links per element (int) - defaults to 50
+    efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+The [zvec](https://github.com/alibaba/zvec) backend is an embedded, path-based vector index.
+
 ### pgvector
 ```yaml
 pgvector:

docs/embeddings/configuration/database.md

@@ -50,14 +50,27 @@ Object encoding options are:
 functions: list
 ```
 
-List of functions with user-defined SQL functions, only used when [content](#content) is enabled. Each list element must be one of the following:
+List of functions with user-defined SQL functions. Each list element must be one of the following:
 
 - function
 - callable object
-- dict with fields for name, argcount and function
+- dict with fields for name, argcount, function and deterministic
 
 [An example can be found here](../../query#custom-sql-functions).
 
+## expressions
+```yaml
+expressions: list
+```
+
+List of expression shortcuts. Each list element must be a dict with the following fields.
+
+- `name`: name of the expression
+- `expression`: SQL expression, defaults to `name` when empty
+- `index`: if this expression should have a database index, defaults to False when not provided
+
+The expression can be a json data column, sql function or anything that can be run as a SQL snippet.
+
 ## query
 ```yaml
 query:

docs/embeddings/configuration/general.md

@@ -66,13 +66,18 @@ Sets the auto id generation method. When this is not set, an autogenerated numer
 columns:
     text: name of the text column
     object: name of the object column
+    store: limit json data fields to this list of columns
 ```
 
 Sets the `text` and `object` column names. Defaults to `text` and `object` if not provided.
 
+`store` sets a list of columns to store in the JSON data field. When this isn't provided, all columns are stored (default). When `store` is set to `None`, no JSON columns are stored. This is useful is a field is only needed at indexing time but not search time.
+
 ## format
 ```yaml
 format: json|pickle
 ```
 
 Sets the configuration storage format. Defaults to `json`.
+
+Note that `pickle` configuration is deprecated and will raise an error with the default settings. Its only kept for reading legacy indexes and requires setting the `ALLOW_PICKLE` environment variable. Only enable for local and/or trusted sources.

docs/embeddings/configuration/scoring.md

@@ -31,25 +31,30 @@ path: sparse vector model path
 vectormethod: vector embeddings method
 vectornormalize: enable vector embeddings normalization (boolean)
 gpu: boolean|int|string|device
-normalize: enable score normalization (boolean | float)
+normalize: enable score normalization (boolean|float|string|dict)
 batch: Sets the transform batch size
 encodebatch: Sets the encode batch size
 vectors: additional model init args
 encodeargs: additional encode() args
-backend: ivfsparse|pgsparse
+backend: ivfsparse|pgsparse|zvecsparse
 ```
 
 Sparse vector scoring options. The sparse scoring instance combines a sparse vector model with a sparse approximate nearest neighbor index (ANN). This method supports both vector normalization and score normalization.
 
 Vector normalization normalizes all vectors to have a magnitude of 1. By extension, all generated scores will be 0 to 1.
 
-Score normalization scales output scores by a scale factor. When `normalize` is set to `True` it uses a default scale factor. If `normalize` is an integer, then that is used as the scale factor. This scales the scores from 0 to 1.
+Score normalization scales the output between 0 and 1. This setting supports:
+
+- `True` for default scale normalization
+- `float` normalize using this as the scale factor
+- `"bayes"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
 
 #### ivfsparse
 ```yaml
 ivfsparse:
   sample: percent of data to use for model training (0.0 - 1.0)
-  nfeatures: top n features to use for model training (int)
+  nfeatures: top n features to use for model training, defaults to all features (int)
   nlist: desired number of clusters (int)
   nprobe: search probe setting (int)
   minpoints: minimum number of points for a cluster (int)
@@ -61,6 +66,16 @@ Inverted file (IVF) index with flat vector file storage and sparse array support
 
 Sparse ANN backed by Postgres. Supports same options as the [pgvector](../ann/#pgvector) ANN.
 
+#### zvecsparse
+```yaml
+zvecsparse:
+  efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+  m: number of HNSW links per element (int) - defaults to 50
+  efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+Sparse ANN backed by [zvec](https://github.com/alibaba/zvec). Stores sparse vectors in an embedded, path-based vector index. Requires the [ann](../../../install/#ann) extras package.
+
 ## terms
 ```yaml
 terms: boolean|dict
@@ -76,9 +91,38 @@ Supports a `dict` with the parameters `cachelimit` and `cutoff`.
 
 When `terms` is set to `True`, default parameters are used for the `cachelimit` and `cutoff`. Normally, these defaults are sufficient.
 
+### Wildcard queries
+
+Keyword queries containing `*` expand matching terms using SQL `LIKE`. Within these
+expressions, `%` matches any sequence and `_` matches one character. To match a literal
+percent sign, underscore or backslash, use `\%`, `\_` or `\\`, respectively.
+For example, `report\_*` matches `report_2026`, while `report_*` also matches `reportX2026`.
+Use a tokenizer that preserves these characters, such as `tokenizer: {whitespace: true}`,
+or pass pre-tokenized query terms. Queries without `*` retain exact-term matching.
+
 ## normalize
 ```yaml
-normalize: boolean
+normalize: boolean|str|dict
 ```
 
-Enables normalized scoring (ranging from 0 to 1). When enabled, statistics from the index will be used to calculate normalized scores.
+Enables normalized scoring (ranging from 0 to 1). This setting supports:
+
+- `True` for standard score normalization
+- `"bayes"` | `"bb25"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
+
+When standard normalization is enabled, statistics from the index are used to calculate normalized scores.
+When Bayesian/BB25 normalization is enabled, it uses positive-score candidates, dynamic `beta=median(scores)`, adaptive
+`alpha_eff=alpha/std(scores)` and a sigmoid transform (likelihood-only variant with flat prior) to map scores to `[0, 1]`.
+
+Bayesian normalization references:
+
+- [https://github.com/instructkr/bb25](https://github.com/instructkr/bb25)
+- [https://github.com/cognica-io/bayesian-bm25](https://github.com/cognica-io/bayesian-bm25)
+
+## tokenizer
+```yaml
+tokenizer: dict
+```
+
+Set tokenization rules. Passes these arguments to the underlying [Tokenization pipeline](../../../pipeline/data/tokenizer#txtai.pipeline.Tokenizer.__init__).

docs/embeddings/configuration/vectors.md

@@ -12,7 +12,8 @@ Sets the path for a vectors model. When using a transformers/sentence-transforme
 
 ## method
 ```yaml
-method: transformers|sentence-transformers|llama.cpp|litellm|model2vec|external|words
+method: transformers|sentence-transformers|llama.cpp|litellm|model2vec|external|
+        words
 ```
 
 Embeddings method to use. If the method is not provided, it is inferred using the `path`.
@@ -24,11 +25,12 @@ Embeddings method to use. If the method is not provided, it is inferred using th
 Builds embeddings using a transformers model. While this can be any transformers model, it works best with
 [models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.
 
-`mean`, `cls` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method
-from `transformers` to `meanpooling`, `clspooling` or `latepooling` respectively.
+`mean`, `cls`, `last`, `late` and `max` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method from `transformers` to `meanpooling`, `clspooling`, `lastpooling`, `latepooling` or `maxpooling` respectively.
 
 Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.
 
+Supports loading ONNX models directly. [See this section for more](../../../examples/#model-training).
+
 ### sentence-transformers
 
 Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
@@ -65,7 +67,9 @@ Embeddings are created via an external model or API. Requires setting the [trans
 transform: function
 ```
 
-When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array or list of numpy arrays.
+When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array, list of numpy arrays or list of float arrays.
+
+Note that this method is deprecated and will raise an error with the default settings. Adding a custom `Vectors` implementation is a cleaner approach. The `ALLOW_RESOLVE_TRANSFORM` environment variable must be set to enable. Only enable for trusted and/or reviewed code.
 
 ## gpu
 ```yaml
@@ -104,7 +108,7 @@ quantize: int|boolean
 
 Enables scalar vector quantization at the specified precision. Supports 1-bit through 8-bit quantization. Scalar quantization transforms continuous floating point values to discrete unsigned integers. The `faiss`, `pgvector`, `numpy` and `torch` ANN backends support storing these vectors.
 
-This parameter supports booleans for backwards compatability. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
+This parameter supports booleans for backwards compatibility. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
 
 In addition to vector-level quantization, some ANN backends have the ability to quantize vectors at the storage layer. See the [ANN](../ann) configuration options for more.
 
@@ -117,7 +121,7 @@ instructions:
 
 Instruction-based models use prefixes to modify how embeddings are computed. This is especially useful with asymmetric search, which is when the query and indexed data are of vastly different lengths. In other words, short queries with long documents.
 
-[E5-base](https://huggingface.co/intfloat/e5-base) is an example of a model that accepts instructions. It takes `query: ` and `passage: ` prefixes and uses those to generate embeddings that work well for asymmetric search.
+`txtai` automatically loads prompts stored in `config_sentence_transformers.json` except if this parameter is set. For some older models such as [E5-base](https://huggingface.co/intfloat/e5-base), instructions still need to be provided via this parameter.
 
 ## models
 ```yaml
@@ -140,6 +144,29 @@ vectors: dict
 
 Passes these additional parameters to the underlying vector model.
 
+### center
+```yaml
+center: boolean | dict
+```
+
+Centers normalized late-interaction token vectors and normalizes them again before MUVERA or LEMUR encoding. Centering is applied to
+both query and document vectors. `true` uses batch scope and `false` disables centering. An explicit setting always takes precedence.
+When omitted, a LEMUR artifact with a stored collection mean uses collection scope. Other LEMUR artifacts and late-interaction models
+default to batch scope only when they load more than one `torch.nn.Linear` layer; models with zero or one loaded linear layer keep their
+previous output.
+
+Dictionary configuration supports the following scopes:
+
+```yaml
+center:
+    scope: document | batch | collection
+```
+
+`document` subtracts the mean of each document or query independently. `batch` subtracts the mean of all real token rows in the current
+model batch. Explicit `collection` scope requires a caller-provided, one-dimensional mean vector through either `mean` (an array-like
+value) or `path` (a safetensors file containing a `center.mean` tensor), but not both. LEMUR artifacts can provide this mean
+automatically. Zero-padding rows are never centered and remain zero.
+
 ### muvera
 ```yaml
 muvera:
@@ -149,6 +176,21 @@ muvera:
 ```
 
 Settings to control the size of MUVERA fixed dimensional outputs. Default is 20 * 2^5 * 16 = 10,240 dimensions.
+Set `muvera` to `false` to disable it or `true` to use the default settings. `false` only makes sense when the multi-vector output
+is consumed directly, for example through `PoolingFactory`. Embeddings vectors require `muvera` or `lemur`.
+
+### lemur
+```yaml
+lemur:
+    path: local artifact directory or Hugging Face Hub path
+```
+
+Loads a trained [LEMUR](../../../pipeline/train/lemur) fixed dimensional encoder. LEMUR artifacts are corpus-specific and must be
+created with `LemurTrainer` before configuring an embeddings index. A bare string is treated as the artifact path. Query vectors are
+summed learned features and document vectors are ordinary least squares weights over the artifact's stored token sample. Each artifact
+contains `config.json` and `model.safetensors`; newly trained artifacts also store the collection token mean as `lemur.center`. Loading that mean automatically
+selects collection centering unless `center` is explicitly configured. Artifacts without `lemur.center` retain the previous default.
+The Safetensors file contains inference state only. See the trainer page for score filtering and Faiss exact/IVF search behavior.
 
 ### trust_remote_code
 ```yaml

docs/embeddings/indexing.md

@@ -23,6 +23,8 @@ As mentioned above, computed vectors are stored in an ANN. There are various ind
 
 Embeddings indexes can optionally [store content](../configuration/database#content). When this is enabled, the input content is saved in a database alongside the computed vectors. This enables filtering on additional fields and content retrieval.
 
+The columns used for text, object and JSON data storage are set via [column configuration](../configuration/general#columns).
+
 ## Index vs Upsert
 
 Data is loaded into an index with either an [index](../methods#txtai.embeddings.base.Embeddings.index) or [upsert](../methods#txtai.embeddings.base.Embeddings.upsert) call.

docs/embeddings/query.md

@@ -54,6 +54,8 @@ The number of candidates should be larger than the desired number of results whe
 - For a single query filter clause, the default is the query limit
 - With multiple filtering clauses, the default is 10x the query limit
 
+When a query orders results by score, its `offset` adds to the query limit.
+
 The index name is only applicable when [subindexes](../configuration/general/#indexes) are enabled. This specifies the index to use for the query.
 
 Weights sets the hybrid score weights when an index has both a sparse and dense index.
@@ -132,8 +134,11 @@ embeddings = Embeddings(content=True, objects=True)
 request = open("demo.gif", "rb")
 
 # Insert record
-embeddings.index([("txtai", {"text": "txtai executes machine-learning workflows.",
-                             "object": request.read()})])
+embeddings.index([(
+  "txtai",
+  {"text": "txtai executes machine-learning workflows.",
+   "object": request.read()}
+)])
 
 # Query txtai and get associated object
 query = "SELECT object FROM txtai WHERE similar('machine learning') LIMIT 1"
@@ -169,6 +174,26 @@ LIMIT 1
 """)
 ```
 
+## Expressions
+
+Expression shortcuts expand into more complex SQL snippets. This is useful for making SQL queries more concise. Indexing is also available on expressions as a performance improvement.
+
+The following example indexes a json extraction field (`filepath`) and the length of each field.
+
+```python
+# Create embeddings index
+embeddings = Embeddings(
+  path="sentence-transformers/nli-mpnet-base-v2",
+  content=True,
+  expressions=[
+    {"name": "filepath", "index": True},
+    {"name": "textlength", "expression": "length(text)", "index": True}
+  ]
+)
+
+embeddings.search("SELECT textlength, filepath FROM txtai LIMIT 1")
+```
+
 ## Query translation
 
 Natural language queries with filters can be converted to txtai-compatible SQL statements with query translation. For example:
@@ -192,7 +217,9 @@ When an embeddings database has both a sparse and dense index, both indexes will
 
 ```python
 embeddings.search("query", weights=0.5)
-embeddings.search("SELECT id, text, score FROM txtai WHERE similar('query', 0.5)")
+embeddings.search(
+  "SELECT id, text, score FROM txtai WHERE similar('query', 0.5)"
+)
 ```
 
 ## Graph search
@@ -203,11 +230,20 @@ Additionally, standard embeddings searches can be returned as graphs.
 
 ```python
 # Find all paths between id: 0 and id: 5 between 1 and 3 hops away
-embeddings.graph.search("""
+embeddings.search("""
 MATCH P=({id: 0})-[*1..3]->({id: 5})
 RETURN P
 """)
 
+# Find related nodes for query matches
+embeddings.search("""
+  MATCH P=(A)-[]->(B)
+  WHERE SIMILAR(A, "query")
+  RETURN B
+  ORDER BY A.score DESC
+  LIMIT 10
+""")
+
 # Standard embeddings search as graph
 embeddings.search("query", graph=True)
 ```
@@ -217,13 +253,28 @@ embeddings.search("query", graph=True)
 Subindexes can be queried as follows:
 
 ```python
+# Build index with subindexes
+embeddings = Embeddings(
+  content=True,
+  defaults=False,
+  indexes={
+    "keyword": {
+      "keyword": True
+    },
+    "dense":{
+      "dense": True
+    }
+  }
+)
+embeddings.index(stream())
+
 # Query with index parameter
-embeddings.search("query", index="subindex1")
+embeddings.search("query", index="keyword")
 
 # Specify with SQL
 embeddings.search("""
 SELECT id, text, score FROM txtai
-WHERE similar('query', 'subindex1')
+WHERE similar('query', 'keyword')
 """)
 ```
 

docs/examples.md

@@ -31,7 +31,7 @@ Autonomous agents, retrieval augmented generation (RAG), chat with your data, pi
 |:----------|:-------------|------:|
 | [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
 | [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
-| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
 | [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
 | [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
 | [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
@@ -49,6 +49,10 @@ Autonomous agents, retrieval augmented generation (RAG), chat with your data, pi
 | [Medical RAG Research with txtai](https://github.com/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) | Analyze PubMed article metadata with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) |
 | [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
 | [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
+| [OpenCode as a txtai LLM](https://github.com/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) | Integrate OpenCode with the txtai ecosystem | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) |
+| [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify a list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 
 ## Pipelines
 
@@ -86,7 +90,7 @@ Efficiently process data at scale.
 
 ## Model Training
 
-Train NLP models.
+Train, distill, fine-tune and export models.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
@@ -94,6 +98,8 @@ Train NLP models.
 | [Train without labels](https://github.com/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) | Use zero-shot classifiers to train new models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) |
 | [Train a QA model](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | Build and fine-tune question-answering models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
 | [Train a language model from scratch](https://github.com/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) | Build new language models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) |
+| [Distilling Knowledge into Tiny LLMs](https://github.com/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) [▶️](https://www.youtube.com/watch?v=Ol560ktgkf0) | Finetune tiny LLMs to enable inference using less resources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) |
+| [Progressive Distillation](https://github.com/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) | Incrementally compress knowledge from a series of teachers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) |
 | [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
 | [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
 

docs/faq.md

@@ -105,11 +105,12 @@ __Solution__
 
 Set the following environment parameters.
 
-- OpenMP threading is handled internally on macOS platforms but it can be disabled via `export OMP_NUM_THREADS=1`
+- Disable OpenMP multithreading via `export OMP_NUM_THREADS=1`
+- Workaround `OMP: Error #15` errors via `export KMP_DUPLICATE_LIB_OK=TRUE`
 - Disable PyTorch MPS device via `export PYTORCH_MPS_DISABLE=1`
 - Disable llama.cpp metal via `export LLAMA_NO_METAL=1`
 
-For more details, refer to [this issue on GitHub](https://github.com/kyamagu/faiss-wheels/issues/100).
+For more details, refer to [this issue on GitHub](https://github.com/kyamagu/faiss-wheels/issues/73).
 
 ----------
 

docs/further.md

@@ -4,6 +4,7 @@
 ![further](images/further-dark.png#only-dark)
 
 - [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
 - [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)

docs/index.md

@@ -51,7 +51,7 @@ Summary of txtai features:
 - 🔋 Batteries included with defaults to get up and running fast
 - ☁️ Run local or scale out with container orchestration
 
-txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
 !!! note
 

docs/install.md

@@ -9,7 +9,7 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
 ## Optional dependencies
 
@@ -130,7 +130,11 @@ Additional environment specific prerequisites are below.
 
 ### Linux
 
-The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library. The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.
+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library.
+
+The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.
+
+The LiteRT LLM pipeline requires libegl1, libgles2 and libvulkan1.
 
 ### macOS
 
@@ -146,7 +150,7 @@ The [txtai build workflow](https://github.com/neuml/txtai/blob/master/.github/wo
 
 ## CPU-only
 
-The default install adds PyTorch with GPU support. There are a number of dependencies that come with that. When running in a CPU-only environment or using Embeddings/LLM models without PyTorch (i.e. llama.cpp or API services), the CPU-only PyTorch package can be installed with txtai as follows.
+The default install adds PyTorch with GPU support. There are a number of dependencies that come with that. When running in a CPU-only environment, the CPU-only PyTorch package can be installed with txtai as follows.
 
 ```
 pip install txtai torch==[version]+cpu \
@@ -173,6 +177,22 @@ A [community-supported txtai package](https://anaconda.org/conda-forge/txtai) is
 conda install -c conda-forge txtai
 ```
 
+## Minimal install
+
+A lightweight zero dependency minimal install package is available. This is helpful if the default libraries are too heavy or there are no plans to use them.
+
+All the same extras mentioned above are available in this version of the package.
+
+```
+# Lightweight minimal install
+pip install txtai_minimal
+
+# Same as standard `txtai` install
+pip install txtai_minimal[default]
+```
+
+Note that in order to use the `Embeddings` or `LLM` interface without `torch` requires either `llama.cpp`, `litellm` or `litert`.
+
 ## Run with containers
 
 Docker images are available for txtai. [See this section](../cloud) for more information on container-based installs.

docs/models.md

@@ -8,9 +8,9 @@ See the table below for the current recommended models. These models all allow c
 | ---------------------------------------------------- | ------------------------------------------------------------------------ |
 | [Embeddings](../embeddings)                          | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
 | [Image Captions](./pipeline/image/caption.md)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
-| [Labels - Zero Shot](./pipeline/text/labels.md)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Zero Shot](./pipeline/text/labels.md)      | [DeBERTa v3 Zeroshot](https://hf.co/MoritzLaurer/deberta-v3-base-zeroshot-v2.0-c)                     |
 | [Labels - Fixed](./pipeline/text/labels.md)          | Fine-tune with [training pipeline](./pipeline/train/trainer.md)          |
-| [Large Language Model (LLM)](./pipeline/text/llm.md) | [gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b)                 |
+| [Large Language Model (LLM)](./pipeline/text/llm.md) | [Gemma 4 31B](https://hf.co/google/gemma-4-31B)                          |
 | [Summarization](./pipeline/text/summary.md)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
 | [Text-to-Speech](./pipeline/audio/texttospeech.md)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
 | [Transcription](./pipeline/audio/transcription.md)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
@@ -22,6 +22,5 @@ See the following links to learn more.
 
 - [Hugging Face Tasks](https://hf.co/tasks)
 - [Hugging Face Model Hub](https://hf.co/models)
-- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
-- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
-- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+- [Embeddings Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LLM Leaderboard](https://hf.co/spaces/lmarena-ai/arena-leaderboard)

docs/pipeline/data/filetohtml.md

@@ -17,6 +17,12 @@ Apache Tika requires [Java](https://en.wikipedia.org/wiki/Java_(programming_lang
 
 See [this link](https://github.com/DS4SD/docling?tab=readme-ov-file#features) for a list of supported document formats.
 
+## LiteParse
+
+[LiteParse](https://github.com/run-llama/liteparse) is a fast, helpful, and open-source document parser. It supports PDF out of the box with additional format support via third-party packages.
+
+See [this link](https://github.com/run-llama/liteparse#multi-format-input-support) for more.
+
 ## Example
 
 The following shows a simple example using this pipeline.

docs/pipeline/data/textractor.md

@@ -16,6 +16,8 @@ The [backend](../filetohtml/#txtai.pipeline.FileToHTML.__init__) parameter sets
 
 See the [FiletoHTML](../filetohtml) and [HTMLToMarkdown](../htmltomd) pipelines to learn more on the dependencies necessary for each of those pipelines.
 
+Note that the default parameters enable access to all local files and all URLs. The `safeopen` parameter limits this to only files in a specified directory (defaults to temp directory) and public URLs. When running the textractor pipeline through an Application or via the API, `safeopen` defaults to True. It can be disabled if desired via configuration.
+
 ## Example
 
 The following shows a simple example using this pipeline.

docs/pipeline/data/tokenizer.md

@@ -0,0 +1,82 @@
+# Tokenizer
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+The Tokenizer pipeline splits text into tokens. This is primarily used for keyword / term indexing.
+
+_Note: Transformers-based models have their own tokenizers and this pipeline isn't designed for working with Transformers models._
+
+## Example
+
+The following shows a simple example using this pipeline.
+
+```python
+from txtai.pipeline import Tokenizer
+
+# Create and run pipeline
+tokenizer = Tokenizer()
+tokenizer("text to tokenize")
+
+# Whitespace tokenization
+tokenizer = Tokenizer(whitespace=True)
+tokenizer("text to tokenize")
+
+# Tokenize using a regular expression
+tokenizer = Tokenizer(regexp=r"\w{5,}")
+tokenizer("text to tokenize")
+
+# Tokenize into trigrams like pg_trgm
+tokenizer = Tokenizer(ngrams={
+  "ngrams": 3, "lpad": "  ", "rpad": " ", "unique": True
+})
+tokenize("text to tokenize")
+
+# Tokenize into edge ngrams
+tokenizer = Tokenizer(ngrams={"nmin": 2, "nmax": 5, "edge": True})
+tokenizer("text to tokenize")
+```
+
+## Configuration-driven example
+
+Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
+
+### config.yml
+```yaml
+# Create pipeline using lower case class name
+tokenizer:
+
+# Run pipeline with workflow
+workflow:
+  tokenizer:
+    tasks:
+      - action: tokenizer
+```
+
+### Run with Workflows
+
+```python
+from txtai import Application
+
+# Create and run pipeline with workflow
+app = Application("config.yml")
+list(app.workflow("tokenizer", ["text to tokenize"]))
+```
+
+### Run with API
+
+```bash
+CONFIG=config.yml uvicorn "txtai.api:app" &
+
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"tokenizer", "elements":["text"]}'
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.Tokenizer.__init__
+### ::: txtai.pipeline.Tokenizer.__call__

docs/pipeline/data/urlretrieve.md

@@ -0,0 +1,62 @@
+# URL Retrieve
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+The URL Retrieve pipeline retrieves content from a HTTP(s) URL.
+
+## Example
+
+The following shows a simple example using this pipeline.
+
+```python
+from txtai.pipeline import URLRetrieve
+
+# Create and run pipeline
+urlretrieve = URLRetrieve()
+urlretrieve("https://github.com/neuml/txtai")
+```
+
+## Configuration-driven example
+
+Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
+
+### config.yml
+```yaml
+# Create pipeline using lower case class name
+urlretrieve:
+
+# Run pipeline with workflow
+workflow:
+  retrieve:
+    tasks:
+      - action: urlretrieve
+```
+
+### Run with Workflows
+
+```python
+from txtai import Application
+
+# Create and run pipeline with workflow
+app = Application("config.yml")
+list(app.workflow("urlretrieve", ["https://github.com/neuml/txtai"]))
+```
+
+### Run with API
+
+```bash
+CONFIG=config.yml uvicorn "txtai.api:app" &
+
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"retrieve", "elements":["http://github.com/neuml/txtai"]}'
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.URLRetrieve.__init__
+### ::: txtai.pipeline.URLRetrieve.__call__

docs/pipeline/index.md

@@ -26,6 +26,8 @@ The `LLM` and `RAG` pipelines also have integrations for [llama.cpp](https://git
     - [Segmentation](data/segmentation)
     - [Tabular](data/tabular)
     - [Text extraction](data/textractor)
+    - [Tokenizer](data/tokenizer)
+    - [URLRetrieve](data/urlretrieve)
 - Image
     - [Caption](image/caption)
     - [Image Hash](image/imagehash)

docs/pipeline/llm/llm.md

@@ -58,11 +58,19 @@ llm("Answer the following question...", defaultrole="user")
 llm("Answer the following question...", defaultrole="prompt")
 ```
 
-The LLM pipeline automatically detects the underlying LLM framework. This can also be manually set.
+The LLM pipeline automatically detects the underlying LLM framework. This can also be manually set. The following methods are supported.
 
-[Hugging Face Transformers](https://github.com/huggingface/transformers), [llama.cpp](https://github.com/abetlen/llama-cpp-python) and [hosted API models via LiteLLM](https://github.com/BerriAI/litellm) are all supported by this pipeline.
+- [Hugging Face Transformers](https://github.com/huggingface/transformers)
+- [llama.cpp](https://github.com/abetlen/llama-cpp-python)
+- [LLM APIs via LiteLLM](https://github.com/BerriAI/litellm)
+- [LiteRT-LM](https://ai.google.dev/edge/litert-lm)
+- [OpenCode server](https://github.com/anomalyco/opencode)
 
-See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models. llama.cpp models support both local and remote GGUF paths on the HF Hub.
+`llama.cpp` and `LiteRT-LM` support both local and remote file paths on the HF Hub.
+
+See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models.
+
+See the [OpenCode documentation](https://opencode.ai/docs/server/) for more on how to integrate the LLM pipeline with a running OpenCode instance.
 
 ```python
 from txtai import LLM
@@ -80,6 +88,15 @@ llm = LLM("unsloth/gpt-oss-20b-GGUF/gpt-oss-20b-Q4_K_M.gguf",
 llm = LLM("ollama/gpt-oss")
 llm = LLM("ollama/gpt-oss", method="litellm")
 
+# LiteRT-LM
+llm = LLM(
+  "litert-community/gemma-4-E2B-it-litert-lm/gemma-4-E2B-it.litertlm"
+)
+llm = LLM(
+  "litert-community/gemma-4-E2B-it-litert-lm/gemma-4-E2B-it.litertlm",
+  method="litert"
+)
+
 # Custom Ollama endpoint
 llm = LLM("ollama/gpt-oss", api_base="http://localhost:11434")
 
@@ -87,9 +104,13 @@ llm = LLM("ollama/gpt-oss", api_base="http://localhost:11434")
 llm = LLM("openai/gpt-oss", api_base="http://localhost:4000")
 
 # LLM APIs - must also set API key via environment variable
-llm = LLM("gpt-5.1")
+llm = LLM("gpt-5.2")
 llm = LLM("claude-opus-4-5-20251101")
 llm = LLM("gemini/gemini-3-pro-preview")
+
+# Local OpenCode server started via `opencode serve`
+llm = LLM("opencode")
+llm = LLM("opencode/big-pickle", url="http://localhost:4000")
 ```
 
 Models can be externally loaded and passed to pipelines. This is useful for models that are not yet supported by Transformers and/or need special initialization.
@@ -117,7 +138,7 @@ See the links below for more detailed examples.
 |:----------|:-------------|------:|
 | [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
 | [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
-| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
 | [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
 | [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
 | [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
@@ -126,7 +147,6 @@ See the links below for more detailed examples.
 | [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
 | [How RAG with txtai works](https://github.com/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) | Create RAG processes, API services and Docker instances | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) |
 | [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |
-| [Generative Audio](https://github.com/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) | Storytelling with generative audio workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) |
 | [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
 | [Getting started with LLM APIs](https://github.com/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) | Generate embeddings and run LLMs with OpenAI, Claude, Gemini, Bedrock and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) |
@@ -136,6 +156,10 @@ See the links below for more detailed examples.
 | [Medical RAG Research with txtai](https://github.com/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) | Analyze PubMed article metadata with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) |
 | [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
 | [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
+| [OpenCode as a txtai LLM](https://github.com/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) | Integrate OpenCode with the txtai ecosystem | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) |
+| [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify a list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 
 ## Configuration-driven example
 

docs/pipeline/llm/rag.md

@@ -93,24 +93,14 @@ Check out this [RAG Quickstart Example](https://github.com/neuml/txtai/blob/mast
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
 | [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
-| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
-| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
 | [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
 | [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
-| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
 | [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
 | [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
 | [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
 | [How RAG with txtai works](https://github.com/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) | Create RAG processes, API services and Docker instances | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) |
 | [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |
-| [Generative Audio](https://github.com/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) | Storytelling with generative audio workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) |
-| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
-| [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
-| [Getting started with LLM APIs](https://github.com/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) | Generate embeddings and run LLMs with OpenAI, Claude, Gemini, Bedrock and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) |
-| [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
-| [Extractive QA with txtai](https://github.com/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) | Introduction to extractive question-answering with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) |
-| [Extractive QA with Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) | Run extractive question-answering queries with Elasticsearch | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) |
-| [Extractive QA to build structured data](https://github.com/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) | Build structured datasets using extractive question-answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) |
 | [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
 | [Chunking your data for RAG](https://github.com/neuml/txtai/blob/master/examples/73_Chunking_your_data_for_RAG.ipynb) | Extract, chunk and index content for effective retrieval | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/73_Chunking_your_data_for_RAG.ipynb) |
 | [Medical RAG Research with txtai](https://github.com/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) | Analyze PubMed article metadata with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) |

docs/pipeline/train/lemur.md

@@ -0,0 +1,98 @@
+# LemurTrainer
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+Trains a LEMUR fixed dimensional encoder for a late interaction model and corpus. Training is a separate pipeline step; the saved
+artifact is then loaded through the embeddings `vectors.lemur` configuration.
+
+`LemurTrainer` owns training-data construction, feature-model setup, epoch training and validation-based selection. The loaded LEMUR
+pooler remains responsible for query-feature and document-weight encoding.
+
+## Example
+
+```python
+from txtai.pipeline import LemurTrainer
+
+corpus = [
+    "First document",
+    "Second document",
+    "Third document",
+]
+
+trainer = LemurTrainer()
+trainer(
+    "colbert-ir/colbertv2.0",
+    corpus,
+    "lemur-model",
+    epochs=100,
+    validationsplit=0.1,
+)
+```
+
+`epochs` is required so the training mode is explicit. Use `epochs=100` for the quality-oriented MLP setting. This can take hours on
+a CPU. Use `epochs=0` to select deterministic random ELM features as a lower-cost fallback.
+
+The learn distribution defaults to `learncategory="query"`. Target documents are always encoded with the data encoder, so the
+default encodes selected corpus texts once as data and once as queries. Set `learncategory="data"` to reuse the data encodings, or
+pass a separate iterable of texts with `learn`.
+
+When `vectors.center` is omitted, the trainer encodes the corpus without centering, computes one mean over all corpus token rows and
+uses that mean to center both data and learn vectors before fitting. The collection mean is stored in the LEMUR artifact. An explicit
+`vectors.center` setting keeps its configured behavior and does not store an automatic collection mean.
+
+Set `corpussubsetsize` to a positive integer to sample that many raw corpus texts under `seed` before either encoding pass. The
+default is `None`, which uses every corpus text. `trainsubsetsize` and `learnsubsetsize` apply later, after token vectors have already
+been created. `corpussubsetsize` applies to `data`; a separate `learn` iterable remains caller-sized.
+
+For trained MLP features, set `validationsplit` to a fraction greater than zero and less than one to retain the epoch with the lowest
+held-out loss. The default is `0.0`, which preserves training-loss selection. The selected one-based epoch, loss and metric are
+available as `selectedepoch`, `selectedloss` and `selectionmetric` on the fitted or reloaded encoder.
+
+MLP training displays a `tqdm` progress bar on an interactive terminal, including percent complete and the current validation loss
+when `validationsplit` is enabled (or training loss otherwise). Progress output is disabled for non-interactive runs.
+
+The artifact contains `config.json` and `model.safetensors`. It stores the inference feature model, output-normalization statistics,
+token sample needed to encode documents added after training and, by default, the collection token mean. The training-only output
+readout is not saved.
+
+Load the artifact in an embeddings configuration.
+
+```yaml
+embeddings:
+    path: colbert-ir/colbertv2.0
+    vectors:
+        lemur:
+            path: lemur-model
+```
+
+## Search behavior
+
+Loading a LEMUR artifact with a stored collection mean automatically centers both query and document token vectors with that mean.
+This keeps training and search on the same representation and makes fixed vectors independent of indexing or query batch size. An
+explicit `vectors.center` setting takes precedence. Older artifacts without a stored mean use the standard late-pooling fallback:
+batch centering for models with multiple linear layers and no centering for models with zero or one linear layer.
+
+LEMUR approximates standardized MaxSim targets, so useful ranking scores can be negative. txtai's dense vector path L2-normalizes the
+fixed vectors and removes results with scores less than or equal to zero. This can change ordering and return fewer than the requested
+number of candidates compared with raw maximum inner-product search.
+
+The Faiss backend uses exact `IDMap,Flat` search through 5,000 rows and switches to an IVF index above that threshold. In the measured
+scifact run, default IVF reduced LEMUR NDCG@10 by 43% relative to exact search, compared with 25% for MUVERA. For LEMUR corpora above
+5,000 rows, either pin `faiss.components` to an exact index or tune the IVF settings for the corpus. The exact configuration is:
+
+```yaml
+embeddings:
+    path: colbert-ir/colbertv2.0
+    vectors:
+        lemur:
+            path: lemur-model
+    faiss:
+        components: IDMap,Flat
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.LemurTrainer.__call__

docs/pipeline/train/trainer.md

@@ -98,6 +98,20 @@ See the following PEFT documentation links for more information.
 - [Quantization](https://huggingface.co/docs/peft/developer_guides/quantization)
 - [LoRA](https://huggingface.co/docs/peft/developer_guides/lora)
 
+## Merge
+
+An important parameter for `language-generation` and `language-modeling` tasks is `merge` or the packing of data into chunks.
+
+It supports the following options.
+
+- `concat` (default) - text is split into chunks up to maxlength, data can be split across multiple chunks
+- `pack` - text is split into chunks up to maxlength, data guaranteed to be in same chunk, chunks can be smaller than maxlength
+- `None` - disables merging
+
+Merging helps reduce training time as data can be processed efficiently without padding. `concat` maximizes this as it guarantees each chunk will be up to maxlength size. `pack` is a middle ground where data is combined but records are preserved.
+
+For general language modeling tasks like masked language modeling, `concat` is the best choice. For instruction/prompt fine-tuning, `pack` or None are the better choices as it guarantees complex logic is not split across chunks.
+
 ## Methods 
 
 Python documentation for the pipeline.

docs/poweredby.md

@@ -7,7 +7,7 @@ The following applications are powered by txtai.
 | Application  | Description  |
 |:------------ |:-------------|
 | [rag](https://github.com/neuml/rag) | Retrieval Augmented Generation (RAG) application |
-| [ragdata](https://github.com/neuml/ragdata) | Build knowledge bases for RAG |
+| [ncoder](https://github.com/neuml/ncoder) | Open-Source AI coding agent |
 | [paperai](https://github.com/neuml/paperai) | AI for medical and scientific papers |
 | [annotateai](https://github.com/neuml/annotateai) | Automatically annotate papers with LLMs |
 

docs/usecases.md

@@ -42,14 +42,15 @@ Agents connect embeddings, pipelines, workflows and other agents together to aut
 
 ![agent](images/agent.png)
 
-txtai agents are built on top of the [smolagents](https://github.com/huggingface/smolagents) framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM).
+txtai agents are built on top of the [smolagents](https://github.com/huggingface/smolagents) framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM). Agent prompting with [`agents.md`](https://github.com/agentsmd/agents.md) and [`skill.md`](https://agentskills.io/specification) are also supported.
 
 Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/agent_quickstart.py). Additional examples are listed below.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
 | [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 | [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
 
 ### Retrieval augmented generation
@@ -63,7 +64,7 @@ Check out this [RAG Quickstart Example](https://github.com/neuml/txtai/blob/mast
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
 | [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
 | [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
 | [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |

docs/workflow/task/console.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: console
+  name:
+    tasks:
+      - task: console
 ```
 
 ## Methods

docs/workflow/task/export.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: export
+  name:
+    tasks:
+      - task: export
 ```
 
 ## Methods

docs/workflow/task/file.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: file
+  name:
+    tasks:
+      - task: file
 ```
 
 ## Methods

docs/workflow/task/image.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: image
+  name:
+    tasks:
+      - task: image
 ```
 
 ## Methods

docs/workflow/task/index.md

@@ -35,12 +35,26 @@ Tasks can also be created with configuration as part of a workflow.
 
 ```yaml
 workflow:
-  tasks:
-    - action: summary 
+  name:
+    tasks:
+      - action: summary
 ```
 
 ::: txtai.workflow.Task.__init__
 
+## Multi-action task inputs
+
+A multi-action `Task` requires reusable inputs. Passing a single-pass iterator, such as a generator or the output of a `StreamTask`, raises a `ValueError` before consuming inputs or running any actions. Automatically buffering an iterator could exhaust memory, so tasks do not convert these inputs to a list.
+
+Iterators passed to a `Workflow` remain supported: the workflow collects bounded batches before passing them to its tasks. For example:
+
+```python
+workflow = Workflow([Task([list, list])], batch=2)
+list(workflow(iter([1, 2, 3])))  # [(1, 1), (2, 2), (3, 3)]
+```
+
+This batching occurs at the workflow input; it does not batch iterator outputs produced by tasks inside that workflow. Single-action tasks continue to accept iterator inputs.
+
 ## Multi-action task concurrency
 
 The default processing mode is to run actions sequentially. Multiprocessing support is already built in at a number of levels. Any of the GPU models will maximize GPU utilization for example and even in CPU mode, concurrency is utilized. But there are still use cases for task action concurrency. For example, if the system has multiple GPUs, the task runs external sequential code, or the task has a large number of I/O tasks.
@@ -72,7 +86,7 @@ workflow = Workflow([Task(lambda x: [y * 3 for y in x], unpack=False, column=0)]
 list(workflow([(2, 8)]))
 ```
 
-For the example input tuple of (2, 2), the workflow will only select the first element (2) and run the task against that element. 
+For the example input tuple of (2, 8), the workflow will only select the first element (2) and run the task against that element.
 
 ```python
 workflow = Workflow([Task([lambda x: [y * 3 for y in x], 

docs/workflow/task/retrieve.md

@@ -22,9 +22,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: retrieve
-      directory: /tmp
+  name:
+    tasks:
+      - task: retrieve
+        directory: /tmp
 ```
 
 ## Methods

docs/workflow/task/service.md

@@ -22,9 +22,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: service
-      url: https://service.url/action
+  name:
+    tasks:
+      - task: service
+        url: https://service.url/action
 ```
 
 ## Methods

docs/workflow/task/storage.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: storage
+  name:
+    tasks:
+      - task: storage
 ```
 
 ## Methods

docs/workflow/task/template.md

@@ -23,9 +23,10 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: template
-      template: This is a {text} task
+  name:
+    tasks:
+      - task: template
+        template: This is a {text} task
 ```
 
 ## Methods

docs/workflow/task/url.md

@@ -22,8 +22,9 @@ This task can also be created with workflow configuration.
 
 ```yaml
 workflow:
-  tasks:
-    - task: url
+  name:
+    tasks:
+      - task: url
 ```
 
 ## Methods

examples/01_Introducing_txtai.ipynb

@@ -27,7 +27,7 @@
         "- 🔋 Batteries included with defaults to get up and running fast\n",
         "- ☁️ Run local or scale out with container orchestration\n",
         "\n",
-        "txtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.\n",
+        "txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.\n",
         "\n",
         "> [NeuML](https://neuml.com) is the company behind txtai and we provide AI consulting services around our stack. [Schedule a meeting](https://cal.com/neuml/intro) or [send a message](mailto:[email protected]) to learn more.\n",
         ">\n",

examples/52_Build_RAG_pipelines_with_txtai.ipynb

@@ -125,7 +125,7 @@
             "## Install\n",
             "The easiest way to install is via pip and PyPI\n",
             "pip install txtai\n",
-            "Python 3.10+ is supported. Using a Python virtual environment is **recommended** .\n",
+            "Python 3.11+ is supported. Using a Python virtual environment is **recommended** .\n",
             "See the detailed install instructions for more information covering optional dependencies, environment specific prerequisites, installing from source, conda support and how to run with containers.\n",
             "\n",
             "\n",
@@ -525,8 +525,8 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "ANSWER: Python 3.10 and later versions (Python 3.10+) are supported.\n",
-            "CITATION: [{'id': '24', 'text': 'Python 3.10+ is supported. Using a Python virtual environment is recommended.'}]\n"
+            "ANSWER: Python 3.11 and later versions (Python 3.11+) are supported.\n",
+            "CITATION: [{'id': '24', 'text': 'Python 3.11+ is supported. Using a Python virtual environment is recommended.'}]\n"
           ]
         }
       ],

examples/73_Chunking_your_data_for_RAG.ipynb

@@ -100,12 +100,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'indexid': 0, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '**GitHub - neuml/txtai: 💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows**\\n\\n*💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows - neuml/txtai*\\n\\n\\n\\n**All-in-one embeddings database** \\ntxtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.\\n\\nEmbeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases.\\n\\nThis foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.\\n\\nBuild autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.\\n\\nSummary of txtai features:\\n\\n- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing\\n- 📄 Create embeddings for text, documents, audio, images and video\\n- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more\\n- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.\\n- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems\\n- ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js) , [Java](https://github.com/neuml/txtai.java) , [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go) .\\n- 🔋 Batteries included with defaults to get up and running fast\\n- ☁️ Run local or scale out with container orchestration\\ntxtai is built with Python 3.10+, [Hugging Face Transformers](https://github.com/huggingface/transformers) , [Sentence Transformers](https://github.'}\n",
+      "{'indexid': 0, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '**GitHub - neuml/txtai: 💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows**\\n\\n*💡 All-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows - neuml/txtai*\\n\\n\\n\\n**All-in-one embeddings database** \\ntxtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.\\n\\nEmbeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases.\\n\\nThis foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.\\n\\nBuild autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.\\n\\nSummary of txtai features:\\n\\n- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing\\n- 📄 Create embeddings for text, documents, audio, images and video\\n- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more\\n- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.\\n- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems\\n- ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js) , [Java](https://github.com/neuml/txtai.java) , [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go) .\\n- 🔋 Batteries included with defaults to get up and running fast\\n- ☁️ Run local or scale out with container orchestration\\ntxtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers) , [Sentence Transformers](https://github.'}\n",
       "{'indexid': 1, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi) . txtai is open-source under an Apache 2.0 license.\\n\\n*Interested in an easy and secure way to run hosted txtai applications? Then join the* [txtai.cloud](https://txtai.cloud) *preview to learn more.* \\n\\n## Why txtai?\\nNew vector databases, LLM frameworks and everything in between are sprouting up daily. Why build with txtai?\\n\\n- Up and running in minutes with [pip](https://neuml.github.io/txtai/install/) or [Docker](https://neuml.github.io/txtai/cloud/) \\n```\\n# Get started in a couple lines\\nimport txtai\\n\\nembeddings = txtai.Embeddings()\\nembeddings.index([\"Correct\", \"Not what we hoped\"])\\nembeddings.search(\"positive\", 1)\\n#[(0, 0.29862046241760254)]\\n```\\n\\n- Built-in API makes it easy to develop applications using your programming language of choice\\n```\\n# app.yml\\nembeddings:\\n path: sentence-transformers/all-MiniLM-L6-v2\\n```\\n\\n```\\nCONFIG=app.yml uvicorn \"txtai.api:app\"\\ncurl -X GET \"http://localhost:8000/search?query=positive\"\\n```\\n\\n- Run local - no need to ship data off to disparate remote services\\n- Work with micromodels all the way up to large language models (LLMs)\\n- Low footprint - install additional dependencies and scale up when needed\\n- [Learn by example](https://neuml.github.io/txtai/examples) - notebooks cover all available functionality\\n\\n## Use Cases\\nThe following sections introduce common txtai use cases. A comprehensive set of over 60 [example notebooks and applications](https://neuml.github.io/txtai/examples) are also available.\\n\\n\\n### Semantic Search\\nBuild semantic/similarity/vector/neural search applications.'}\n",
       "{'indexid': 2, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'Traditional search systems use keywords to find data. Semantic search has an understanding of natural language and identifies results that have the same meaning, not necessarily the same keywords.\\n\\nGet started with the following examples.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |Overview of the functionality provided by txtai||\\n|[Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |Embed images and text into the same space for search||\\n|[Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |Question matching with semantic search||\\n|[Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |Explore topics, data connectivity and run network analysis||\\n\\n### LLM Orchestration\\nAutonomous agents, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).\\n\\nSee below to learn more.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |Build model prompts and connect tasks together with workflows||\\n|[Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |Integrate llama.cpp, LiteLLM and custom generation frameworks||\\n|[Build knowledge graphs with LLMs](https://github.'}\n",
       "{'indexid': 3, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |Build knowledge graphs with LLM-driven entity extraction||\\n\\n#### Agents\\nAgents connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems.\\n\\ntxtai agents are built on top of the Transformers Agent framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM).\\n\\nSee the link below to learn more.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |Explore a rich dataset with Graph Analysis and Agents||\\n|[Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |Agents that iteratively solve problems as they see fit||\\n|[Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |Exploring how to improve social media engagement with AI||\\n\\n#### Retrieval augmented generation\\nRetrieval augmented generation (RAG) reduces the risk of LLM hallucinations by constraining the output with a knowledge base as context. RAG is commonly used to \"chat with your data\".\\n\\nA novel feature of txtai is that it can provide both an answer and source citation.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Build RAG pipelines with txtai](https://github.'}\n",
       "{'indexid': 4, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |Guide on retrieval augmented generation including how to create citations||\\n|[How RAG with txtai works](https://github.com/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) |Create RAG processes, API services and Docker instances||\\n|[Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |Graph path traversal to collect complex sets of data for advanced RAG||\\n|[Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |Full cycle speech to speech workflow with RAG||\\n\\n### Language Model Workflows\\nLanguage model workflows, also known as semantic workflows, connect language models together to build intelligent applications.\\n\\nWhile LLMs are powerful, there are plenty of smaller, more specialized models that work better and faster for specific tasks. This includes models for extractive question-answering, automatic summarization, text-to-speech, transcription and translation.\\n\\n|Notebook|Description||\\n|---|---|---|\\n|[Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |Simple yet powerful constructs to efficiently process data||\\n|[Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |Run abstractive text summarization||\\n|[Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.'}\n",
-      "{'indexid': 5, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'ipynb) |Convert audio files to text||\\n|[Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |Streamline machine translation and language detection||\\n\\n## Installation\\nThe easiest way to install is via pip and PyPI\\n\\n```\\npip install txtai\\n```\\n\\nPython 3.10+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.\\n\\nSee the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies) , [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) , [installing from source](https://neuml.github.io/txtai/install/#install-from-source) , [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud) .\\n\\n\\n## Model guide\\nSee the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.\\n\\n|Component|Model(s)|\\n|---|---|\\n|[Embeddings](https://neuml.github.io/txtai/embeddings) |[all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) |\\n|[Image Captions](https://neuml.github.io/txtai/pipeline/image/caption) |[BLIP](https://hf.co/Salesforce/blip-image-captioning-base) |'}\n",
+      "{'indexid': 5, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'ipynb) |Convert audio files to text||\\n|[Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |Streamline machine translation and language detection||\\n\\n## Installation\\nThe easiest way to install is via pip and PyPI\\n\\n```\\npip install txtai\\n```\\n\\nPython 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.\\n\\nSee the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies) , [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) , [installing from source](https://neuml.github.io/txtai/install/#install-from-source) , [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud) .\\n\\n\\n## Model guide\\nSee the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.\\n\\n|Component|Model(s)|\\n|---|---|\\n|[Embeddings](https://neuml.github.io/txtai/embeddings) |[all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) |\\n|[Image Captions](https://neuml.github.io/txtai/pipeline/image/caption) |[BLIP](https://hf.co/Salesforce/blip-image-captioning-base) |'}\n",
       "{'indexid': 6, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': '|[Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels) |[BART-Large-MNLI](https://hf.co/facebook/bart-large) |\\n|[Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels) |Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer) |\\n|[Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) |[Llama 3.1 Instruct](https://hf.co/meta-llama/Llama-3.1-8B-Instruct) |\\n|[Summarization](https://neuml.github.io/txtai/pipeline/text/summary) |[DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6) |\\n|[Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech) |[ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx) |\\n|[Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription) |[Whisper](https://hf.co/openai/whisper-base) |\\n|[Translation](https://neuml.github.io/txtai/pipeline/text/translation) |[OPUS Model Series](https://hf.co/Helsinki-NLP) |\\nModels can be loaded as either a path from the Hugging Face Hub or a local directory. Model paths are optional, defaults are loaded when not specified. For tasks with no recommended model, txtai uses the default models as shown in the Hugging Face Tasks guide.\\n\\nSee the following links to learn more.\\n\\n\\n## Powered by txtai\\nThe following applications are powered by txtai.'}\n",
       "{'indexid': 7, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': \"|Application|Description|\\n|---|---|\\n|[rag](https://github.com/neuml/rag) |Retrieval Augmented Generation (RAG) application|\\n|[ragdata](https://github.com/neuml/ragdata) |Build knowledge bases for RAG|\\n|[paperai](https://github.com/neuml/paperai) |Semantic search and workflows for medical/scientific papers|\\n|[annotateai](https://github.com/neuml/annotateai) |Automatically annotate papers with LLMs|\\nIn addition to this list, there are also many other [open-source projects](https://github.com/neuml/txtai/network/dependents) , [published research](https://scholar.google.com/scholar?q=txtai&hl=en&as_ylo=2022) and closed proprietary/commercial projects that have built on txtai in production.\\n\\n\\n## Further Reading\\n- [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg) \\n- [What's new in txtai 8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.\"}\n",
       "{'indexid': 8, 'id': '0', 'url': 'https://github.com/neuml/txtai', 'text': 'com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c) \\n- [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748) \\n\\n## Documentation\\n[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for embeddings, pipelines, workflows, API and a FAQ with common questions/issues is available.\\n\\n\\n## Contributing\\nFor those who would like to contribute to txtai, please see [this guide](https://github.com/neuml/.github/blob/master/CONTRIBUTING.md) .'}\n",

examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb

@@ -0,0 +1,360 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c65bdb85",
+   "metadata": {},
+   "source": [
+    "# Distilling Knowledge into Tiny LLMs\n",
+    "\n",
+    "Large Language Models (LLMs) are the magic behind AI. These massive billion and trillion parameter models have been shown to generalize well when trained on enough data.\n",
+    "\n",
+    "A big problem is that they are hard to run and expensive. So many just call LLMs through APIs such as OpenAI or Claude. Additionally, in many instances, developers spend a lot of time with complex prompt logic hoping to cover all the edge cases and believe they need a model that's large enough to handle all the rules.\n",
+    "\n",
+    "If you truly want control over your business processes, running a local model is a better choice. And the good news is that it doesn't have to be a giant and expensive multi-billion parameter model. We can finetune LLMs to handle our specific business logic, which helps us take control and limit prompt complexity. \n",
+    "\n",
+    "This notebook will show how we can distill knowledge into tiny LLMs."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "279ecd4b",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4f7bca3a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7b1b4be",
+   "metadata": {},
+   "source": [
+    "# The LLM\n",
+    "\n",
+    "We'll use a [600M parameter Qwen3 model](https://hf.co/qwen/qwen3-0.6b) for this example. Our target task will be translating user requests into linux commands."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "e1d95dd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from txtai import LLM\n",
+    "\n",
+    "llm = LLM(\"Qwen/Qwen3-0.6B\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8f276cfc",
+   "metadata": {},
+   "source": [
+    "Let's try one with the base model as it is."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "686c3983",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'ps -e'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "llm(\"\"\"\n",
+    "Translate the following request into a linux command. Only print the command.\n",
+    "\n",
+    "Find number of logged in users\n",
+    "\"\"\", maxlength=1024)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dcd92414",
+   "metadata": {},
+   "source": [
+    "As we can see, the model actually has a good understanding and at least prints a command. But in this case it's not correct. Let's get to fine-tuning!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "73666982",
+   "metadata": {},
+   "source": [
+    "# Finetuning the LLM with knowledge\n",
+    "\n",
+    "Yes, 600M parameters is small and we can't possibly expect it to do well with everything. But the good news is that we can distill knowledge into this tiny LLM and make it better. We'll use this [linux commands dataset](https://huggingface.co/datasets/mecha-org/linux-command-dataset) from the Hugging Face Hub. We'll also use this [training pipeline from txtai](https://neuml.github.io/txtai/pipeline/train/trainer).\n",
+    "\n",
+    "First, we'll create the training dataset. We'll use the same prompt strategy from above.\n",
+    "\n",
+    "```python\n",
+    "\"\"\"\n",
+    "Translate the following request into a linux command. Only print the command.\n",
+    "\n",
+    "{user request}\n",
+    "\"\"\"\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "43d6b563",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "from transformers import AutoTokenizer\n",
+    "\n",
+    "# LLM path\n",
+    "path = \"Qwen/Qwen3-0.6B\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(path)\n",
+    "\n",
+    "# Load the training dataset\n",
+    "dataset = load_dataset(\"mecha-org/linux-command-dataset\", split=\"train\")\n",
+    "\n",
+    "def prompt(row):\n",
+    "    text = tokenizer.apply_chat_template([\n",
+    "        {\"role\": \"system\", \"content\": \"Translate the following request into a linux command. Only print the command.\"},\n",
+    "        {\"role\": \"user\", \"content\": row[\"input\"]},\n",
+    "        {\"role\": \"assistant\", \"content\": row[\"output\"]}\n",
+    "    ], tokenize=False, enable_thinking=False)\n",
+    "\n",
+    "    return {\"text\": text}\n",
+    "\n",
+    "# Map to training prompts\n",
+    "train = dataset.map(prompt, remove_columns=[\"input\", \"output\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7f71dce0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='210' max='210' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [210/210 01:12, Epoch 1/1]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>0.625300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>0.490200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>0.403300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>0.391800</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from txtai.pipeline import HFTrainer\n",
+    "\n",
+    "# Load the training pipeline\n",
+    "trainer = HFTrainer()\n",
+    "\n",
+    "# Train the model\n",
+    "# Set output_dir to save, trained in memory for this example\n",
+    "model = trainer(\n",
+    "    \"Qwen/Qwen3-0.6B\",\n",
+    "    train,\n",
+    "    task=\"language-generation\",\n",
+    "    maxlength=512,\n",
+    "    bf16=True,\n",
+    "    per_device_train_batch_size=4,\n",
+    "    num_train_epochs=1,\n",
+    "    logging_steps=50,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68b5bc4e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'who | wc -l'"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from txtai import LLM\n",
+    "\n",
+    "llm = LLM(model)\n",
+    "\n",
+    "llm([\n",
+    "    {\"role\": \"system\", \"content\": \"Translate the following request into a linux command. Only print the command.\"},\n",
+    "    {\"role\": \"user\", \"content\": \"Find number of logged in users\"}\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "7427186d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'ls ~/'"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "llm([\n",
+    "    {\"role\": \"system\", \"content\": \"Translate the following request into a linux command. Only print the command.\"},\n",
+    "    {\"role\": \"user\", \"content\": \"List the files in my home directory\"}\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a141d63b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'zip -r data.zip data'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "llm([\n",
+    "    {\"role\": \"system\", \"content\": \"Translate the following request into a linux command. Only print the command.\"},\n",
+    "    {\"role\": \"user\", \"content\": \"Zip the data directory with all it's contents\"}\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fc7d0fda",
+   "metadata": {},
+   "source": [
+    "It even works well without the system prompt."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "a7d0c671",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'du -sh ~'"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "llm(\"Calculate the total amount of disk space used for my home directory. Only print the total.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "48cfcacf",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This notebook demonstrated how it's very straightforward to distill knowledge into LLMs with `txtai`. Don't always go for the giant LLM, spend a little time finetuning a tiny LLM, it is well worth it!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/81_OpenCode_as_a_txtai_LLM.ipynb

@@ -0,0 +1,377 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f95d2373",
+   "metadata": {},
+   "source": [
+    "# OpenCode as a txtai LLM\n",
+    "\n",
+    "[OpenCode](https://github.com/anomalyco/opencode) is an open source AI coding agent. It's rapidly growing in popularity as an open alternative to [Claude Code](https://code.claude.com/docs/en/overview). \n",
+    "\n",
+    "OpenCode shows the power of permissive open source. It's rapidly undergoing mass adoption and as of January 2026 sits at `81K+` GitHub ⭐'s. OpenCode supports running as a [local server](https://opencode.ai/docs/server/) via `opencode serve`.\n",
+    "\n",
+    "This notebook will explore how TxtAI integrates this as a LLM pipeline."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a8719b92",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e00c85bc",
+   "metadata": {
+    "vscode": {
+     "languageId": "shellscript"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[api,pipeline-llm]\n",
+    "\n",
+    "# Install OpenCode if not already installed and run serve\n",
+    "!curl -fsSL https://opencode.ai/install | bash\n",
+    "!opencode serve &"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "73ffe60e",
+   "metadata": {},
+   "source": [
+    "# Create a OpenCode LLM\n",
+    "\n",
+    "Now that everything is installed, let's test it out!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0e35a304",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "print(\"Hello, World!\")\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from IPython.display import Markdown, display\n",
+    "\n",
+    "from txtai import LLM\n",
+    "\n",
+    "def generate(request):\n",
+    "    display(Markdown(llm(request)))\n",
+    "\n",
+    "# Connect to OpenCode server and use default LLM provider\n",
+    "llm = LLM(\"opencode\")\n",
+    "\n",
+    "generate(\"Show a Python Hello World example\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "55dd0fad",
+   "metadata": {},
+   "source": [
+    "OK, now let's try this for a more advanced use case."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "b3080841",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "from txtai import Embeddings, LLM\n",
+       "\n",
+       "# Create embeddings model\n",
+       "embeddings = Embeddings(path=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
+       "\n",
+       "# Create LLM\n",
+       "llm = LLM(\"huggingface/Qwen/Qwen2.5-1.5B-Instruct\")\n",
+       "\n",
+       "# Sample documents\n",
+       "documents = [\n",
+       "    \"Python is a programming language created by Guido van Rossum\",\n",
+       "    \"Machine learning is a subset of artificial intelligence\",\n",
+       "    \"Deep learning uses neural networks with multiple layers\"\n",
+       "]\n",
+       "\n",
+       "# Build the index\n",
+       "embeddings.index([(i, doc) for i, doc in enumerate(documents)])\n",
+       "\n",
+       "# RAG function\n",
+       "def rag_query(question):\n",
+       "    # Search for relevant documents\n",
+       "    results = embeddings.search(question, 2)\n",
+       "    \n",
+       "    # Build context from retrieved documents\n",
+       "    context = \" \".join([documents[result[0]] for result in results])\n",
+       "    \n",
+       "    # Generate response using LLM with context\n",
+       "    prompt = f\"Context: {context}\\n\\nQuestion: {question}\\n\\nAnswer:\"\n",
+       "    return llm(prompt)\n",
+       "\n",
+       "# Example usage\n",
+       "question = \"What is Python?\"\n",
+       "answer = rag_query(question)\n",
+       "print(f\"Question: {question}\")\n",
+       "print(f\"Answer: {answer}\")\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "generate(\"Show a simple and basic TxtAI RAG example\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b44e0309",
+   "metadata": {},
+   "source": [
+    "Interesting! It's not limited to Python though."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "0a2ec1b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```assembly\n",
+       "section .data\n",
+       "    hello db 'Hello, World!', 10    ; String with newline\n",
+       "    hello_len equ $ - hello         ; Length of string\n",
+       "\n",
+       "section .text\n",
+       "    global _start\n",
+       "\n",
+       "_start:\n",
+       "    ; sys_write system call\n",
+       "    mov eax, 4                      ; sys_write\n",
+       "    mov ebx, 1                      ; stdout\n",
+       "    mov ecx, hello                  ; message\n",
+       "    mov edx, hello_len              ; message length\n",
+       "    int 0x80                        ; kernel interrupt\n",
+       "\n",
+       "    ; sys_exit system call\n",
+       "    mov eax, 1                      ; sys_exit\n",
+       "    mov ebx, 0                      ; exit code 0\n",
+       "    int 0x80                        ; kernel interrupt\n",
+       "```\n",
+       "\n",
+       "Compile and run:\n",
+       "```bash\n",
+       "nasm -f elf32 hello.asm -o hello.o\n",
+       "ld -m elf_i386 hello.o -o hello\n",
+       "./hello\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "generate(\"Show a x86 assembly hello world\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4e28d5c4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "a = None\n",
+       "if a is not None:\n",
+       "    a.startswith(\"hello\")\n",
+       "else:\n",
+       "    print(\"a is None, cannot call startswith\")\n",
+       "```\n",
+       "\n",
+       "**What's wrong:** The variable `a` is `None`, which is a `NoneType` object. The `startswith()` method only exists for strings, not for `NoneType`. You need to check if `a` is not `None` before calling string methods on it."
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "generate(\"\"\"\n",
+    "Fix the following error in the code below and explain what's wrong.\n",
+    "\n",
+    "Error:\n",
+    "AttributeError: 'NoneType' object has no attribute 'startswith'\n",
+    "\n",
+    "Code:\n",
+    "a = None\n",
+    "a.startswith(\"hello\")\n",
+    "\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "156122f0",
+   "metadata": {},
+   "source": [
+    "# OpenAI-compatible Endpoint\n",
+    "\n",
+    "Now that OpenCode is integrated into the TxtAI ecosystem, it opens a lot of different opportunities. Let's say we want to host a local OpenAI compatible endpoint, no problem!\n",
+    "\n",
+    "Write the following file.\n",
+    "\n",
+    "## config.yml\n",
+    "\n",
+    "```yaml\n",
+    "# Enable OpenAI compat endpoint\n",
+    "openai: True\n",
+    "\n",
+    "llm:\n",
+    "  path: opencode\n",
+    "```\n",
+    "\n",
+    "and start the following process.\n",
+    "\n",
+    "```\n",
+    "CONFIG=config.yml uvicorn \"txtai.api:app\"\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "33057f59",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "def factorial(n):\n",
+       "    if n < 0:\n",
+       "        raise ValueError(\"Factorial is not defined for negative numbers\")\n",
+       "    if n == 0 or n == 1:\n",
+       "        return 1\n",
+       "    result = 1\n",
+       "    for i in range(2, n + 1):\n",
+       "        result *= i\n",
+       "    return result\n",
+       "\n",
+       "# Example usage:\n",
+       "# print(factorial(5))  # Output: 120\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from IPython.display import Markdown, display\n",
+    "\n",
+    "from openai import OpenAI\n",
+    "\n",
+    "client = OpenAI(\n",
+    "    base_url=\"http://localhost:8000/v1\",\n",
+    "    api_key=\"api-key\",\n",
+    ")\n",
+    "\n",
+    "response = client.chat.completions.create(\n",
+    "    messages=[{\n",
+    "        \"role\": \"user\",\n",
+    "        \"content\": \"Show a Python factorial function\",\n",
+    "    }],\n",
+    "    model=\"opencode\",\n",
+    ")\n",
+    "\n",
+    "display(Markdown((response.choices[0].message.content)))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "58a9169b",
+   "metadata": {},
+   "source": [
+    "Just like before we get an answer! This time via the OpenAI client. Fun times."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47f9dbbd",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This notebook showed how to connect TxtAI and OpenCode. This will lead to some interesting integrations. One such example is [ncoder](https://github.com/neuml/ncoder), an AI coding agent that integrates with Jupyter Notebooks. Stay tuned for more!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/82_Agentic_College_Search.ipynb

@@ -0,0 +1,358 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "5a13697b",
+   "metadata": {},
+   "source": [
+    "# Agentic College Search\n",
+    "\n",
+    "This example will demonstrate how to use `txtai` agents to identify list of strong engineering colleges with baseball programs 🎓⚙️⚾.\n",
+    "\n",
+    "We'll setup a default [agents.md](https://github.com/agentsmd/agents.md) file and then give an LLM access to the web.\n",
+    "\n",
+    "Let's get started!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "face0f9e",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f5da3842",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[agent]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9a6a500",
+   "metadata": {},
+   "source": [
+    "# Define an `agents.md` file\n",
+    "\n",
+    "Next, we'll define an agents.md file. This file defines default behavior for our college search agent and is inserted into the system prompt of every agent run."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4e0c500d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile agents.md\n",
+    "# College Researcher\n",
+    "\n",
+    "You are a college research agent. Your job is to help find the best matching schools given a set of criteria.\n",
+    "\n",
+    "## Academics\n",
+    "\n",
+    "Work to match the request with a set of schools that match the academic requirements\n",
+    "\n",
+    "## Athletics\n",
+    "\n",
+    "Some requests may be interested in collegiate athletics. Make sure to balance that with other parts of the request."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "83dee7de",
+   "metadata": {},
+   "source": [
+    "# Identify the Top 20 schools\n",
+    "\n",
+    "Next, let's run a prompt to identify the best 20 colleges given a set of criteria.\n",
+    "\n",
+    "First, we'll setup the scaffolding code to create and run an agent. We'll use a [Qwen3 4B non-thinking LLM](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) as the agent's model. We'll add the `websearch` and `webview` tools to the agent along with the `agents.md` file previously created.\n",
+    "\n",
+    "Additionally, we'll add a sliding window of the last 5 responses as \"agent memory\". This will help create a rolling dialogue. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "410f2585",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from txtai import Agent\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "def run(query, reset=False):\n",
+    "    answer = agent(query, maxlength=60000, reset=reset)\n",
+    "    display(Markdown(answer))\n",
+    "\n",
+    "agent = Agent(model=\"Qwen/Qwen3-4B-Instruct-2507\", tools=[\"websearch\", \"webview\"], memory=5, instructions=\"agents.md\", max_steps=50)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "53aa302b",
+   "metadata": {},
+   "source": [
+    "Now we can run the search. To come up with our list of colleges, a series of web searches were executed by the agent. It's like if you had the superpower 🚀 to kick off 10+ web searches with slight variations each time as you go. It's a rapid fire set of operations to collect information for the LLM for analysis.\n",
+    "\n",
+    "_Note: The output below has the `verbosity_level` set to 0 for brevity. If you run this again, you'll get the full details with each step._"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f2d1ef2e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #d4b702; text-decoration-color: #d4b702\">╭──────────────────────────────────────────────────── </span><span style=\"color: #d4b702; text-decoration-color: #d4b702; font-weight: bold\">New run</span><span style=\"color: #d4b702; text-decoration-color: #d4b702\"> ────────────────────────────────────────────────────╮</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">List 20 universities located in the Mid-Atlantic or Northeast that would be good candidates to consider with </span>   <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">top 50 engineering programs and an active baseball team. Visit websites covering the top engineering schools in</span> <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">the Mid-Atlantic and Northeast. Add a rationale for each of the picks in a few words.</span>                           <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">╰─ PipelineModel - Qwen/Qwen3-4B-Instruct-2507 ───────────────────────────────────────────────────────────────────╯</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[38;2;212;183;2m╭─\u001b[0m\u001b[38;2;212;183;2m───────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m \u001b[0m\u001b[1;38;2;212;183;2mNew run\u001b[0m\u001b[38;2;212;183;2m \u001b[0m\u001b[38;2;212;183;2m───────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m─╮\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mList 20 universities located in the Mid-Atlantic or Northeast that would be good candidates to consider with \u001b[0m   \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mtop 50 engineering programs and an active baseball team. Visit websites covering the top engineering schools in\u001b[0m \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mthe Mid-Atlantic and Northeast. Add a rationale for each of the picks in a few words.\u001b[0m                           \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m╰─\u001b[0m\u001b[38;2;212;183;2m PipelineModel - Qwen/Qwen3-4B-Instruct-2507 \u001b[0m\u001b[38;2;212;183;2m──────────────────────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m─╯\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "Here are 20 universities in the Mid-Atlantic or Northeast with top 50 engineering programs and active baseball teams, along with a brief rationale for each:\n",
+       "\n",
+       "1. **Massachusetts Institute of Technology (MIT)** – Top-ranked engineering program in the Northeast with a strong research focus and active baseball team.\n",
+       "2. **Princeton University** – Renowned for engineering excellence and competitive baseball in the Ivy League.\n",
+       "3. **Columbia University** – Strong engineering programs and a consistently active baseball team.\n",
+       "4. **University of Pennsylvania** – Top-tier engineering and a well-established baseball program in the Ivy League.\n",
+       "5. **Carnegie Mellon University** – Leading in computer and engineering fields with a competitive baseball team.\n",
+       "6. **Cornell University** – Excellent engineering programs and a strong baseball presence.\n",
+       "7. **Northeastern University** – Strong engineering programs and active baseball team.\n",
+       "8. **New York University** – Top engineering programs with a competitive baseball team.\n",
+       "9. **Rensselaer Polytechnic Institute (RPI)** – Strong engineering focus and active baseball in the Liberty League.\n",
+       "10. **Rutgers University** – Top engineering programs and a well-established baseball team.\n",
+       "11. **Lehigh University** – Strong engineering and active baseball in the Patriot League.\n",
+       "12. **University of Connecticut** – Top engineering programs and a competitive baseball team.\n",
+       "13. **University of Massachusetts Amherst** – Solid engineering programs and active baseball.\n",
+       "14. **University of Massachusetts Lowell** – Strong engineering and active baseball in the America East Conference.\n",
+       "15. **Syracuse University** – Top engineering programs and a competitive baseball team.\n",
+       "16. **Boston University** – Strong engineering and active baseball.\n",
+       "17. **Dartmouth College** – Excellent engineering programs and competitive baseball.\n",
+       "18. **Brown University** – Top engineering and active baseball.\n",
+       "19. **Bucknell University** – Solid engineering and active baseball.\n",
+       "20. **University of Rochester** – Strong engineering and competitive baseball.\n",
+       "\n",
+       "These institutions combine top-tier engineering programs with active collegiate baseball teams, making them excellent candidates for students seeking both academic excellence and athletic engagement."
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run((\n",
+    "    \"List 20 universities located in the Mid-Atlantic or Northeast that would be good candidates to consider \"\n",
+    "    \"with top 50 engineering programs and an active baseball team. \"\n",
+    "    \"Visit websites covering the top engineering schools in the Mid-Atlantic and Northeast. \"\n",
+    "    \"Add a rationale for each of the picks in a few words. \"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7533137",
+   "metadata": {},
+   "source": [
+    "Interesting! Looks like quite an impressive list of colleges. This output is the product of taking the results of 10+ web searches and analyzing the results with an LLM. A very practical and realistic use of \"Agentic AI\". This is a great list of strong engineering schools with baseball programs 🎓⚙️⚾.\n",
+    "\n",
+    "Let's narrow it down to the top 5 engineering schools."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 105,
+   "id": "5e261825",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #d4b702; text-decoration-color: #d4b702\">╭──────────────────────────────────────────────────── </span><span style=\"color: #d4b702; text-decoration-color: #d4b702; font-weight: bold\">New run</span><span style=\"color: #d4b702; text-decoration-color: #d4b702\"> ────────────────────────────────────────────────────╮</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">Read the list below and pick the Top 5 engineering programs.</span>                                                    <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">Use the following conversation history to help answer the question above.</span>                                       <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">User: List 20 universities located in the Mid-Atlantic or Northeast that would be good candidates to consider </span>  <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">with top 50 engineering programs and an active baseball team. Visit websites covering the top engineering </span>      <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">schools in the Mid-Atlantic and Northeast. Add a rationale for each of the picks in a few words. </span>               <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">Assistant: Here are 20 universities in the Mid-Atlantic or Northeast with top 50 engineering programs and </span>      <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">active baseball teams, along with a brief rationale for each:</span>                                                   <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">1. **Massachusetts Institute of Technology (MIT)** – Top-ranked engineering program in the Northeast with a </span>    <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">strong research focus and active baseball team.</span>                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">2. **Princeton University** – Renowned for engineering excellence and competitive baseball in the Ivy League.</span>   <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">3. **Columbia University** – Strong engineering programs and a consistently active baseball team.</span>               <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">4. **University of Pennsylvania** – Top-tier engineering and a well-established baseball program in the Ivy </span>    <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">League.</span>                                                                                                         <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">5. **Carnegie Mellon University** – Leading in computer and engineering fields with a competitive baseball </span>     <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">team.</span>                                                                                                           <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">6. **Cornell University** – Excellent engineering programs and a strong baseball presence.</span>                      <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">7. **Northeastern University** – Strong engineering programs and active baseball team.</span>                          <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">8. **New York University** – Top engineering programs with a competitive baseball team.</span>                         <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">9. **Rensselaer Polytechnic Institute (RPI)** – Strong engineering focus and active baseball in the Liberty </span>    <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">League.</span>                                                                                                         <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">10. **Rutgers University** – Top engineering programs and a well-established baseball team.</span>                     <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">11. **Lehigh University** – Strong engineering and active baseball in the Patriot League.</span>                       <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">12. **University of Connecticut** – Top engineering programs and a competitive baseball team.</span>                   <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">13. **University of Massachusetts Amherst** – Solid engineering programs and active baseball.</span>                   <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">14. **University of Massachusetts Lowell** – Strong engineering and active baseball in the America East </span>        <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">Conference.</span>                                                                                                     <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">15. **Syracuse University** – Top engineering programs and a competitive baseball team.</span>                         <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">16. **Boston University** – Strong engineering and active baseball.</span>                                             <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">17. **Dartmouth College** – Excellent engineering programs and competitive baseball.</span>                            <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">18. **Brown University** – Top engineering and active baseball.</span>                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">19. **Bucknell University** – Solid engineering and active baseball.</span>                                            <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">20. **University of Rochester** – Strong engineering and competitive baseball.</span>                                  <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">These institutions combine top-tier engineering programs with active collegiate baseball teams, making them </span>    <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">excellent candidates for students seeking both academic excellence and athletic engagement.</span>                     <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span> <span style=\"font-weight: bold\">If the history is irrelevant, forget it and use other tools to answer the question.</span>                             <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>                                                                                                                 <span style=\"color: #d4b702; text-decoration-color: #d4b702\">│</span>\n",
+       "<span style=\"color: #d4b702; text-decoration-color: #d4b702\">╰─ PipelineModel - Qwen/Qwen3-4B-Instruct-2507 ───────────────────────────────────────────────────────────────────╯</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[38;2;212;183;2m╭─\u001b[0m\u001b[38;2;212;183;2m───────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m \u001b[0m\u001b[1;38;2;212;183;2mNew run\u001b[0m\u001b[38;2;212;183;2m \u001b[0m\u001b[38;2;212;183;2m───────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m─╮\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mRead the list below and pick the Top 5 engineering programs.\u001b[0m                                                    \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mUse the following conversation history to help answer the question above.\u001b[0m                                       \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mUser: List 20 universities located in the Mid-Atlantic or Northeast that would be good candidates to consider \u001b[0m  \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mwith top 50 engineering programs and an active baseball team. Visit websites covering the top engineering \u001b[0m      \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mschools in the Mid-Atlantic and Northeast. Add a rationale for each of the picks in a few words. \u001b[0m               \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mAssistant: Here are 20 universities in the Mid-Atlantic or Northeast with top 50 engineering programs and \u001b[0m      \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mactive baseball teams, along with a brief rationale for each:\u001b[0m                                                   \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m1. **Massachusetts Institute of Technology (MIT)** – Top-ranked engineering program in the Northeast with a \u001b[0m    \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mstrong research focus and active baseball team.\u001b[0m                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m2. **Princeton University** – Renowned for engineering excellence and competitive baseball in the Ivy League.\u001b[0m   \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m3. **Columbia University** – Strong engineering programs and a consistently active baseball team.\u001b[0m               \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m4. **University of Pennsylvania** – Top-tier engineering and a well-established baseball program in the Ivy \u001b[0m    \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mLeague.\u001b[0m                                                                                                         \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m5. **Carnegie Mellon University** – Leading in computer and engineering fields with a competitive baseball \u001b[0m     \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mteam.\u001b[0m                                                                                                           \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m6. **Cornell University** – Excellent engineering programs and a strong baseball presence.\u001b[0m                      \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m7. **Northeastern University** – Strong engineering programs and active baseball team.\u001b[0m                          \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m8. **New York University** – Top engineering programs with a competitive baseball team.\u001b[0m                         \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m9. **Rensselaer Polytechnic Institute (RPI)** – Strong engineering focus and active baseball in the Liberty \u001b[0m    \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mLeague.\u001b[0m                                                                                                         \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m10. **Rutgers University** – Top engineering programs and a well-established baseball team.\u001b[0m                     \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m11. **Lehigh University** – Strong engineering and active baseball in the Patriot League.\u001b[0m                       \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m12. **University of Connecticut** – Top engineering programs and a competitive baseball team.\u001b[0m                   \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m13. **University of Massachusetts Amherst** – Solid engineering programs and active baseball.\u001b[0m                   \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m14. **University of Massachusetts Lowell** – Strong engineering and active baseball in the America East \u001b[0m        \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mConference.\u001b[0m                                                                                                     \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m15. **Syracuse University** – Top engineering programs and a competitive baseball team.\u001b[0m                         \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m16. **Boston University** – Strong engineering and active baseball.\u001b[0m                                             \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m17. **Dartmouth College** – Excellent engineering programs and competitive baseball.\u001b[0m                            \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m18. **Brown University** – Top engineering and active baseball.\u001b[0m                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m19. **Bucknell University** – Solid engineering and active baseball.\u001b[0m                                            \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1m20. **University of Rochester** – Strong engineering and competitive baseball.\u001b[0m                                  \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mThese institutions combine top-tier engineering programs with active collegiate baseball teams, making them \u001b[0m    \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mexcellent candidates for students seeking both academic excellence and athletic engagement.\u001b[0m                     \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m \u001b[1mIf the history is irrelevant, forget it and use other tools to answer the question.\u001b[0m                             \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m│\u001b[0m                                                                                                                 \u001b[38;2;212;183;2m│\u001b[0m\n",
+       "\u001b[38;2;212;183;2m╰─\u001b[0m\u001b[38;2;212;183;2m PipelineModel - Qwen/Qwen3-4B-Instruct-2507 \u001b[0m\u001b[38;2;212;183;2m──────────────────────────────────────────────────────────────────\u001b[0m\u001b[38;2;212;183;2m─╯\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "The top 5 engineering programs from the list of universities in the Mid-Atlantic or Northeast with active baseball teams are:\n",
+       "\n",
+       "1. **Massachusetts Institute of Technology (MIT)** – Top-ranked engineering program in the Northeast with a strong research focus and active baseball team.\n",
+       "2. **Princeton University** – Renowned for engineering excellence and competitive baseball in the Ivy League.\n",
+       "3. **Carnegie Mellon University** – Leading in computer and engineering fields with a competitive baseball team.\n",
+       "4. **Cornell University** – Excellent engineering programs and a strong baseball presence.\n",
+       "5. **University of Pennsylvania** – Top-tier engineering and a well-established baseball program in the Ivy League.\n",
+       "\n",
+       "These schools stand out due to their exceptional engineering programs and active collegiate baseball teams, offering a balanced academic and athletic experience."
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run(\"Read the list below and pick the Top 5 engineering programs.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e6c27f16",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This example showed how to run an Agentic Loop that identified the best set of colleges to research further. It's an agent with memory and can be used to build an ongoing dialogue with an agent. Give it a try!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/83_TxtAI_got_skills.ipynb

@@ -0,0 +1,410 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8d8cb3ad",
+   "metadata": {},
+   "source": [
+    "# TxtAI got skills\n",
+    "\n",
+    "This example will demonstrate how to use `txtai` agents with [`skill.md`](https://agentskills.io/specification) files.\n",
+    "\n",
+    "We'll setup a `skill.md` file with details on how to use TxtAI and run a series of agent requests.\n",
+    "\n",
+    "Let's get started!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "00a5a0ea",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ff7aa19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[agent]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "98bcd5c3",
+   "metadata": {},
+   "source": [
+    "# Define a `skill.md` file\n",
+    "\n",
+    "Next, we'll create our `skill.md` file. This file has examples on how to build embeddings databases, how to use re-ranker pipelines, RAG pipelines and more.\n",
+    "\n",
+    "The upside of a `skill.md` file vs an `agents.md` file is that it can be dynamically added to the agent context. The description helps the agent decide if the `skill` is necessary given the request. Think of it like a dynamic knowledge base that's easy to modify."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49179643",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%writefile skill.md\n",
+    "---\n",
+    "name: txtai\n",
+    "description: Examples on how to build txtai embeddings databases, txtai RAG pipelines, txtai reranker pipelines and txtai translation pipelines\n",
+    "---\n",
+    "\n",
+    "# Build an embeddings database\n",
+    "\n",
+    "```python\n",
+    "from txtai import Embeddings\n",
+    "\n",
+    "# Create embeddings model, backed by sentence-transformers & transformers\n",
+    "embeddings = Embeddings(path=\"sentence-transformers/nli-mpnet-base-v2\")\n",
+    "\n",
+    "data = [\n",
+    "  \"US tops 5 million confirmed virus cases\",\n",
+    "  \"Canada's last fully intact ice shelf has suddenly collapsed, \" +\n",
+    "  \"forming a Manhattan-sized iceberg\",\n",
+    "  \"Beijing mobilises invasion craft along coast as Taiwan tensions escalate\",\n",
+    "  \"The National Park Service warns against sacrificing slower friends \" +\n",
+    "  \"in a bear attack\",\n",
+    "  \"Maine man wins $1M from $25 lottery ticket\",\n",
+    "  \"Make huge profits without work, earn up to $100,000 a day\"\n",
+    "]\n",
+    "\n",
+    "# Index the list of text\n",
+    "embeddings.index(data)\n",
+    "```\n",
+    "\n",
+    "# Search an embeddings database\n",
+    "\n",
+    "```python\n",
+    "embeddings.search(\"Search query\")\n",
+    "```\n",
+    "\n",
+    "# Build a RAG pipeline\n",
+    "\n",
+    "```python\n",
+    "from txtai import Embeddings, RAG\n",
+    "\n",
+    "# Input data\n",
+    "data = [\n",
+    "  \"US tops 5 million confirmed virus cases\",\n",
+    "  \"Canada's last fully intact ice shelf has suddenly collapsed, \" +\n",
+    "  \"forming a Manhattan-sized iceberg\",\n",
+    "  \"Beijing mobilises invasion craft along coast as Taiwan tensions escalate\",\n",
+    "  \"The National Park Service warns against sacrificing slower friends \" +\n",
+    "  \"in a bear attack\",\n",
+    "  \"Maine man wins $1M from $25 lottery ticket\",\n",
+    "  \"Make huge profits without work, earn up to $100,000 a day\"\n",
+    "]\n",
+    "\n",
+    "# Build embeddings index\n",
+    "embeddings = Embeddings(content=True)\n",
+    "embeddings.index(data)\n",
+    "\n",
+    "# Create the RAG pipeline\n",
+    "rag = RAG(embeddings, \"Qwen/Qwen3-0.6B\", template=\"\"\"\n",
+    "  Answer the following question using the provided context.\n",
+    "\n",
+    "  Question:\n",
+    "  {question}\n",
+    "\n",
+    "  Context:\n",
+    "  {context}\n",
+    "\"\"\")\n",
+    "\n",
+    "# Run RAG pipeline\n",
+    "rag(\"What was won?\")\n",
+    "```\n",
+    "\n",
+    "# Translate text from English into French\n",
+    "\n",
+    "```python\n",
+    "from txtai.pipeline import Translation\n",
+    "\n",
+    "# Create and run pipeline\n",
+    "translate = Translation()\n",
+    "translate(\"This is a test translation\", \"fr\")\n",
+    "```\n",
+    "\n",
+    "# Re-ranker pipeline\n",
+    "\n",
+    "```python\n",
+    "from txtai import Embeddings\n",
+    "from txtai.pipeline import Reranker, Similarity\n",
+    "\n",
+    "# Embeddings instance\n",
+    "embeddings = Embeddings()\n",
+    "embeddings.load(provider=\"huggingface-hub\", container=\"neuml/txtai-wikipedia\")\n",
+    "\n",
+    "# Similarity instance\n",
+    "similarity = Similarity(path=\"colbert-ir/colbertv2.0\", lateencode=True)\n",
+    "\n",
+    "# Reranking pipeline\n",
+    "reranker = Reranker(embeddings, similarity)\n",
+    "reranker(\"Tell me about AI\")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ed26969",
+   "metadata": {},
+   "source": [
+    "# Query for TxtAI questions\n",
+    "\n",
+    "Now, let's try this out and see if the LLM is smart enough to use the defined skill vs. going out on the web.\n",
+    "\n",
+    "Let's setup the scaffolding code to create and run an agent. We'll use a [Qwen3 4B non-thinking LLM](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) as the agent's model. We'll add the `websearch` and `webview` tools to the agent along with the `skill.md` file previously created.\n",
+    "\n",
+    "Additionally, we'll add a sliding window of the last 2 responses as \"agent memory\". This will help create a rolling dialogue. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b6a1aaf1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from txtai import Agent\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "def run(query, reset=False):\n",
+    "    answer = agent(query, maxlength=50000, reset=reset)\n",
+    "    display(Markdown(answer))\n",
+    "\n",
+    "agent = Agent(\n",
+    "    model=\"Qwen/Qwen3-4B-Instruct-2507\",\n",
+    "    tools=[\"websearch\", \"webview\", \"skill.md\"],\n",
+    "    memory=2,\n",
+    "    verbosity_level=0\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "19302f54",
+   "metadata": {},
+   "source": [
+    "First, we'll ask how to build a TxtAI embeddings database."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "121d0b8a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "To create a txtai embeddings program that indexes data, use the following Python code:\n",
+       "\n",
+       "```python\n",
+       "from txtai import Embeddings\n",
+       "\n",
+       "# Create embeddings model using sentence-transformers\n",
+       "embeddings = Embeddings(path=\"sentence-transformers/nli-mpnet-base-v2\")\n",
+       "\n",
+       "# Sample data to index\n",
+       "data = [\n",
+       "  \"US tops 5 million confirmed virus cases\",\n",
+       "  \"Canada's last fully intact ice shelf has suddenly collapsed, \" +\n",
+       "  \"forming a Manhattan-sized iceberg\",\n",
+       "  \"Beijing mobilises invasion craft along coast as Taiwan tensions escalate\",\n",
+       "  \"The National Park Service warns against sacrificing slower friends \" +\n",
+       "  \"in a bear attack\",\n",
+       "  \"Maine man wins $1M from $25 lottery ticket\",\n",
+       "  \"Make huge profits without work, earn up to $100,000 a day\"\n",
+       "]\n",
+       "\n",
+       "# Index the data\n",
+       "embeddings.index(data)\n",
+       "```\n",
+       "\n",
+       "This program initializes an embeddings database using the `sentence-transformers/nli-mpnet-base-v2` model and indexes a list of text data. The indexed data can later be searched or used in other applications like retrieval, RAG, or translation."
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run(\"Write a txtai embeddings program that indexes data\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f723e07f",
+   "metadata": {},
+   "source": [
+    "The Agent pulled the correct section from the `skill.md` file. Now, let's look for an example on how to use a re-ranker pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "221a590c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "from txtai import Embeddings\n",
+       "from txtai.pipeline import Reranker, Similarity\n",
+       "\n",
+       "# Embeddings instance\n",
+       "embeddings = Embeddings()\n",
+       "embeddings.load(provider=\"huggingface-hub\", container=\"neuml/txtai-wikipedia\")\n",
+       "\n",
+       "# Similarity instance\n",
+       "similarity = Similarity(path=\"colbert-ir/colbertv2.0\", lateencode=True)\n",
+       "\n",
+       "# Reranking pipeline\n",
+       "reranker = Reranker(embeddings, similarity)\n",
+       "reranker(\"Tell me about AI\")\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run(\"Write a txtai re-ranker pipeline\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4af3a6b6",
+   "metadata": {},
+   "source": [
+    "Great! This also works. But remember we have access to an LLM here. It doesn't have to blindly just pull the text. Let's ask it to modify the last example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c9f1eed4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "from txtai import Embeddings\n",
+       "from txtai.pipeline import Reranker, Similarity\n",
+       "\n",
+       "# Embeddings instance\n",
+       "embeddings = Embeddings()\n",
+       "embeddings.load(provider=\"huggingface-hub\", container=\"neuml/txtai-wikipedia\")\n",
+       "\n",
+       "# Similarity instance with a different reranker model and lateencode disabled\n",
+       "similarity = Similarity(path=\"BAAI/bge-reranker-base\", lateencode=False)\n",
+       "\n",
+       "# Reranking pipeline\n",
+       "reranker = Reranker(embeddings, similarity)\n",
+       "reranker(\"Tell me about AI\")\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run(\"Update the similarity path to use another reranker model. Disable lateencode.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4a3bb21e",
+   "metadata": {},
+   "source": [
+    "Notice it just edited the code with a different reranker model and even added a comment to note this change.\n",
+    "\n",
+    "We can also clear the rolling dialogue and start fresh. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ad53f7a8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "```python\n",
+       "from txtai.pipeline import Translation\n",
+       "\n",
+       "# Create and run pipeline for English to Spanish translation\n",
+       "translate = Translation()\n",
+       "translated_text = translate(\"This is a test translation\", \"es\")\n",
+       "print(translated_text)\n",
+       "```"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "run(\"Write a txtai program that translate text from English to Spanish\", reset=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "007daee0",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This example shows how to add a `skill.md` file to `txtai` agents. Go give it a try!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/85_Progressive_Distillation.ipynb

@@ -0,0 +1,337 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "40f124c0",
+   "metadata": {},
+   "source": [
+    "# Progressive Distillation\n",
+    "\n",
+    "Now that almost everyone has thought about or is actively integrating AI workflows into their projects, some might ask is this all worth the cost? Many think the current economics of the AI space don't scale and that there will be upward price movement. Others still might not be comfortable with sending their data to remote services for processing. Then there is the crowd that wants to deploy models in small spaces with limited compute.\n",
+    "\n",
+    "Are there ways we can deploy small models locally and run at a lower cost? Yes with [Knowledge Distillation](https://en.wikipedia.org/wiki/Knowledge_distillation). Knowledge distillation can get a bad rap due to it's questionable use in training some Large Language Models (LLMs). But it's a perfectly valid way to transfer performance from a larger model to a smaller one. Especially when both models are yours and/or open. \n",
+    "\n",
+    "This notebook will explore progressive distillation which is a technique to incrementally transfer knowledge from a series of larger teacher models into a smaller student."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4aad867",
+   "metadata": {},
+   "source": [
+    "# Install dependencies\n",
+    "\n",
+    "Install `txtai` and all dependencies."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "16e2c760",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3a23082",
+   "metadata": {},
+   "source": [
+    "# Setup the Training Pipeline\n",
+    "\n",
+    "The first step we need to do is setup up the training pipeline. We'll use the Hugging Face Training framework to build a series of models.\n",
+    "\n",
+    "The following code establishes a `train` method, `test` method and loads the classification training data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea7ee576",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+    "from txtai.pipeline import HFTrainer, Labels\n",
+    "\n",
+    "def train(teacher, student, distillation, **kwargs):\n",
+    "    trainer = HFTrainer()\n",
+    "\n",
+    "    model = AutoModelForSequenceClassification.from_pretrained(student, trust_remote_code=True)\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(student, trust_remote_code=True)\n",
+    "\n",
+    "    return trainer(\n",
+    "        (model, tokenizer),\n",
+    "        ds[\"train\"],\n",
+    "        columns=(\"sentence\", \"label\"),\n",
+    "        maxlength=maxlength,\n",
+    "        teacher=teacher,\n",
+    "        distillation=distillation,\n",
+    "        **kwargs\n",
+    "    )\n",
+    "\n",
+    "def test(model):\n",
+    "    labels = Labels(model, dynamic=False, trust_remote_code=True)\n",
+    "\n",
+    "    # Determine accuracy on test set\n",
+    "    results = [row[\"label\"] == labels(row[\"sentence\"], max_length=maxlength)[0][0] for row in ds[\"validation\"]]\n",
+    "    print(sum(results) / len(ds[\"validation\"]))\n",
+    "\n",
+    "# Hugging Face dataset\n",
+    "ds = load_dataset(\"nyu-mll/glue\", \"sst2\")\n",
+    "maxlength = 128"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "732fad7d",
+   "metadata": {},
+   "source": [
+    "# Progressive Distillation\n",
+    "\n",
+    "We're going to build a [bert-hash-femto](https://huggingface.co/NeuML/bert-hash-femto) classifier which is a extremely small 250K parameter model. This model was pretrained using the same recipe as [BERT](https://arxiv.org/abs/1810.04805). The paper [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962) established that small models perform better with Knowledge Distillation tasks when they are pretrained. [This article](https://huggingface.co/blog/NeuML/bert-hash-embeddings) is also a good source for more information on the topic.\n",
+    "\n",
+    "The next series of steps will do the following:\n",
+    "\n",
+    "- Train a [BERT Mini](https://huggingface.co/google/bert_uncased_L-4_H-256_A-4) model using a [BERT Large](https://huggingface.co/assemblyai/bert-large-uncased-sst2) teacher\n",
+    "- Train a [BERT Tiny](https://huggingface.co/google/bert_uncased_L-2_H-128_A-2) model using the BERT Mini teacher above\n",
+    "- Train a [BERT Hash Femto](https://huggingface.co/NeuML/bert-hash-femto) model using the BERT Tiny teacher above\n",
+    "\n",
+    "This path goes from a 11M parameter model -> 4.4M parameter model -> 250K parameter model\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "43baccd6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertForSequenceClassification LOAD REPORT\u001b[0m from: google/bert_uncased_L-4_H-256_A-4\n",
+      "Key                                        | Status     | \n",
+      "-------------------------------------------+------------+-\n",
+      "cls.predictions.transform.dense.weight     | UNEXPECTED | \n",
+      "cls.predictions.decoder.bias               | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.bias   | UNEXPECTED | \n",
+      "cls.predictions.decoder.weight             | UNEXPECTED | \n",
+      "cls.seq_relationship.bias                  | UNEXPECTED | \n",
+      "cls.seq_relationship.weight                | UNEXPECTED | \n",
+      "cls.predictions.transform.dense.bias       | UNEXPECTED | \n",
+      "cls.predictions.bias                       | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.weight | UNEXPECTED | \n",
+      "classifier.weight                          | MISSING    | \n",
+      "classifier.bias                            | MISSING    | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.875\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"assemblyai/bert-large-uncased-sst2\",\n",
+    "    \"google/bert_uncased_L-4_H-256_A-4\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=1e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-mini-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9ba3158e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertForSequenceClassification LOAD REPORT\u001b[0m from: google/bert_uncased_L-2_H-128_A-2\n",
+      "Key                                        | Status     | \n",
+      "-------------------------------------------+------------+-\n",
+      "cls.predictions.transform.dense.weight     | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.bias   | UNEXPECTED | \n",
+      "cls.seq_relationship.weight                | UNEXPECTED | \n",
+      "cls.seq_relationship.bias                  | UNEXPECTED | \n",
+      "cls.predictions.transform.dense.bias       | UNEXPECTED | \n",
+      "cls.predictions.bias                       | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.weight | UNEXPECTED | \n",
+      "classifier.weight                          | MISSING    | \n",
+      "classifier.bias                            | MISSING    | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.8325688073394495\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"bert-mini-sst2\",\n",
+    "    \"google/bert_uncased_L-2_H-128_A-2\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=1e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-tiny-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "e98d950e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertHashForSequenceClassification LOAD REPORT\u001b[0m from: neuml/bert-hash-femto\n",
+      "Key                      | Status  | \n",
+      "-------------------------+---------+-\n",
+      "bert.pooler.dense.bias   | MISSING | \n",
+      "bert.pooler.dense.weight | MISSING | \n",
+      "classifier.bias          | MISSING | \n",
+      "classifier.weight        | MISSING | \n",
+      "\n",
+      "Notes:\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.8084862385321101\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    \"bert-tiny-sst2\",\n",
+    "    \"neuml/bert-hash-femto\",\n",
+    "    (1.0, 1.0),\n",
+    "    learning_rate=3e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-femto-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f1c94e7",
+   "metadata": {},
+   "source": [
+    "Let's look at the performance. The 11M parameter model registered an accuracy score of `0.8750` on the SST2 dev set. Then the 4M parameter model scored `0.8326` and finally the tiny femto model scored `0.8085`. As we can see each score got progressively worse but each model has less capability. Let's train a femto model directly to compare."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "b89d8292",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[transformers] \u001b[1mBertHashForSequenceClassification LOAD REPORT\u001b[0m from: neuml/bert-hash-femto\n",
+      "Key                      | Status  | \n",
+      "-------------------------+---------+-\n",
+      "bert.pooler.dense.bias   | MISSING | \n",
+      "bert.pooler.dense.weight | MISSING | \n",
+      "classifier.bias          | MISSING | \n",
+      "classifier.weight        | MISSING | \n",
+      "\n",
+      "Notes:\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.801605504587156\n"
+     ]
+    }
+   ],
+   "source": [
+    "test(train(\n",
+    "    None,\n",
+    "    \"neuml/bert-hash-femto\",\n",
+    "    None,\n",
+    "    learning_rate=3e-4,\n",
+    "    num_train_epochs=5,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    output_dir=\"bert-femto-sst2\"\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4864b8d2",
+   "metadata": {},
+   "source": [
+    "This scored `0.8016` a sizable shift down from the progressively distilled model. Now keep in mind the femto model only has `250K` parameters but we gave it a sizable accuracy boost within it's capabilities."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e4e04226",
+   "metadata": {},
+   "source": [
+    "# Wrapping up\n",
+    "\n",
+    "This example showed how progressive distillation can boost overall model performance, especially for tiny models. Incrementally compressing knowledge into a series of smaller subsets enabled the final model to learn more efficiently than directly training the model on the dataset without distillation. Put progressive distillation in the toolkit when working with tiny models!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.20"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/baseball.py

@@ -15,7 +15,7 @@ import numpy as np
 import pandas as pd
 import streamlit as st
 
-from txtai.embeddings import Embeddings
+from txtai import Embeddings
 
 
 class Stats:
@@ -38,7 +38,7 @@ class Stats:
         self.names = self.loadnames()
 
         # Build index
-        self.vectors, self.data, self.embeddings = self.index()
+        self.vectors, self.data, self.maxyear, self.embeddings = self.index()
 
     def loadcolumns(self):
         """
@@ -122,16 +122,12 @@ class Stats:
         # Build data dictionary
         vectors = {f'{row["yearID"]}{row["playerID"]}': self.transform(row) for _, row in self.stats.iterrows()}
         data = {f'{row["yearID"]}{row["playerID"]}': dict(row) for _, row in self.stats.iterrows()}
+        maxyear = max(row["yearID"] for _, row in self.stats.iterrows())
 
-        embeddings = Embeddings(
-            {
-                "transform": self.transform,
-            }
-        )
-
+        embeddings = Embeddings({"transform": Stats.transform})
         embeddings.index((uid, vectors[uid], None) for uid in vectors)
 
-        return vectors, data, embeddings
+        return vectors, data, maxyear, embeddings
 
     def metrics(self, name):
         """
@@ -159,13 +155,14 @@ class Stats:
 
         return range(1871, datetime.datetime.today().year), 1950, None
 
-    def search(self, name=None, year=None, row=None, limit=10):
+    def search(self, name=None, year=None, window=None, row=None, limit=10):
         """
         Runs an embeddings search. This method takes either a player-year or stats row as input.
 
         Args:
             name: player name to search
             year: year to search
+            window: limit to window recent seasons
             row: row of stats to search
             limit: max results to return
 
@@ -183,13 +180,17 @@ class Stats:
 
         results, ids = [], set()
         if query is not None:
-            for uid, _ in self.embeddings.search(query, limit * 5):
+            candidates = limit * 100 if window else limit * 5
+            for uid, _ in self.embeddings.search(query, candidates):
                 # Only add unique players
                 if uid[4:] not in ids:
                     result = self.data[uid].copy()
-                    result["link"] = f'https://www.baseball-reference.com/players/{result["nameLast"].lower()[0]}/{result["bbrefID"]}.shtml'
-                    results.append(result)
-                    ids.add(uid[4:])
+
+                    # Add first player if this is a player comparison. Limit results to window, if necessary
+                    if (not ids and not row) or not window or result["yearID"] > self.maxyear - window:
+                        result["link"] = f'https://www.baseball-reference.com/players/{result["nameLast"].lower()[0]}/{result["bbrefID"]}.shtml'
+                        results.append(result)
+                        ids.add(uid[4:])
 
                     if len(ids) >= limit:
                         break
@@ -253,10 +254,10 @@ class Batting(Stats):
         ]
 
     def load(self):
-        # Retrieve raw data from GitHub
-        players = pd.read_csv("https://raw.githubusercontent.com/chadwickbureau/baseballdatabank/master/core/People.csv")
-        batting = pd.read_csv("https://raw.githubusercontent.com/chadwickbureau/baseballdatabank/master/core/Batting.csv")
-        fielding = pd.read_csv("https://raw.githubusercontent.com/chadwickbureau/baseballdatabank/master/core/Fielding.csv")
+        # Retrieve raw data
+        players = pd.read_csv("https://hf.co/datasets/neuml/baseballdata/resolve/main/People.csv")
+        batting = pd.read_csv("https://hf.co/datasets/neuml/baseballdata/resolve/main/Batting.csv")
+        fielding = pd.read_csv("https://hf.co/datasets/neuml/baseballdata/resolve/main/Fielding.csv")
 
         # Merge player data in
         batting = pd.merge(players, batting, how="inner", on=["playerID"])
@@ -387,9 +388,9 @@ class Pitching(Stats):
         ]
 
     def load(self):
-        # Retrieve raw data from GitHub
-        players = pd.read_csv("https://raw.githubusercontent.com/chadwickbureau/baseballdatabank/master/core/People.csv")
-        pitching = pd.read_csv("https://raw.githubusercontent.com/chadwickbureau/baseballdatabank/master/core/Pitching.csv")
+        # Retrieve raw data
+        players = pd.read_csv("https://hf.co/datasets/neuml/baseballdata/resolve/main/People.csv")
+        pitching = pd.read_csv("https://hf.co/datasets/neuml/baseballdata/resolve/main/Pitching.csv")
 
         # Merge player data in
         pitching = pd.merge(players, pitching, how="inner", on=["playerID"])
@@ -435,13 +436,14 @@ class Application:
         Runs a Streamlit application.
         """
 
-        st.title("⚾ Baseball Statistics")
+        st.set_page_config(layout="wide", page_title="Baseball Stats", page_icon="⚾")
+        st.title("⚾ Baseball Stats")
         st.markdown(
             """
-            This application finds the best matching historical players using vector search with [txtai](https://github.com/neuml/txtai).
-            Raw data is from the [Baseball Databank](https://github.com/chadwickbureau/baseballdatabank) GitHub project. Read [this
+            This application finds the best matching players using vector search with [txtai](https://github.com/neuml/txtai).
+            Raw data is from the [Lahman Baseball Database](https://sabr.org/lahman-database/). Read [this
             article](https://medium.com/neuml/explore-baseball-history-with-vector-search-5778d98d6846) for more details.
-        """
+            """
         )
 
         player, search = st.tabs(["Player", "Search"])
@@ -471,6 +473,9 @@ class Application:
         # Player name
         name = self.name(stats.names, params.get("name"))
 
+        # Limit player-year comparisons using this window
+        window = self.window(params.get("window"), "window")
+
         # Player metrics
         active, best, metrics = stats.metrics(name)
 
@@ -482,13 +487,16 @@ class Application:
             self.chart(category, metrics)
 
         # Run search
-        results = stats.search(name, year)
+        results = stats.search(name, year, window)
 
         # Display results
         self.table(results, ["link", "nameFirst", "nameLast", "teamID"] + stats.columns[1:])
 
-        # Save parameters
-        st.experimental_set_query_params(category=category, name=name, year=year)
+        # Save query parameters
+        st.query_params.clear()
+        for key, value in [("category", category), ("name", name), ("window", window), ("year", year)]:
+            if value:
+                st.query_params[key] = value
 
     def search(self):
         """
@@ -498,6 +506,10 @@ class Application:
         st.markdown("Find players with similar statistics.")
 
         stats, category = None, self.category("Batting", "searchcategory")
+
+        # Limit player-year comparisons using this window
+        window = self.window(None, "searchwindow")
+
         with st.form("search"):
             if category == "Batting":
                 stats, columns = self.batting, self.batting.columns[:-6]
@@ -510,7 +522,7 @@ class Application:
             submitted = st.form_submit_button("Search")
             if submitted:
                 # Run search
-                results = stats.search(row=inputs.to_dict(orient="records")[0])
+                results = stats.search(window=window, row=inputs.to_dict(orient="records")[0])
 
                 # Display table
                 self.table(results, ["link", "nameFirst", "nameLast", "teamID"] + stats.columns[1:])
@@ -523,18 +535,18 @@ class Application:
             parameters
         """
 
-        # Get parameters
-        params = st.experimental_get_query_params()
-        params = {x: params[x][0] for x in params}
+        # Get query parameters
+        params = {x: st.query_params.get(x) for x in ["category", "name", "window", "year"]}
 
         # Sync parameters with session state
         if all(x in st.session_state for x in ["category", "name", "year"]):
             # Copy session year if category and name are unchanged
             params["year"] = str(st.session_state["year"]) if all(params.get(x) == st.session_state[x] for x in ["category", "name"]) else None
 
-            # Copy category and name from session state
+            # Copy category, name and window from session state
             params["category"] = st.session_state["category"]
             params["name"] = st.session_state["name"]
+            params["window"] = st.session_state["window"]
 
         return params
 
@@ -559,6 +571,29 @@ class Application:
         # Radio box component
         return st.radio("Stat", categories, index=default, horizontal=True, key=key)
 
+    def window(self, window, key):
+        """
+        Limit results to last N seasons.
+
+        Args:
+            window: limit to window seasons
+            key: widget key
+
+        Returns:
+            window component
+        """
+
+        # Get window size
+        window = st.text_input("Limit to last N seasons", value=window, key=key)
+
+        # Clear invalid input
+        if window and (not window.isnumeric() or int(window) < 1):
+            st.error("Window must be a number greater or equal to 1")
+            return None
+
+        # Convert to int
+        return int(window) if window else window
+
     def name(self, names, name):
         """
         Builds name input widget.
@@ -627,7 +662,7 @@ class Application:
         chart = (chart + rule).encode(y=alt.Y(title=metric)).properties(height=200).configure_axis(grid=False)
 
         # Draw chart
-        st.altair_chart(chart + rule, theme="streamlit", use_container_width=True)
+        st.altair_chart(chart + rule, theme="streamlit", width="stretch")
 
     def table(self, results, columns):
         """
@@ -643,7 +678,7 @@ class Application:
                 results,
                 column_order=columns,
                 column_config={
-                    "link": st.column_config.LinkColumn("Link", width="small"),
+                    "link": st.column_config.LinkColumn("Link", width="small", display_text=":material/open_in_new:"),
                     "yearID": st.column_config.NumberColumn("Year", format="%d"),
                     "nameFirst": "First",
                     "nameLast": "Last",

mkdocs.yml

@@ -91,6 +91,8 @@ nav:
             - Segmentation: pipeline/data/segmentation.md
             - Tabular: pipeline/data/tabular.md
             - Textractor: pipeline/data/textractor.md
+            - Tokenizer: pipeline/data/tokenizer.md
+            - URL Retrieve: pipeline/data/urlretrieve.md
         - Image:
             - Caption: pipeline/image/caption.md
             - Image Hash: pipeline/image/imagehash.md
@@ -107,6 +109,7 @@ nav:
             - Translation: pipeline/text/translation.md
         - Train:
             - HF ONNX: pipeline/train/hfonnx.md
+            - LEMUR Trainer: pipeline/train/lemur.md
             - ML ONNX: pipeline/train/mlonnx.md
             - Trainer: pipeline/train/trainer.md
     - Workflow:

setup.py

@@ -1,4 +1,6 @@
-# pylint: disable = C0111
+# pylint: disable = C0103,C0111
+import os
+
 from setuptools import find_packages, setup
 
 with open("README.md", "r", encoding="utf-8") as f:
@@ -6,14 +8,38 @@ with open("README.md", "r", encoding="utf-8") as f:
     DESCRIPTION = "".join([line for line in f if "gh-dark-mode-only" not in line])
 
 # Required dependencies
-install = ["faiss-cpu>=1.7.1.post2", "msgpack>=1.0.7", "torch>=2.4", "transformers>=4.56.2"]
+install = []
 
-# Required dependencies that are also base transformers dependencies
-install += ["huggingface-hub>=0.34.0", "numpy>=1.18.4", "pyyaml>=5.3", "regex>=2022.8.17", "safetensors>=0.4.5"]
+# Default dependencies
+default = [
+    "faiss-cpu>=1.7.1.post2",
+    "huggingface-hub>=0.34.0",
+    "msgpack>=1.0.7",
+    "numpy>=1.18.4",
+    "regex>=2022.8.17",
+    "pyyaml>=5.3",
+    "safetensors>=0.4.5",
+    "torch>=2.4",
+    "tqdm>=4.66.3",
+    "transformers>=5.9.0",
+]
 
 # Optional dependencies
 extras = {}
 
+# Package name
+package = "txtai"
+
+if os.getenv("MINIMAL"):
+    # Add default extra
+    extras["default"] = default
+
+    # Rename this package to have minimal suffix
+    package += "_minimal"
+else:
+    # Default dependencies are required with standard install
+    install += default
+
 # Development dependencies - not included in "all" install
 extras["dev"] = [
     "black",
@@ -27,24 +53,28 @@ extras["dev"] = [
     "pylint",
 ]
 
-extras["agent"] = ["mcpadapt>=0.1.0", "smolagents>=1.23"]
+extras["agent"] = ["beautifulsoup4>=4.9.3", "jinja2>=3.1.6", "mcp<2.0", "mcpadapt>=0.1.0", "smolagents>=1.23"]
 
 extras["ann"] = [
     "annoy>=1.16.3",
     "bitsandbytes>=0.42.0",
-    "ggml-py>=0.9.4",
+    "ggml-python>=0.0.41",
     "hnswlib>=0.5.0",
+    "milvus-lite>=3.1.1",
     "pgvector>=0.4.1",
+    "rabitqlib>=0.5.1",
     "scikit-learn>=0.23.1",
     "scipy>=1.4.1",
     "sqlalchemy>=2.0.20",
     "sqlite-vec>=0.1.1",
+    "turbovec>=0.7.0",
+    "zvec>=0.5.1",
 ]
 
 extras["api"] = [
     "aiohttp>=3.8.1",
     "fastapi>=0.94.0",
-    "fastapi-mcp>=0.2.0",
+    "fastapi-mcp>=0.4.0",
     "httpx>=0.28.1",
     "pillow>=7.1.2",
     "python-multipart>=0.0.7",
@@ -55,7 +85,7 @@ extras["cloud"] = ["apache-libcloud>=3.3.1", "fasteners>=0.14.1"]
 
 extras["console"] = ["rich>=12.0.1"]
 
-extras["database"] = ["duckdb>=0.7.1", "pillow>=7.1.2", "sqlalchemy>=2.0.20"]
+extras["database"] = ["duckdb>=0.8.0", "pillow>=7.1.2", "sqlalchemy>=2.0.20"]
 
 extras["graph"] = ["grand-cypher>=0.6.0", "grand-graph>=0.6.0", "networkx>=2.7.1", "sqlalchemy>=2.0.20"]
 
@@ -71,13 +101,21 @@ extras["pipeline-audio"] = [
     "webrtcvad-wheels>=2.0.14",
 ]
 
-extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "chonkie>=1.0.2", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]
+extras["pipeline-data"] = [
+    "beautifulsoup4>=4.9.3",
+    "chonkie>=1.0.2",
+    "docling>=2.8.2",
+    "liteparse>=2.1.1",
+    "nltk>=3.5",
+    "pandas>=1.1.0",
+    "tika>=1.24",
+]
 
 extras["pipeline-image"] = ["imagehash>=4.2.1", "pillow>=7.1.2", "timm>=0.4.12"]
 
-extras["pipeline-llm"] = ["litellm>=1.37.16", "llama-cpp-python>=0.2.75"]
+extras["pipeline-llm"] = ["httpx>=0.28.1", "litert-lm-api>=0.11.0", "litellm>=1.37.16", "llama-cpp-python>=0.2.75"]
 
-extras["pipeline-text"] = ["gliner>=0.2.23", "sentencepiece>=0.1.91", "staticvectors>=0.2.0"]
+extras["pipeline-text"] = ["gliner-py>=0.2.27", "sentencepiece>=0.1.91", "staticvectors>=0.2.0"]
 
 extras["pipeline-train"] = [
     "accelerate>=0.26.0",
@@ -101,14 +139,17 @@ extras["pipeline"] = (
 extras["scoring"] = ["sqlalchemy>=2.0.20"]
 
 extras["vectors"] = [
+    "ai-edge-litert>=2.1.5",
     "litellm>=1.37.16",
     "llama-cpp-python>=0.2.75",
     "model2vec>=0.3.0",
+    "onnxruntime>=1.11.0",
     "scikit-learn>=0.23.1",
     "scipy>=1.4.1",
     "sentence-transformers>=5.0.0",
     "skops>=0.9.0",
     "staticvectors>=0.2.0",
+    "tokenizers>=0.22.2",
 ]
 
 extras["workflow"] = [
@@ -139,8 +180,8 @@ extras["all"] = (
 )
 
 setup(
-    name="txtai",
-    version="9.3.0",
+    name=package,
+    version="9.15.0",
     author="NeuML",
     description="All-in-one open-source AI framework for semantic search, LLM orchestration and language model workflows",
     long_description=DESCRIPTION,
@@ -155,7 +196,7 @@ setup(
     packages=find_packages(where="src/python"),
     package_dir={"": "src/python"},
     keywords="search embedding machine-learning nlp",
-    python_requires=">=3.10",
+    python_requires=">=3.11",
     install_requires=install,
     extras_require=extras,
     classifiers=[

src/python/txtai/agent/base.py

@@ -2,6 +2,12 @@
 Agent module
 """
 
+import os
+
+from collections import deque
+
+from jinja2.sandbox import SandboxedEnvironment
+
 from .factory import ProcessFactory
 
 
@@ -15,11 +21,13 @@ class Agent:
     should be explored.
     """
 
-    def __init__(self, **kwargs):
+    def __init__(self, template=None, memory=None, **kwargs):
         """
         Creates a new Agent.
 
         Args:
+            template: optional prompt jinja template, must include {{ text }} and {{ memory }} placeholders
+            memory: number of prior outputs to keep as "memory", defaults to None for no memory
             kwargs: arguments to pass to the underlying Agent backend and LLM pipeline instance
         """
 
@@ -27,13 +35,36 @@ class Agent:
         if "max_iterations" in kwargs:
             kwargs["max_steps"] = kwargs.pop("max_iterations")
 
+        # Custom instructions
+        if "instructions" in kwargs:
+            kwargs["instructions"] = self.instructions(kwargs)
+
         # Create agent process runner
         self.process = ProcessFactory.create(kwargs)
 
         # Tools dictionary
         self.tools = self.process.tools
 
-    def __call__(self, text, maxlength=8192, stream=False, **kwargs):
+        # Agent memory
+        self.memory = {}
+        self.window = memory
+
+        # Create template
+        self.template = SandboxedEnvironment().from_string(
+            template
+            if template
+            else """{{ text }}
+{% if memory %}
+Use the following conversation history to help answer the question above.
+
+{{ memory }}
+
+If the history is irrelevant, forget it and use other tools to answer the question.
+{% endif %}
+"""
+        )
+
+    def __call__(self, text, maxlength=8192, stream=False, session=None, reset=False, **kwargs):
         """
         Runs an agent loop.
 
@@ -41,6 +72,8 @@ class Agent:
             text: instructions to run
             maxlength: maximum sequence length
             stream: stream response if True, defaults to False
+            session: session id for stored memory, defaults to None which shares all memory
+            reset: clears previously stored memory if True, defaults to False
             kwargs: additional keyword arguments
 
         Returns:
@@ -50,5 +83,60 @@ class Agent:
         # Process parameters
         self.process.model.parameters(maxlength)
 
+        # Create memory, if necessary
+        if self.window and (session not in self.memory or reset):
+            self.memory[session] = deque(maxlen=self.window)
+
         # Run agent loop
-        return self.process.run(text, stream=stream, **kwargs)
+        output = self.process.run(self.prompt(text, session), stream=stream, **kwargs)
+
+        # Add output to memory, if necessary
+        if session in self.memory:
+            self.memory[session].append((text, output))
+
+        return output
+
+    def prompt(self, text, session):
+        """
+        Generates the full agent prompt using the input instructions. Adds in agent memory
+        if available.
+
+        Args:
+            text: instructions to run
+            session: session id for stored memory
+
+        Returns:
+            formatted instructions
+        """
+
+        # pylint: disable=E1133
+        memory = []
+        if self.memory.get(session):
+            # Add memory context
+            for request, output in self.memory[session]:
+                memory.append(f"User: {request}\nAssistant: {output}")
+
+            memory = "\n\n".join(memory)
+
+        # Command template with memory
+        return self.template.render(text=text, memory=memory)
+
+    def instructions(self, config):
+        """
+        Reads and formats custom instructions. Supports agents.md files.
+
+        Args:
+            config: agent configuration
+
+        Returns:
+            agent instructions, if any
+        """
+
+        # Check if this is a file path (i.e. agents.md)
+        path = config.pop("instructions", None)
+        if path and os.path.isfile(path):
+            with open(path, encoding="utf-8") as f:
+                # Read entire file
+                return f"\nBelow is a set of general instructions to follow:\n\n{f.read()}"
+
+        return path

src/python/txtai/agent/tool/__init__.py

@@ -5,3 +5,10 @@ Tool imports
 from .embeddings import EmbeddingsTool
 from .factory import ToolFactory
 from .function import FunctionTool
+from .bash import BashTool
+from .edit import EditTool
+from .glob import GlobTool
+from .grep import GrepTool
+from .read import ReadTool
+from .skill import SkillTool
+from .todo import TodoWriteTool

src/python/txtai/agent/tool/bash.py

@@ -0,0 +1,67 @@
+"""
+Bash imports
+"""
+
+import subprocess
+
+from smolagents import Tool
+
+
+class BashTool(Tool):
+    """
+    The BashTool runs a command through a subprocess. This tool only allows a small subset of commands.
+    More can be added through configuration.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self, allowed=None, timeout=30):
+        """
+        Creates a BashTool.
+
+        Args:
+            allowed: list of allowed commands to run, has limited set of defaults which are a best effort not a sandbox
+            timeout: maximum number of seconds to wait for a command to complete, defaults to 30, None disables the timeout
+        """
+
+        # Tool parameters
+        self.name = "bash"
+        self.description = "Implementation of a bash shell subprocess tool. Runs a shell command and returns the output."
+        self.inputs = {
+            "command": {"type": "array", "description": "Command to run. Follows Python subprocess.open pattern for command as a list of arguments."}
+        }
+        self.output_type = "any"
+
+        # Default list of allowed commands
+        self.allowed = allowed if allowed is not None else ["cat", "cut", "diff", "grep", "head", "ls", "tail"]
+
+        # Maximum time to wait for a command, None waits indefinitely
+        self.timeout = timeout
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, command):
+        """
+        Runs a shell command as a subprocess.
+
+        stdin is closed and a timeout is applied. Otherwise an allowed command called with no file arguments
+        (i.e. `cat`) reads from stdin and blocks the agent indefinitely.
+
+        Args:
+            command: command arguments as a list
+
+        Returns:
+            command output
+        """
+
+        output = None
+        if command and command[0] in self.allowed:
+            try:
+                output = subprocess.run(
+                    command, capture_output=True, text=True, encoding="utf-8", check=False, stdin=subprocess.DEVNULL, timeout=self.timeout
+                ).stdout
+            except subprocess.TimeoutExpired:
+                output = f"Command timed out after {self.timeout} seconds"
+
+        return output

src/python/txtai/agent/tool/edit.py

@@ -0,0 +1,59 @@
+"""
+Edit imports
+"""
+
+import difflib
+
+from smolagents import Tool
+
+
+class EditTool(Tool):
+    """
+    The EditTool modifies content in a file.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates an EditTool.
+        """
+
+        # Tool parameters
+        self.name = "edit"
+        self.description = "Implementation of a file edit tool. Makes in-place edits to content in a file."
+        self.inputs = {
+            "path": {"type": "string", "description": "Path of file to edit"},
+            "search": {"type": "string", "description": "String to replace"},
+            "replace": {"type": "string", "description": "Replacement string"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path, search, replace):
+        """
+        Modifies content in a file. Returns a diff of the changes, if any.
+
+        Args:
+            path: file to edit
+            search: string to replace
+            replace: replacement
+
+        Returns:
+            diff
+        """
+
+        content = None
+        with open(path, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        # Make edits
+        modified = content.replace(search, replace)
+        if modified != content:
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(modified)
+
+        # Return diff of file edits
+        return "".join(difflib.unified_diff(content.splitlines(True), modified.splitlines(True), path, path))

src/python/txtai/agent/tool/factory.py

@@ -9,12 +9,20 @@ from types import FunctionType, MethodType
 import mcpadapt.core
 
 from mcpadapt.smolagents_adapter import SmolAgentsAdapter
-from smolagents import PythonInterpreterTool, Tool, tool as CreateTool, VisitWebpageTool, WebSearchTool
+from smolagents import PythonInterpreterTool, Tool, tool as CreateTool, UserInputTool, WebSearchTool
 from transformers.utils import chat_template_utils, TypeHintParsingException
 
 from ...embeddings import Embeddings
+from .bash import BashTool
+from .edit import EditTool
 from .embeddings import EmbeddingsTool
 from .function import FunctionTool
+from .glob import GlobTool
+from .grep import GrepTool
+from .read import ReadTool
+from .skill import SkillTool
+from .todo import TodoWriteTool
+from .write import WriteTool
 
 
 class ToolFactory:
@@ -23,7 +31,21 @@ class ToolFactory:
     """
 
     # Default toolkit
-    DEFAULTS = {"python": PythonInterpreterTool(), "websearch": WebSearchTool(), "webview": VisitWebpageTool()}
+    DEFAULTS = {
+        "bash": BashTool,
+        "edit": EditTool,
+        "glob": GlobTool,
+        "grep": GrepTool,
+        "python": PythonInterpreterTool,
+        "question": UserInputTool,
+        "read": ReadTool,
+        "todowrite": TodoWriteTool,
+        "websearch": WebSearchTool,
+        "write": WriteTool,
+    }
+
+    # Backwards compatible mappings
+    DEFAULTS["webview"] = DEFAULTS["read"]
 
     @staticmethod
     def create(config):
@@ -59,13 +81,22 @@ class ToolFactory:
 
             # Get default tool, if applicable
             elif isinstance(tool, str) and tool in ToolFactory.DEFAULTS:
-                tool = ToolFactory.DEFAULTS[tool]
+                tool = ToolFactory.default(tool)
+
+            # Get ALL default tools, if applicable
+            elif isinstance(tool, str) and tool == "defaults":
+                tools.extend(ToolFactory.default(name) for name in ToolFactory.DEFAULTS if name != "webview")
+                tool = None
 
             # Support importing MCP tool collections
             elif isinstance(tool, str) and tool.startswith("http"):
                 tools.extend(mcpadapt.core.MCPAdapt({"url": tool}, SmolAgentsAdapter()).tools())
                 tool = None
 
+            # Load skill.md files
+            elif isinstance(tool, str) and tool.endswith(".md"):
+                tool = SkillTool(tool)
+
             # Add tool
             if tool:
                 tools.append(tool)
@@ -91,6 +122,20 @@ class ToolFactory:
         except (TypeHintParsingException, TypeError):
             return ToolFactory.fromdocs(target, config if config else {})
 
+    @staticmethod
+    def default(name):
+        """
+        Creates a default tool by alias name.
+
+        Args:
+            name: default tool alias name
+
+        Returns:
+            Tool
+        """
+
+        return ToolFactory.DEFAULTS[name]()
+
     @staticmethod
     def fromdocs(target, config):
         """

src/python/txtai/agent/tool/function.py

@@ -7,8 +7,7 @@ from smolagents import Tool
 
 class FunctionTool(Tool):
     """
-    Creates a FunctionTool. A FunctionTool takes descriptive configuration and injects it along with a target function
-    into an LLM prompt.
+    A FunctionTool takes descriptive configuration and injects it along with a target function into an LLM prompt.
     """
 
     # pylint: disable=W0231

src/python/txtai/agent/tool/glob.py

@@ -0,0 +1,49 @@
+"""
+Glob imports
+"""
+
+import glob as g
+import os
+
+from smolagents import Tool
+
+
+class GlobTool(Tool):
+    """
+    The GlobTool finds matching file patterns.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a GlobTool.
+        """
+
+        # Tool parameters
+        self.name = "glob"
+        self.description = "Implementation of a glob tool. Finds files that match glob patterns."
+        self.inputs = {
+            "files": {"type": "string", "description": "File or file glob pattern to search"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, files):
+        """
+        Lists files matching a glob pattern
+
+        Args:
+            files: files glob pattern
+
+        Returns:
+            list of matching files
+        """
+
+        # Format file pattern
+        files = os.path.join(files, "*") if os.path.isdir(files) else files
+
+        # Iterate over each found file
+        return g.glob(files, recursive=True)

src/python/txtai/agent/tool/grep.py

@@ -0,0 +1,75 @@
+"""
+Grep imports
+"""
+
+import glob
+import os
+import re
+
+from smolagents import Tool
+
+
+class GrepTool(Tool):
+    """
+    The GrepTool "greps" for file matches.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a GrepTool.
+        """
+
+        # Tool parameters
+        self.name = "grep"
+        self.description = (
+            "Implementation of a grep tool. Searches a list of files for matches. "
+            "Returns a dictionary with the file path as the key and list of matches as the value."
+        )
+        self.inputs = {
+            "search": {"type": "string", "description": "Search grep pattern"},
+            "files": {"type": "string", "description": "File or file glob pattern to search"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, search, files):
+        """
+        Greps a set of files for a pattern.
+
+        Args:
+            search: search pattern
+            files: file or file glob pattern to search
+
+        Returns:
+            {path: [matches]}
+        """
+
+        # Compile the search pattern
+        pattern = re.compile(search)
+
+        # Format file pattern
+        files = os.path.join(files, "*") if os.path.isdir(files) else files
+
+        # Iterate over each found file
+        results = {}
+        for path in glob.glob(files, recursive=True):
+            if os.path.isfile(path):
+                try:
+                    with open(path, "r", encoding="utf-8") as f:
+                        # Search for matches line-by-line
+                        matches = []
+                        for line in f:
+                            if pattern.search(line):
+                                matches.append(line)
+
+                        if matches:
+                            results[path] = matches
+
+                except (UnicodeDecodeError, FileNotFoundError):
+                    pass
+
+        return results

src/python/txtai/agent/tool/read.py

@@ -0,0 +1,59 @@
+"""
+Read imports
+"""
+
+import re
+
+from smolagents import Tool
+
+from ...pipeline import Textractor
+
+
+class ReadTool(Tool):
+    """
+    The ReadTool retrieves file or url content. This tool automatically extracts text content from
+    binary files using the Textractor pipeline.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self, maxlength=40000):
+        """
+        Creates a ReadTool.
+
+        Args:
+            maxlength: Truncate content above this maxlength
+        """
+
+        # Tool parameters
+        self.name = "read"
+        self.description = (
+            "Implementation of a file read tool. Returns file content. Also supports reading web content. "
+            "Use this tool to browse webpages in addition to reading files."
+        )
+        self.inputs = {"path": {"type": "string", "description": "File path or url"}}
+        self.output_type = "any"
+
+        # Create textractor instance
+        self.textractor = Textractor()
+        self.maxlength = maxlength
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path):
+        """
+        Reads content from path.
+
+        Args:
+            path: file path or url
+
+        Returns:
+            content
+        """
+
+        content = self.textractor(path)
+        content = re.sub(r"\n{3,}", "\n\n", content)
+
+        # Truncate content to max length
+        return content[: self.maxlength]

src/python/txtai/agent/tool/skill.py

@@ -0,0 +1,81 @@
+"""
+Skill imports
+"""
+
+from smolagents import Tool
+
+# Core library imports
+from ...util import Library
+
+yaml = Library().yaml()
+
+
+class SkillTool(Tool):
+    """
+    A SkillTool loads a skill.md file.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self, path):
+        """
+        Creates a SkillTool.
+
+        Args:
+            path: skill file path
+        """
+
+        # Load skill.md
+        metadata, content = self.load(path)
+
+        # Tool parameters
+        self.name = metadata["name"]
+        self.description = metadata["description"]
+        self.inputs = {"request": {"type": "string", "description": "The user requested action"}}
+        self.output_type = "any"
+        self.target = content
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, request):
+        """
+        Searchs the skill markdown for the best answer.
+
+        Args:
+            request: user request
+
+        Returns:
+            result
+        """
+
+        return f"""Given the request {request}, find the best answer using the content below.
+
+{self.target}
+"""
+
+    def load(self, path):
+        """
+        Loads a skill.md file.
+
+        Args:
+            path: path to skill.md
+
+        Returns:
+            metadata, content
+        """
+
+        # Read file content
+        with open(path, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        # Stores frontmatter metadata, if any
+        metadata = {}
+
+        # Split by "---"" to separate frontmatter and markdown
+        if content.startswith("---"):
+            _, frontmatter, content = content.split("---", 2)
+            metadata = yaml.safe_load(frontmatter)
+
+        # Return parsed metadata and content
+        return metadata, content

src/python/txtai/agent/tool/todo.py

@@ -0,0 +1,40 @@
+"""
+Todo imports
+"""
+
+from smolagents import Tool
+
+
+class TodoWriteTool(Tool):
+    """
+    The TodoWriteTool plans for task execution.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a TodoWriteTool.
+        """
+
+        # Tool parameters
+        self.name = "todowrite"
+        self.description = (
+            "Implementation of a todo write tool. Generates a structured task list to help organize complex tasks. "
+            "Only use this tool for complex tasks with multiple steps. Skip for simple tasks that can be done easily."
+        )
+        self.inputs = {"plan": {"type": "string", "description": "The task plan"}}
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, plan):
+        """
+        Returns the plan.
+
+        Returns:
+            plan
+        """
+
+        return plan

src/python/txtai/agent/tool/write.py

@@ -0,0 +1,42 @@
+"""
+Write imports
+"""
+
+from smolagents import Tool
+
+
+class WriteTool(Tool):
+    """
+    The WriteTool writes file content.
+    """
+
+    # pylint: disable=W0231
+    def __init__(self):
+        """
+        Creates a WriteTool.
+        """
+
+        # Tool parameters
+        self.name = "write"
+        self.description = "Implementation of a file write tool. Writes content to file."
+        self.inputs = {
+            "path": {"type": "string", "description": "Output file path"},
+            "content": {"type": "string", "description": "File content to write"},
+        }
+        self.output_type = "any"
+
+        # Validate parameters and initialize tool
+        super().__init__()
+
+    # pylint: disable=W0221
+    def forward(self, path, content):
+        """
+        Writes content to path.
+
+        Args:
+            path: output file path
+            content: content to write
+        """
+
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(content)

src/python/txtai/ann/dense/__init__.py

@@ -6,6 +6,10 @@ from .annoy import Annoy
 from .factory import ANNFactory
 from .faiss import Faiss
 from .hnsw import HNSW
+from .milvus import Milvus
 from .numpy import NumPy
 from .pgvector import PGVector
+from .rabitq import RabitQ
 from .torch import Torch
+from .turbovec import TurboVec
+from .zvec import Zvec

src/python/txtai/ann/dense/factory.py

@@ -4,14 +4,19 @@ Factory module
 
 from ...util import Resolver
 
+from ..base import ANN
 from .annoy import Annoy
-from .faiss import Faiss
+from .faiss import Faiss, FAISS
 from .ggml import GGML
 from .hnsw import HNSW
+from .milvus import Milvus
 from .numpy import NumPy
 from .pgvector import PGVector
+from .rabitq import RabitQ
 from .sqlite import SQLite
 from .torch import Torch
+from .turbovec import TurboVec
+from .zvec import Zvec
 
 
 class ANNFactory:
@@ -31,27 +36,36 @@ class ANNFactory:
             ANN
         """
 
+        # pylint: disable=R0912
         # ANN instance
         ann = None
-        backend = config.get("backend", "faiss")
+        backend = config.get("backend", "faiss" if FAISS else "numpy")
 
         # Create ANN instance
         if backend == "annoy":
             ann = Annoy(config)
         elif backend == "faiss":
             ann = Faiss(config)
-        elif backend == "hnsw":
-            ann = HNSW(config)
         elif backend == "ggml":
             ann = GGML(config)
+        elif backend == "hnsw":
+            ann = HNSW(config)
+        elif backend == "milvus":
+            ann = Milvus(config)
         elif backend == "numpy":
             ann = NumPy(config)
         elif backend == "pgvector":
             ann = PGVector(config)
+        elif backend == "rabitq":
+            ann = RabitQ(config)
         elif backend == "sqlite":
             ann = SQLite(config)
         elif backend == "torch":
             ann = Torch(config)
+        elif backend == "turbovec":
+            ann = TurboVec(config)
+        elif backend == "zvec":
+            ann = Zvec(config)
         else:
             ann = ANNFactory.resolve(backend, config)
 
@@ -74,6 +88,6 @@ class ANNFactory:
         """
 
         try:
-            return Resolver()(backend)(config)
+            return Resolver()(backend, ANN)(config)
         except Exception as e:
             raise ImportError(f"Unable to resolve ann backend: '{backend}'") from e

src/python/txtai/ann/dense/faiss.py

@@ -3,19 +3,32 @@ Faiss module
 """
 
 import math
+import os
 import platform
 
-import numpy as np
+# pylint: disable=C0413
+if platform.system() == "Darwin" or os.name == "nt":
+    # Workaround for a Faiss issue causing segmentation faults. See txtai FAQ for more.
+    os.environ["OMP_NUM_THREADS"] = os.environ.get("OMP_NUM_THREADS", "1")
 
-from faiss import omp_set_num_threads
-from faiss import index_factory, IO_FLAG_MMAP, METRIC_INNER_PRODUCT, read_index, write_index
-from faiss import index_binary_factory, read_index_binary, write_index_binary, IndexBinaryIDMap
+    # Workaround for a Faiss issue with OMP: Error #15. See txtai FAQ for more.
+    os.environ["KMP_DUPLICATE_LIB_OK"] = os.environ.get("KMP_DUPLICATE_LIB_OK", "TRUE")
+
+# Conditional import
+try:
+    from faiss import index_factory, IO_FLAG_MMAP, METRIC_INNER_PRODUCT, read_index, write_index
+    from faiss import index_binary_factory, read_index_binary, write_index_binary, IndexBinaryIDMap
+
+    FAISS = True
+except ImportError:
+    FAISS = False
 
 from ..base import ANN
 
-if platform.system() == "Darwin":
-    # Workaround for a Faiss issue causing segmentation faults on macOS. See txtai FAQ for more.
-    omp_set_num_threads(1)
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
 
 
 class Faiss(ANN):
@@ -26,6 +39,9 @@ class Faiss(ANN):
     def __init__(self, config):
         super().__init__(config)
 
+        if not FAISS:
+            raise ImportError("Faiss is not available - install faiss-cpu to enable")
+
         # Scalar quantization
         quantize = self.config.get("quantize")
         self.qbits = quantize if quantize and isinstance(quantize, int) and not isinstance(quantize, bool) else None
@@ -126,7 +142,7 @@ class Faiss(ANN):
 
         # Derive quantization. Prefer backend-specific setting. Fallback to root-level parameter.
         quantize = self.setting("quantize", self.config.get("quantize"))
-        quantize = 8 if isinstance(quantize, bool) else quantize
+        quantize = 8 if isinstance(quantize, bool) and quantize else quantize
 
         # Get storage setting
         storage = f"SQ{quantize}" if quantize else "Flat"
@@ -213,6 +229,7 @@ class Faiss(ANN):
         count = self.count()
 
         default = 6 if count <= 5000 else round(self.cells(count) / 16)
+        default = min(default, getattr(self.backend, "nlist", default))
         return self.setting("nprobe", default)
 
     def scores(self, scores):

src/python/txtai/ann/dense/ggml.py

@@ -5,8 +5,6 @@ GGML module
 import ctypes
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     import ggml
@@ -18,6 +16,11 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class GGML(ANN):
     """
@@ -152,7 +155,7 @@ class GGMLTensors:
         gguf = ggml.gguf_init_from_file(path.encode("utf-8"), params)
 
         # Load tensors from GGUF file
-        self.loadtensors(context)
+        self.loadtensors(gguf, context)
 
         # Create graph operation
         self.graph, self.output = self.creategraph()
@@ -215,11 +218,12 @@ class GGMLTensors:
             ids: ids to delete
         """
 
-        shape = utils.get_shape(self.data)
+        rows = self.rows()
 
-        # Filter any index greater than size of array
-        ids = [x for x in ids if x < shape[1]]
-        self.deletes.extend(ids)
+        # Set index ids as deleted, ignoring ids outside the array and ids already marked deleted
+        for x in ids:
+            if x < rows and x not in self.deletes:
+                self.deletes.append(x)
 
     def search(self, queries):
         """
@@ -247,7 +251,7 @@ class GGMLTensors:
             ggml.ggml_backend_graph_compute(self.backend, self.graph)
 
             # Get size of embeddings data
-            size = utils.get_shape(self.data)[1]
+            size = self.rows()
 
             # Get and return results
             results = np.zeros((batch.shape[0], size), dtype=np.float32)
@@ -268,7 +272,7 @@ class GGMLTensors:
             count
         """
 
-        return utils.get_shape(self.data)[1] - len(self.deletes) if self.data else 0
+        return self.rows() - len(self.deletes) if self.data else 0
 
     def save(self, path):
         """
@@ -284,9 +288,12 @@ class GGMLTensors:
         # Init and save data tensor
         gguf = ggml.gguf_init_empty()
 
-        # Add the data tensor
-        ggml.ggml_set_name(self.data, b"data")
-        ggml.gguf_add_tensor(gguf, self.data)
+        # Add the data tensor. GGUF can't store a tensor with zero rows, only save the dimensions for an empty index.
+        if self.rows():
+            ggml.ggml_set_name(self.data, b"data")
+            ggml.gguf_add_tensor(gguf, self.data)
+        else:
+            ggml.gguf_set_val_u32(gguf, b"dimensions", utils.get_shape(self.data)[0])
 
         # Optionally create and add the deletes tensor
         if self.deletes:
@@ -409,11 +416,12 @@ class GGMLTensors:
         # Copy embeddings data
         self.copy(data, self.data, tensortype, 0)
 
-    def loadtensors(self, context):
+    def loadtensors(self, gguf, context):
         """
         Loads existing tensors from context.
 
         Args:
+            gguf: gguf context
             context: ggml context
         """
 
@@ -432,6 +440,10 @@ class GGMLTensors:
 
             # Copy tensor data to backend
             ggml.ggml_backend_tensor_set(self.data, ggml.ggml_get_data(data), 0, ggml.ggml_nbytes(data))
+        else:
+            # Empty index, create tensors using the saved dimensions
+            dimensions = ggml.gguf_get_val_u32(gguf, ggml.gguf_find_key(gguf, b"dimensions"))
+            self.createtensors(np.zeros((0, dimensions), dtype=np.float32))
 
         # Load deletes tensor
         data = ggml.ggml_get_tensor(context, b"deletes")
@@ -461,8 +473,7 @@ class GGMLTensors:
         queries = ggml.ggml_new_tensor_2d(context, ggml.GGML_TYPE_F32, data.shape[1], self.querysize)
 
         # Embeddings data with space for both existing and new data
-        shape = utils.get_shape(self.data)
-        merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + shape[1])
+        merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + self.rows())
 
         # Create new buffer
         buffer = ggml.ggml_backend_alloc_ctx_tensors(context, backend)
@@ -508,7 +519,11 @@ class GGMLTensors:
 
         # Read tensor type
         tensortype = self.quantize
-        tensortype = "Q8_0" if isinstance(tensortype, bool) else f"Q{int(tensortype)}_0" if isinstance(tensortype, int) else tensortype
+        tensortype = (
+            "Q8_0"
+            if isinstance(tensortype, bool) and tensortype
+            else f"Q{int(tensortype)}_0" if isinstance(tensortype, int) and not isinstance(tensortype, bool) else tensortype
+        )
         tensortype = tensortype.upper() if tensortype else "F32"
 
         # Validate tensor type
@@ -568,3 +583,13 @@ class GGMLTensors:
         """
 
         return [queries[x : x + self.querysize] for x in range(0, len(queries), self.querysize)]
+
+    def rows(self):
+        """
+        Number of rows in the data tensor. Tensors with a single row or no rows report a 1-D shape, so the shape can't be used.
+
+        Returns:
+            row count
+        """
+
+        return ggml.ggml_nrows(self.data)

src/python/txtai/ann/dense/hnsw.py

@@ -2,8 +2,6 @@
 HNSW module
 """
 
-import numpy as np
-
 # Conditional import
 try:
     # pylint: disable=E0611
@@ -15,6 +13,11 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class HNSW(ANN):
     """
@@ -32,6 +35,9 @@ class HNSW(ANN):
         self.backend = Index(dim=self.config["dimensions"], space=self.config["metric"])
         self.backend.load_index(path)
 
+        # Default the delete counter for configurations that don't have it
+        self.config["deletes"] = self.config.get("deletes", 0)
+
     def index(self, embeddings):
         # Inner product is equal to cosine similarity on normalized vectors
         self.config["metric"] = "ip"
@@ -82,8 +88,8 @@ class HNSW(ANN):
         if ef:
             self.backend.set_ef(ef)
 
-        # Run the query
-        ids, distances = self.backend.knn_query(queries, k=limit)
+        # Run the query, clamp k to the current element count as hnswlib errors when k exceeds it
+        ids, distances = self.backend.knn_query(queries, k=min(limit, self.count()))
 
         # Map results to [(id, score)]
         results = []

src/python/txtai/ann/dense/milvus.py

@@ -0,0 +1,144 @@
+"""
+milvus-lite module
+"""
+
+import os
+import shutil
+import tempfile
+
+# Conditional import
+try:
+    import milvus_lite
+
+    MILVUS = True
+except ImportError:
+    MILVUS = False
+
+from ...archive import ArchiveFactory
+from ..base import ANN
+
+
+class Milvus(ANN):
+    """
+    Builds an ANN index using the milvus-lite library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not MILVUS:
+            raise ImportError('milvus-lite is not available - install "ann" extra to enable')
+
+        self.collection = None
+        self.directory = None
+        self.path = None
+
+    def load(self, path):
+        self.close()
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        try:
+            archive = ArchiveFactory.create(self.directory)
+            archive.load(path, "tar")
+            self.open()
+        except Exception:
+            self.close()
+            raise
+
+    def index(self, embeddings):
+        self.close()
+
+        # Lookup index settings
+        m = self.setting("m", 50)
+
+        # Create collection
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        # Use milvus-lite's native embedded API directly and keep this integration thin
+        self.backend = milvus_lite.MilvusLite(self.path)
+        schema = milvus_lite.CollectionSchema(
+            fields=[
+                milvus_lite.FieldSchema("id", milvus_lite.DataType.INT64, is_primary=True),
+                milvus_lite.FieldSchema("embedding", milvus_lite.DataType.FLOAT_VECTOR, dim=self.config["dimensions"]),
+            ]
+        )
+        self.collection = self.backend.create_collection("txtai", schema)
+        self.collection.create_index("embedding", {"index_type": "HNSW", "metric_type": "IP", "params": {"M": m}})
+        self.collection.load()
+
+        # Add items - position in embeddings is used as the id
+        self.insert(embeddings, 0)
+
+        # Add id offset and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"m": m, "milvus": milvus_lite.__version__})
+
+    def append(self, embeddings):
+        self.insert(embeddings, self.config["offset"])
+
+        # Update id offset and index metadata
+        self.config["offset"] += embeddings.shape[0]
+        self.metadata()
+
+    def delete(self, ids):
+        # Check length, ids can be a NumPy array
+        if len(ids):
+            self.collection.delete([int(x) for x in ids])
+
+    def search(self, queries, limit):
+        matches = self.collection.search(queries.tolist(), top_k=limit, metric_type="IP", anns_field="embedding")
+        return [[(int(match["id"]), float(match["distance"])) for match in results] for results in matches]
+
+    def count(self):
+        return self.collection.num_entities
+
+    def save(self, path):
+        self.collection.flush()
+        self.backend.close()
+        self.collection = None
+        self.backend = None
+
+        archive = ArchiveFactory.create(self.directory)
+        archive.save(path, "tar")
+        self.open()
+
+    def close(self):
+        # Release milvus-lite collection handles and the directory lock
+        if self.backend:
+            self.backend.close()
+
+        self.collection = None
+        super().close()
+
+        if self.directory and os.path.exists(self.directory):
+            shutil.rmtree(self.directory)
+
+        self.directory = None
+        self.path = None
+
+    def insert(self, embeddings, offset):
+        """
+        Inserts embeddings starting at offset.
+
+        Args:
+            embeddings: embeddings array
+            offset: starting id
+        """
+
+        if embeddings.shape[0]:
+            for start in range(0, embeddings.shape[0], 1024):
+                batch = embeddings[start : start + 1024]
+                self.collection.insert([{"id": offset + start + uid, "embedding": embedding.tolist()} for uid, embedding in enumerate(batch)])
+
+    def open(self):
+        """
+        Opens the milvus-lite database and collection
+        """
+
+        self.backend = milvus_lite.MilvusLite(self.path)
+        self.collection = self.backend.get_collection("txtai")
+
+        # Reopened collections with an index start released, load to enable search
+        self.collection.load()

src/python/txtai/ann/dense/numpy.py

@@ -2,15 +2,17 @@
 NumPy module
 """
 
-import numpy as np
-
-from safetensors import safe_open
-from safetensors.numpy import save_file
-
 from ...serialize import SerializeFactory
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+safetensors = library.safetensors()
+
 
 class NumPy(ANN):
     """
@@ -58,16 +60,20 @@ class NumPy(ANN):
         self.metadata()
 
     def delete(self, ids):
-        # Filter any index greater than size of array
-        ids = [x for x in ids if x < self.backend.shape[0]]
+        # Ignore IDs outside the array; negative IDs must not wrap to live rows.
+        ids = [x for x in ids if 0 <= x < self.backend.shape[0]]
 
-        # Clear specified ids
-        self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1])))
+        # Clear specified ids, zeros must match the array data type (e.g. uint8 for quantized data)
+        self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1]), dtype=self.backend.dtype))
 
     def search(self, queries, limit):
         if self.qbits:
             # Calculate hamming score for integer vectors
             scores = self.hammingscore(queries)
+
+            # Zero out deleted (all-zero) rows. Unlike dot product, hamming scores them nonzero,
+            # so they'd otherwise pass the caller's score > 0 filter and resurface in results.
+            scores[:, self.all(self.backend == 0, axis=1)] = 0
         else:
             # Dot product on normalized vectors is equal to cosine similarity
             scores = self.dot(self.tensor(queries), self.backend.T)
@@ -157,7 +163,7 @@ class NumPy(ANN):
         """
 
         # Merge metadata and tensors into single dictionary
-        with safe_open(path, framework="np") as f:
+        with safetensors.safe_open(path, framework="np") as f:
             return {**(f.metadata() if f.metadata() else {}), **{k: f.get_tensor(k) for k in f.keys()}}
 
     def savesafetensors(self, data, path, metadata=None):
@@ -170,7 +176,7 @@ class NumPy(ANN):
             metadata: additional metadata to save
         """
 
-        save_file(data, path, metadata)
+        safetensors.numpy.save_file(data, path, metadata)
 
     def hammingscore(self, queries):
         """

src/python/txtai/ann/dense/pgvector.py

@@ -4,8 +4,6 @@ PGVector module
 
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     from pgvector.sqlalchemy import BIT, HALFVEC, VECTOR
@@ -20,8 +18,12 @@ except ImportError:
 
 from ..base import ANN
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
-# pylint: disable=R0904
 class PGVector(ANN):
     """
     Builds an ANN index backed by a Postgres database.
@@ -69,7 +71,8 @@ class PGVector(ANN):
         self.metadata()
 
     def delete(self, ids):
-        self.database.execute(delete(self.table).where(self.table.c["indexid"].in_(ids)))
+        # Bind ids as int, NumPy integers don't match the indexid column
+        self.database.execute(delete(self.table).where(self.table.c["indexid"].in_([int(x) for x in ids])))
 
     def search(self, queries, limit):
         results = []

src/python/txtai/ann/dense/rabitq.py

@@ -0,0 +1,176 @@
+"""
+RabitQ module
+"""
+
+import math
+
+# Conditional import
+try:
+    from rabitqlib import HnswIndex, IvfIndex
+    from sklearn.cluster import KMeans
+
+    RABITQ = True
+except ImportError:
+    RABITQ = False
+
+from ..base import ANN
+
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+
+# Sentinel for unfilled top-k slots in native search results
+SENTINEL = 0xFFFFFFFF
+
+
+class RabitQ(ANN):
+    """
+    Builds an ANN index using the rabitqlib library (RaBitQ quantization).
+
+    Only the native index is stored, as a single file. No vectors are kept outside of it. ivf indexes support append
+    and delete natively. The native id of a row is its index id. hnsw indexes can't be extended or have rows removed,
+    so append and delete are not supported in that mode.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not RABITQ:
+            raise ImportError('rabitqlib is not available - install "ann" extra to enable')
+
+        # Number of rows removed from the native index (ivf mode only)
+        self.config["deletes"] = self.config.get("deletes", 0)
+
+    def load(self, path):
+        self.close()
+
+        # Invalid modes raise ValueError before native code runs
+        self.backend = IvfIndex.load(path) if self.mode() == "ivf" else HnswIndex.load(path)
+
+    def index(self, embeddings):
+        self.close()
+
+        # Mode is validated before any state is stored
+        mode = self.mode()
+
+        vectors = np.ascontiguousarray(embeddings, dtype=np.float32)
+        rows = vectors.shape[0]
+
+        # Quantization bits per dimension, validated by rabitqlib
+        nbits = self.setting("nbits", 1)
+
+        # Default clusters to 4 * sqrt(N), bounded to [1, N] so KMeans always fits
+        clusters = self.setting("clusters", None)
+        clusters = max(1, min(clusters if clusters else round(4 * math.sqrt(rows)), rows))
+
+        cluster = KMeans(n_clusters=clusters, random_state=0, n_init=1).fit(vectors)
+        centroids = np.ascontiguousarray(cluster.cluster_centers_, dtype=np.float32)
+        clusterids = np.ascontiguousarray(cluster.labels_, dtype=np.uint32)
+
+        dim = self.config["dimensions"]
+        if mode == "ivf":
+            backend = IvfIndex(dim=dim, max_elements=rows, num_clusters=clusters, nbits=nbits, metric="ip")
+            settings = {"mode": mode, "nbits": nbits, "clusters": clusters, "nprobe": self.setting("nprobe", self.nprobe(clusters))}
+        else:
+            backend = HnswIndex(
+                dim=dim,
+                max_elements=rows,
+                M=self.setting("m", 16),
+                ef_construction=self.setting("efconstruction", 200),
+                nbits=nbits,
+                metric="ip",
+                random_seed=self.setting("randomseed", 100),
+            )
+            settings = {
+                "mode": mode,
+                "nbits": nbits,
+                "m": self.setting("m", 16),
+                "efconstruction": self.setting("efconstruction", 200),
+                "efsearch": self.setting("efsearch", None),
+                "randomseed": self.setting("randomseed", 100),
+            }
+
+        backend.build(vectors, centroids, clusterids)
+        self.backend = backend
+
+        # Add id offset and index build metadata
+        self.config["offset"] = rows
+        self.config["deletes"] = 0
+        self.metadata(settings)
+
+    def append(self, embeddings):
+        if self.mode() != "ivf":
+            super().append(embeddings)
+
+        # Native ids continue from the last row ever added
+        embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+        self.backend.add(embeddings)
+
+        # Update id offset and index metadata
+        self.config["offset"] = self.backend.max_elements
+        self.metadata()
+
+    def delete(self, ids):
+        if self.mode() != "ivf":
+            super().delete(ids)
+
+        # Native remove hides rows from search. Unknown ids are silently ignored.
+        ids = [uid for uid in ids if 0 <= uid < self.backend.max_elements]
+        if ids:
+            self.config["deletes"] += self.backend.remove(np.array(ids, dtype=np.int64))
+
+    def search(self, queries, limit):
+        if not self.count():
+            return [[] for _ in queries]
+
+        queries = np.ascontiguousarray(queries, dtype=np.float32)
+
+        # The native index errors when k exceeds its row count
+        k = min(limit, self.count())
+
+        if self.mode() == "ivf":
+            nprobe = self.setting("nprobe", self.nprobe(self.backend.num_clusters))
+            ids, distances = self.backend.search(queries, k, nprobe)
+        else:
+            ids, distances = self.backend.search(queries, k, self.setting("efsearch", None) or 0)
+
+        # Map results to [(id, score)], dropping unfilled sentinel slots
+        return [
+            [(int(uid), float(1.0 - dist)) for uid, dist in zip(uids, dists) if uid != SENTINEL]
+            for uids, dists in zip(ids.tolist(), distances.tolist())
+        ]
+
+    def count(self):
+        return self.backend.max_elements - self.config["deletes"] if self.backend is not None else 0
+
+    def save(self, path):
+        self.backend.save(path)
+
+    def mode(self):
+        """
+        Returns the configured index mode.
+
+        Returns:
+            index mode, only "ivf" or "hnsw" are accepted
+        """
+
+        mode = self.setting("mode", "ivf")
+        if mode not in ("ivf", "hnsw"):
+            raise ValueError(f'Invalid RabitQ mode: {mode}. Supported modes are "ivf" and "hnsw".')
+
+        return mode
+
+    def nprobe(self, clusters):
+        """
+        Returns the default number of clusters to probe at search time.
+
+        Args:
+            clusters: number of IVF clusters
+
+        Returns:
+            default nprobe
+        """
+
+        return max(1, round(clusters / 16))

src/python/txtai/ann/dense/sqlite.py

@@ -32,9 +32,11 @@ class SQLite(ANN):
 
         # Quantization setting
         self.quantize = self.setting("quantize")
-        self.quantize = 8 if isinstance(self.quantize, bool) else int(self.quantize) if self.quantize else None
+        self.quantize = 8 if isinstance(self.quantize, bool) and self.quantize else int(self.quantize) if self.quantize else None
 
     def load(self, path):
+        if self.path != path:
+            self.close()
         self.path = path
 
     def index(self, embeddings):
@@ -55,7 +57,8 @@ class SQLite(ANN):
         self.metadata()
 
     def delete(self, ids):
-        self.database().executemany(self.deletesql(), [(x,) for x in ids])
+        # Bind ids as int, sqlite3 binds NumPy integers as blobs that match the wrong rows
+        self.database().executemany(self.deletesql(), [(int(x),) for x in ids])
 
     def search(self, queries, limit):
         results = []
@@ -73,6 +76,9 @@ class SQLite(ANN):
         return self.cursor.fetchone()[0]
 
     def save(self, path):
+        # Loading defers opening the connection until the first database operation.
+        self.database()
+
         # Temporary database
         if not self.path:
             # Save temporary database
@@ -193,10 +199,13 @@ class SQLite(ANN):
             connection.execute(self.tablesql())
 
             # The backup call will hang if there are uncommitted changes, need to copy over
-            # with iterdump (which is much slower)
-            for sql in self.connection.iterdump():
-                if self.tosql('insert into "{table}"') in sql.lower():
-                    connection.execute(sql)
+            # the rows (which is much slower). Quantized vectors must be inserted with their type.
+            embedding = "vec_bit(?)" if self.quantize == 1 else "vec_int8(?)" if self.quantize == 8 else "?"
+            rows = self.connection.execute(self.tosql("SELECT indexid, embedding FROM {table}"))
+            connection.executemany(self.tosql(f"INSERT INTO {{table}}(indexid, embedding) VALUES (?, {embedding})"), rows)
+
+            # Commit copied rows, otherwise they are rolled back when the connection is closed
+            connection.commit()
         else:
             # Database is up to date, can do a more efficient copy with SQLite C API
             self.connection.backup(connection)
@@ -254,7 +263,12 @@ class SQLite(ANN):
             SELECT
         """
 
-        return self.tosql(("SELECT indexid, 1 - distance FROM {table} " f"WHERE embedding MATCH {self.embeddingsql()} AND k = ? ORDER BY distance"))
+        # BIT distance counts differing bits; normalize it before converting to similarity.
+        # SQLite binary dimensions are the number of bits, not the packed byte count.
+        distance = f"distance / {float(self.config['dimensions'])}" if self.quantize == 1 else "distance"
+        return self.tosql(
+            f"SELECT indexid, 1 - ({distance}) FROM {{table}} " f"WHERE embedding MATCH {self.embeddingsql()} AND k = ? ORDER BY distance"
+        )
 
     def countsql(self):
         """

src/python/txtai/ann/dense/torch.py

@@ -2,9 +2,6 @@
 PyTorch module
 """
 
-import numpy as np
-import torch
-
 try:
     from bitsandbytes import matmul_4bit
     from bitsandbytes.functional import (
@@ -23,6 +20,13 @@ except ImportError:
 
 from .numpy import NumPy
 
+# Core library imports
+from ...util import Library
+
+library = Library()
+np = library.numpy()
+torch = library.torch()
+
 
 class Torch(NumPy):
     """

src/python/txtai/ann/dense/turbovec.py

@@ -0,0 +1,82 @@
+"""
+TurboVec module
+"""
+
+# Conditional import
+try:
+    from turbovec import IdMapIndex
+
+    TURBOVEC = True
+except ImportError:
+    TURBOVEC = False
+
+from ..base import ANN
+
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
+
+class TurboVec(ANN):
+    """
+    Builds an ANN index using the turbovec library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not TURBOVEC:
+            raise ImportError('turbovec is not available - install "ann" extra to enable')
+
+    def load(self, path):
+        # Load index
+        self.backend = IdMapIndex.load(path)
+
+    def index(self, embeddings):
+        # Lookup index settings
+        bitwidth = self.setting("bitwidth", 4)
+
+        # Create index
+        self.backend = IdMapIndex(dim=self.config["dimensions"], bit_width=bitwidth)
+
+        # Add items - position in embeddings is used as the id
+        self.backend.add_with_ids(embeddings, np.arange(embeddings.shape[0], dtype=np.uint64))
+
+        # Add id offset, delete counter and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"bitwidth": bitwidth})
+
+    def append(self, embeddings):
+        new = embeddings.shape[0]
+
+        self.backend.add_with_ids(embeddings, np.arange(self.config["offset"], self.config["offset"] + new, dtype=np.uint64))
+
+        # Update id offset and index metadata
+        self.config["offset"] += new
+        self.metadata()
+
+    def delete(self, ids):
+        # Mark elements as deleted to omit from search results
+        for uid in ids:
+            self.backend.remove(uid)
+
+    def search(self, queries, limit):
+        # Run the query
+        scores, ids = self.backend.search(queries=queries, k=limit)
+
+        # Map results to [(id, score)]
+        results = []
+        for x, score in enumerate(scores):
+
+            # Build (id, score) tuples, convert to ints and floats
+            results.append(list(zip(ids[x].tolist(), score.tolist())))
+
+        return results
+
+    def count(self):
+        return len(self.backend)
+
+    def save(self, path):
+        # Write index
+        self.backend.write(path)

src/python/txtai/ann/dense/zvec.py

@@ -0,0 +1,166 @@
+"""
+zvec module
+"""
+
+import os
+import shutil
+import tempfile
+
+# Conditional import
+try:
+    import zvec
+
+    ZVEC = True
+except ImportError:
+    ZVEC = False
+
+from ...archive import ArchiveFactory
+from ..base import ANN
+
+
+class Zvec(ANN):
+    """
+    Builds an ANN index using the zvec library.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not ZVEC:
+            raise ImportError('zvec is not available - install "ann" extra to enable')
+
+        self.directory = None
+        self.path = None
+
+    def load(self, path):
+        self.close()
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+
+        try:
+            archive = ArchiveFactory.create(self.directory)
+            archive.load(path, "tar")
+            self.backend = zvec.open(self.path)
+        except Exception:
+            self.close()
+            raise
+
+    def index(self, embeddings):
+        self.close()
+
+        # Lookup index settings
+        efconstruction = self.setting("efconstruction", 200)
+        m = self.setting("m", 50)
+
+        # Create collection
+        self.directory = tempfile.mkdtemp()
+        self.path = os.path.join(self.directory, "index")
+        schema = zvec.CollectionSchema(
+            name="txtai",
+            vectors=zvec.VectorSchema(
+                "embedding",
+                self.datatype(),
+                self.config["dimensions"],
+                index_param=zvec.HnswIndexParam(metric_type=zvec.MetricType.IP, m=m, ef_construction=efconstruction),
+            ),
+        )
+        self.backend = zvec.create_and_open(path=self.path, schema=schema)
+
+        # Add items - position in embeddings is used as the id
+        self.insert(embeddings, 0)
+
+        # Build HNSW index
+        self.backend.optimize()
+
+        # Add id offset and index build metadata
+        self.config["offset"] = embeddings.shape[0]
+        self.metadata({"efconstruction": efconstruction, "m": m, "zvec": zvec.__version__})
+
+    def append(self, embeddings):
+        self.insert(embeddings, self.config["offset"])
+
+        # Add new items to HNSW index
+        self.backend.optimize()
+
+        # Update id offset and index metadata
+        self.config["offset"] += embeddings.shape[0]
+        self.metadata()
+
+    def delete(self, ids):
+        # Check length, ids can be a NumPy array
+        if len(ids):
+            self.backend.delete([str(uid) for uid in ids])
+
+    def search(self, queries, limit):
+        # Lookup search settings
+        param = zvec.HnswQueryParam(ef=self.setting("efsearch", 300))
+
+        results = []
+        for query in queries:
+            matches = self.backend.query(
+                zvec.Query(field_name="embedding", vector=self.prepare(query), param=param),
+                topk=limit,
+            )
+            results.append([(int(match.id), float(match.score)) for match in matches])
+
+        return results
+
+    def count(self):
+        return self.backend.stats.doc_count
+
+    def save(self, path):
+        self.backend.flush()
+        self.backend = None
+
+        archive = ArchiveFactory.create(self.directory)
+        archive.save(path, "tar")
+        self.backend = zvec.open(self.path)
+
+    def close(self):
+        # Parent logic releases the collection and its file locks
+        super().close()
+
+        if self.directory and os.path.exists(self.directory):
+            shutil.rmtree(self.directory)
+
+        self.directory = None
+        self.path = None
+
+    def insert(self, embeddings, offset):
+        """
+        Inserts embeddings starting at offset.
+
+        Args:
+            embeddings: embeddings array
+            offset: starting id
+        """
+
+        if embeddings.shape[0]:
+            for start in range(0, embeddings.shape[0], 1024):
+                batch = embeddings[start : start + 1024]
+                self.backend.insert(
+                    [zvec.Doc(id=str(offset + start + uid), vectors={"embedding": self.prepare(embedding)}) for uid, embedding in enumerate(batch)]
+                )
+
+    def datatype(self):
+        """
+        Gets the vector data type.
+
+        Returns:
+            zvec vector data type
+        """
+
+        return zvec.DataType.VECTOR_FP32
+
+    def prepare(self, data):
+        """
+        Prepares data for the vector field.
+
+        Args:
+            data: input data
+
+        Returns:
+            data formatted for zvec
+        """
+
+        return data.tolist()

src/python/txtai/ann/sparse/__init__.py

@@ -5,3 +5,4 @@ Sparse ANN imports
 from .factory import SparseANNFactory
 from .ivfsparse import IVFSparse
 from .pgsparse import PGSparse
+from .zvecsparse import ZvecSparse

src/python/txtai/ann/sparse/factory.py

@@ -4,8 +4,10 @@ Factory module
 
 from ...util import Resolver
 
+from ..base import ANN
 from .ivfsparse import IVFSparse
 from .pgsparse import PGSparse
+from .zvecsparse import ZvecSparse
 
 
 class SparseANNFactory:
@@ -34,6 +36,8 @@ class SparseANNFactory:
             ann = IVFSparse(config)
         elif backend == "pgsparse":
             ann = PGSparse(config)
+        elif backend == "zvecsparse":
+            ann = ZvecSparse(config)
         else:
             ann = SparseANNFactory.resolve(backend, config)
 
@@ -56,6 +60,6 @@ class SparseANNFactory:
         """
 
         try:
-            return Resolver()(backend)(config)
+            return Resolver()(backend, ANN)(config)
         except Exception as e:
             raise ImportError(f"Unable to resolve sparse ann backend: '{backend}'") from e

src/python/txtai/ann/sparse/ivfsparse.py

@@ -7,8 +7,6 @@ import os
 
 from multiprocessing.pool import ThreadPool
 
-import numpy as np
-
 # Conditional import
 try:
     from scipy.sparse import csr_matrix, vstack
@@ -22,9 +20,12 @@ except ImportError:
     IVFSPARSE = False
 
 from ...serialize import SerializeFactory
-from ...util import SparseArray
+from ...util import Library, SparseArray
 from ..base import ANN
 
+# Core library imports
+np = Library().numpy()
+
 
 class IVFSparse(ANN):
     """
@@ -114,8 +115,12 @@ class IVFSparse(ANN):
         self.metadata()
 
     def delete(self, ids):
-        # Set index ids as deleted
-        self.deletes.extend(ids)
+        # Set index ids as deleted, ignoring ids that were never indexed and ids already marked deleted
+        # Ids are stored as int, the deletes list is serialized with msgpack
+        size = self.size()
+        for x in ids:
+            if x < size and x not in self.deletes:
+                self.deletes.append(int(x))
 
     def search(self, queries, limit):
         results = []
@@ -203,10 +208,10 @@ class IVFSparse(ANN):
             cluster centroids
         """
 
-        # Select top n most important features that contribute to L2 vector norm
-        indices = np.argsort(-norm(train, axis=0))[: self.setting("nfeatures", 25)]
-        data = train[:, indices]
-        data = train
+        # Select top n most important features that contribute to L2 vector norm.
+        # nfeatures is opt-in: the full feature space is used when it isn't set.
+        nfeatures = self.setting("nfeatures")
+        data = train[:, np.argsort(-norm(train, axis=0))[:nfeatures]] if nfeatures else train
 
         # Cluster data using k-means
         kmeans = MiniBatchKMeans(n_clusters=clusters, random_state=0, n_init=5).fit(data)
@@ -269,9 +274,13 @@ class IVFSparse(ANN):
             scores[:, deletes] = 0
 
         # Get top n matching indices and scores
-        indices = np.argpartition(-scores, limit if limit < scores.shape[0] else scores.shape[0] - 1)[:, :limit]
+        indices = np.argpartition(-scores, limit if limit < scores.shape[1] else scores.shape[1] - 1)[:, :limit]
         scores = np.take_along_axis(scores, indices, axis=1)
 
+        # argpartition doesn't order within the top n - sort each row by score descending
+        order = np.argsort(-scores, axis=1)
+        indices, scores = np.take_along_axis(indices, order, axis=1), np.take_along_axis(scores, order, axis=1)
+
         return indices, scores
 
     def scan(self, query, limit, blockids):

src/python/txtai/ann/sparse/pgsparse.py

@@ -4,8 +4,6 @@ PGSparse module
 
 import os
 
-import numpy as np
-
 # Conditional import
 try:
     from pgvector import SparseVector
@@ -17,6 +15,11 @@ except ImportError:
 
 from ..dense import PGVector
 
+# Core library imports
+from ...util import Library
+
+np = Library().numpy()
+
 
 class PGSparse(PGVector):
     """

src/python/txtai/ann/sparse/zvecsparse.py

@@ -0,0 +1,35 @@
+"""
+ZvecSparse module
+"""
+
+# Conditional import
+try:
+    import zvec
+except ImportError:
+    pass
+
+from ..dense import Zvec
+
+
+class ZvecSparse(Zvec):
+    """
+    Builds a Sparse ANN index using the zvec library.
+    """
+
+    def search(self, queries, limit):
+        # Lookup search settings
+        param = zvec.HnswQueryParam(ef=self.setting("efsearch", 300))
+
+        results = []
+        for query in queries:
+            # zvec requires at least one non-zero value per query
+            matches = self.backend.query(zvec.Query(field_name="embedding", vector=self.prepare(query), param=param), topk=limit) if query.nnz else []
+            results.append([(int(match.id), float(match.score)) for match in matches])
+
+        return results
+
+    def datatype(self):
+        return zvec.DataType.SPARSE_VECTOR_FP32
+
+    def prepare(self, data):
+        return {int(index): float(value) for index, value in zip(data.indices, data.data)}

src/python/txtai/api/application.py

@@ -8,10 +8,11 @@ import sys
 
 from fastapi import APIRouter, Depends, FastAPI
 from fastapi_mcp import FastApiMCP
-from httpx import AsyncClient
+from httpx import AsyncClient, ASGITransport
 
 from .authorization import Authorization
 from .base import API
+from .extension import Extension
 from .factory import APIFactory
 
 from ..app import Application
@@ -95,15 +96,15 @@ def lifespan(application):
 
     # Conditionally add routes based on configuration
     for name, router in routers.items():
-        if name in config:
+        if name in config and enabled(config, name):
             application.include_router(router)
 
     # Special case for embeddings clusters
-    if "cluster" in config and "embeddings" not in config:
+    if "cluster" in config and "embeddings" not in config and enabled(config, "embeddings"):
         application.include_router(routers["embeddings"])
 
     # Special case to add similarity instance for embeddings
-    if "embeddings" in config and "similarity" not in config:
+    if "embeddings" in config and "similarity" not in config and enabled(config, "similarity"):
         application.include_router(routers["similarity"])
 
     # Execute extensions if present
@@ -111,17 +112,56 @@ def lifespan(application):
     if extensions:
         for extension in extensions.split(","):
             # Create instance and execute extension
-            extension = APIFactory.get(extension.strip())()
+            extension = APIFactory.get(extension.strip(), Extension)()
             extension(application)
 
     # Add Model Context Protocol (MCP) service, if applicable
-    if config.get("mcp"):
-        mcp = FastApiMCP(application, http_client=AsyncClient(timeout=100))
-        mcp.mount()
+    createmcp(application, config)
 
     yield
 
 
+def enabled(config, name):
+    """
+    Check configuration to determine if a route is enabled or disabled
+
+    Args:
+        config: configuration
+        name: route name
+
+    Returns:
+        True if a route is enabled, False otherwise
+    """
+
+    return config.get("routes", {}).get(name, True)
+
+
+def createmcp(application, config):
+    """
+    Create a MCP service if necessary.
+
+    Args:
+        application: FastAPI Application
+        config: configuration
+    """
+
+    mcp = config.get("mcp")
+    if mcp:
+        # HTTP Client arguments
+        defaults = {"base_url": "http://apiserver", "timeout": 100}
+        clientargs = mcp.get("clientargs", {}) if isinstance(mcp, dict) else {}
+        clientargs = {**defaults, **clientargs}
+
+        # Create HTTP client with custom options
+        client = AsyncClient(transport=ASGITransport(app=application, raise_app_exceptions=False), **clientargs)
+
+        # MCP service arguments
+        mcpargs = mcp.get("mcpargs", {}) if isinstance(mcp, dict) else {}
+
+        mcp = FastApiMCP(application, http_client=client, **mcpargs)
+        mcp.mount()
+
+
 def start():
     """
     Runs application lifespan handler.

src/python/txtai/api/authorization.py

@@ -3,9 +3,10 @@ Authorization module
 """
 
 import hashlib
-import os
+import hmac
 
-from fastapi import Header, HTTPException
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 
 
 class Authorization:
@@ -13,17 +14,17 @@ class Authorization:
     Basic token authorization.
     """
 
-    def __init__(self, token=None):
+    def __init__(self, token):
         """
         Creates a new Authorization instance.
 
         Args:
-            token: SHA-256 hash of token to check
+            token: SHA-256 hash of the valid authorization token
         """
 
-        self.token = token if token else os.environ.get("TOKEN")
+        self.token = token
 
-    def __call__(self, authorization: str = Header(default=None)):
+    def __call__(self, authorization: HTTPAuthorizationCredentials = Depends(HTTPBearer())):
         """
         Validates authorization header is present and equal to current token.
 
@@ -31,23 +32,19 @@ class Authorization:
             authorization: authorization header
         """
 
-        if not authorization or self.token != self.digest(authorization):
+        # Authorization header is parsed into {scheme="Bearer", credentials="<token>"}
+        if not hmac.compare_digest(self.token, self.digest(authorization.credentials)):
             raise HTTPException(status_code=401, detail="Invalid Authorization Token")
 
-    def digest(self, authorization):
+    def digest(self, token):
         """
         Computes a SHA-256 hash for input authorization token.
 
         Args:
-            authorization: authorization header
+            token: authorization token
 
         Returns:
             SHA-256 hash of authorization token
         """
 
-        # Replace Bearer prefix
-        prefix = "Bearer "
-        token = authorization[len(prefix) :] if authorization.startswith(prefix) else authorization
-
-        # Compute SHA-256 hash
         return hashlib.sha256(token.encode("utf-8")).hexdigest()

src/python/txtai/api/base.py

@@ -32,7 +32,7 @@ class API(Application):
         weights = self.weights(request.query_params.get("weights") if request and hasattr(request, "query_params") else weights)
         index = request.query_params.get("index") if request and hasattr(request, "query_params") else index
         parameters = request.query_params.get("parameters") if request and hasattr(request, "query_params") else parameters
-        graph = request.query_params.get("graph") if request and hasattr(request, "query_params") else graph
+        graph = self.graph(request.query_params.get("graph") if request and hasattr(request, "query_params") else graph)
 
         # Decode parameters
         parameters = json.loads(parameters) if parameters and isinstance(parameters, str) else parameters
@@ -157,3 +157,17 @@ class API(Application):
         """
 
         return float(weights) if weights else weights
+
+    def graph(self, graph):
+        """
+        Parses the graph parameter from the request.
+
+        Args:
+            graph: graph parameter
+
+        Returns:
+            graph as a bool
+        """
+
+        # Query parameters are strings - accept the same values the batchsearch endpoint coerces
+        return graph.strip().lower() in ("1", "true", "yes", "on") if isinstance(graph, str) else bool(graph)

src/python/txtai/api/cluster.py

@@ -56,17 +56,19 @@ class Cluster:
         """
 
         # Build URL
-        action = f"search?query={urllib.parse.quote_plus(query)}"
+        params = {"query": query}
         if limit:
-            action += f"&limit={limit}"
-        if weights:
-            action += f"&weights={weights}"
+            params["limit"] = limit
+        if weights is not None:
+            params["weights"] = weights
         if index:
-            action += f"&index={index}"
+            params["index"] = index
         if parameters:
-            action += f"&parameters={json.dumps(parameters) if isinstance(parameters, dict) else parameters}"
+            params["parameters"] = json.dumps(parameters) if isinstance(parameters, dict) else parameters
         if graph is not None:
-            action += f"&graph={graph}"
+            params["graph"] = graph
+
+        action = f"search?{urllib.parse.urlencode(params)}"
 
         # Run query and flatten results into single results list
         results = []
@@ -100,7 +102,7 @@ class Cluster:
         params = {"queries": queries}
         if limit:
             params["limit"] = limit
-        if weights:
+        if weights is not None:
             params["weights"] = weights
         if index:
             params["index"] = index
@@ -196,11 +198,11 @@ class Cluster:
         shards = [[] for _ in range(len(self.shards))]
         for document in documents:
             uid = document.get("id") if isinstance(document, dict) else document
-            if uid and isinstance(uid, str):
-                # Quick int hash of string to help derive shard id
+            if isinstance(uid, str):
+                # Quick int hash of string to help derive shard id (handles empty strings too)
                 uid = zlib.adler32(uid.encode("utf-8"))
-            elif uid is None:
-                # Get random shard id when uid isn't set
+            elif not isinstance(uid, int):
+                # Get random shard id when uid isn't a usable int (None, a tuple document, etc.)
                 uid = random.randint(0, len(shards) - 1)
 
             shards[uid % len(self.shards)].append(document)

src/python/txtai/api/factory.py

@@ -4,6 +4,8 @@ API factory module
 
 from ..util import Resolver
 
+from .base import API
+
 
 class APIFactory:
     """
@@ -11,18 +13,19 @@ class APIFactory:
     """
 
     @staticmethod
-    def get(api):
+    def get(path, base=None):
         """
-        Gets a new instance of api class.
+        Gets a new instance of a class.
 
         Args:
-            api: API instance class
+            path: class path
+            base: optional required base class
 
         Returns:
-            API
+            Class
         """
 
-        return Resolver()(api)
+        return Resolver()(path, base)
 
     @staticmethod
     def create(config, api):
@@ -37,4 +40,4 @@ class APIFactory:
             API instance
         """
 
-        return APIFactory.get(api)(config)
+        return APIFactory.get(api, API)(config)

src/python/txtai/api/route.py

@@ -2,9 +2,12 @@
 Route module
 """
 
-from fastapi.routing import APIRoute, get_request_handler
+import json
 
-from .responses import ResponseFactory
+from fastapi.responses import StreamingResponse
+from fastapi.routing import APIRoute
+
+from .responses import ResponseFactory, MessagePackResponse
 
 
 class EncodingAPIRoute(APIRoute):
@@ -20,22 +23,26 @@ class EncodingAPIRoute(APIRoute):
             route handler function
         """
 
+        # Get handle to the original route handler
+        original = super().get_route_handler()
+
         async def handler(request):
-            route = get_request_handler(
-                dependant=self.dependant,
-                body_field=self.body_field,
-                status_code=self.status_code,
-                response_class=ResponseFactory.create(request),
-                response_field=self.secure_cloned_response_field,
-                response_model_include=self.response_model_include,
-                response_model_exclude=self.response_model_exclude,
-                response_model_by_alias=self.response_model_by_alias,
-                response_model_exclude_unset=self.response_model_exclude_unset,
-                response_model_exclude_defaults=self.response_model_exclude_defaults,
-                response_model_exclude_none=self.response_model_exclude_none,
-                dependency_overrides_provider=self.dependency_overrides_provider,
+            # Target response class
+            target = ResponseFactory.create(request)
+            request.state.response_class = target
+
+            # Get response
+            response = await original(request)
+
+            # Force MessagePackResponse when it's requested and response type doesn't match
+            return (
+                target(
+                    content=json.loads(response.body),
+                    status_code=response.status_code,
+                    headers={k: v for k, v in response.headers.items() if k.lower() not in ["content-length", "content-type"]},
+                )
+                if not isinstance(response, (target, StreamingResponse)) and target == MessagePackResponse
+                else response
             )
 
-            return await route(request)
-
         return handler

src/python/txtai/api/routers/embeddings.py

@@ -14,7 +14,7 @@ from .. import application
 from ..responses import ResponseFactory
 from ..route import EncodingAPIRoute
 
-from ...app import ReadOnlyError
+from ...app import ReadOnlyError, ReindexDisabledError
 from ...graph import Graph
 
 router = APIRouter(route_class=EncodingAPIRoute)
@@ -198,7 +198,7 @@ def reindex(config: dict = Body(...), function: str = Body(default=None)):
 
     try:
         application.get().reindex(config, function)
-    except ReadOnlyError as e:
+    except (ReadOnlyError, ReindexDisabledError) as e:
         raise HTTPException(status_code=403, detail=e.args[0]) from e
 
 

src/python/txtai/api/routers/openai.py

@@ -44,8 +44,8 @@ def chat(
     # Build keyword arguments
     kwargs = {key: value for key, value in [("stream", stream), ("maxlength", max_completion_tokens)] if value}
 
-    # Get first message
-    message = messages[0]["content"]
+    # Get latest message
+    message = messages[-1]["content"]
 
     # Agent
     if model in application.get().agents:
@@ -165,9 +165,9 @@ class ChatResponse:
         return {
             "id": str(uuid.uuid4()),
             "object": "chat.completion",
-            "created": int(time.time() * 1000),
+            "created": int(time.time()),
             "model": model,
-            "choices": [{"id": 0, "message": {"role": "assistant", "content": result}, "finish_reason": "stop"}],
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": result}, "finish_reason": "stop"}],
         }
 
 
@@ -177,15 +177,38 @@ class StreamingChatResponse:
     """
 
     def __call__(self, model, result):
+        # All chunks of a response share the same id and creation time
+        uid, created = str(uuid.uuid4()), int(time.time())
+
         for chunk in result:
-            yield "data: " + json.dumps(
-                {
-                    "id": str(uuid.uuid4()),
-                    "object": "chat.completion.chunk",
-                    "created": int(time.time() * 1000),
-                    "model": model,
-                    "choices": [{"id": 0, "delta": {"content": chunk}}],
-                }
-            ) + "\n\n"
+            yield self.chunk(uid, created, model, {"content": chunk}, None)
+
+        # Final chunk signals the end of the response
+        yield self.chunk(uid, created, model, {}, "stop")
 
         yield "data: [DONE]\n\n"
+
+    def chunk(self, uid, created, model, delta, reason):
+        """
+        Builds a server-sent event for a single chat completion chunk.
+
+        Args:
+            uid: response id
+            created: response creation time in seconds
+            model: model name
+            delta: chunk content
+            reason: finish reason, None until the final chunk
+
+        Returns:
+            server-sent event
+        """
+
+        data = {
+            "id": uid,
+            "object": "chat.completion.chunk",
+            "created": created,
+            "model": model,
+            "choices": [{"index": 0, "delta": delta, "finish_reason": reason}],
+        }
+
+        return "data: " + json.dumps(data) + "\n\n"

src/python/txtai/app/__init__.py

@@ -2,4 +2,4 @@
 App imports
 """
 
-from .base import Application, ReadOnlyError
+from .base import Application, ReadOnlyError, ReindexDisabledError