deepset-ai/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2026-10-12 02:12:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Merge main into v3 (#11495)

Browse Source 
Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Pragnyan Ramtha <[email protected]>
Co-authored-by: Julian Risch <[email protected]>
Co-authored-by: Claude Sonnet 4.6 <[email protected]>
Co-authored-by: Aarkin Karnik <[email protected]>
Co-authored-by: Haystack Bot <[email protected]>
Co-authored-by: Stefano Fiorucci <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: davidsbatista <[email protected]>
Co-authored-by: bogdankostic <[email protected]>
Co-authored-by: bogdankostic <[email protected]>
Co-authored-by: Sachin Yadav <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: anakin87 <[email protected]>
Co-authored-by: Abdulhamid Onawole <[email protected]>
Co-authored-by: David S. Batista <[email protected]>
Co-authored-by: Aditya Raut <[email protected]>
Co-authored-by: Nhat Huy Vu <[email protected]>
Co-authored-by: julian-risch <[email protected]>
Co-authored-by: kota <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Co-authored-by: devteamaegis <[email protected]>
Co-authored-by: Ritwij Aryan Parmar <[email protected]>
Co-authored-by: Srivatsa Kamballa <[email protected]>
This commit is contained in:
Sebastian Husch Lee
2026-06-03 11:43:55 +00:00
committed by GitHub
co-authored by Pragnyan Ramtha Julian Risch Claude Sonnet 4.6 Aarkin Karnik Haystack Bot Stefano Fiorucci dependabot[bot] davidsbatista bogdankostic bogdankostic Sachin Yadav Copilot Autofix powered by AI anakin87 Abdulhamid Onawole David S. Batista Aditya Raut Nhat Huy Vu julian-risch kota Copilot Autofix powered by AI devteamaegis Ritwij Aryan Parmar Srivatsa Kamballa
parent 3b1558fabd
commit aac56ef8b6
595 changed files with 113360 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/dependabot.yml
+11
View File
@@ -20,3 +20,14 @@ updates:
interval: 'daily'
cooldown:
default-days: 1
# Tracks the digest-pinned `COPY --from` image (ghcr.io/astral-sh/uv) in
# docker/Dockerfile.base. Note: the python:3.12-slim base/builder images are
# passed via build args from docker/docker-bake.hcl and the `docker run` image
# strings in workflows are not parsed by Dependabot; bump those manually.
- package-ecosystem: 'docker'
directory: '/docker'
schedule:
interval: 'daily'
cooldown:
default-days: 1
.github/workflows/auto_approve_api_ref_sync.yml
+4 -2
View File
@@ -12,14 +12,16 @@ on:
- "docs-website/reference_versioned_docs/**"
permissions:
pull-requests: write
contents: write
contents: read
env:
GH_TOKEN: ${{ github.token }}
jobs:
auto-approve-and-merge:
permissions:
contents: write
pull-requests: write
if: |
github.event.pull_request.user.login == 'HaystackBot' &&
startsWith(github.event.pull_request.head.ref, 'sync-docusaurus-api-reference') &&
.github/workflows/branch_off.yml
+8 -1
View File
@@ -1,7 +1,6 @@
name: Branch off
on:
workflow_dispatch:
workflow_call:
outputs:
bump_version_pr_url:
@@ -10,6 +9,9 @@ on:
env:
PYTHON_VERSION: "3.10"
permissions:
contents: read
jobs:
branch-off:
runs-on: ubuntu-slim
@@ -21,6 +23,11 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: main
# Persist the bot token so the branch/tag pushes below authenticate as
# HaystackBot (a ruleset-bypass actor) instead of the default
# github-actions[bot], which only has contents:read and is blocked by
# the release rulesets.
token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
- name: Define all versions
id: versions
.github/workflows/check_api_ref.yml
+3
View File
@@ -6,6 +6,9 @@ on:
- "haystack/**/*.py"
- "pydoc/*.yml"
permissions:
contents: read
jobs:
test-api-reference-build:
runs-on: ubuntu-slim
.github/workflows/ci_metrics.yml
-23
View File
@@ -1,23 +0,0 @@
name: CI Metrics
on:
workflow_run:
workflows:
- "end-to-end"
- "Tests"
- "Slow Integration Tests"
types:
- completed
pull_request:
types:
- opened
- closed
jobs:
send:
runs-on: ubuntu-slim
steps:
- uses: int128/datadog-actions-metrics@614f144c7824871780f09ab0925c9048edc5e9de # v1.166.0
with:
datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
datadog-site: "datadoghq.eu"
collect-job-metrics: true
.github/workflows/claude-code-review.yml
+3
View File
@@ -7,6 +7,9 @@ on:
pull_request_target:
types: [labeled]
permissions:
contents: read
jobs:
claude-review:
if: |
.github/workflows/claude.yml
+3
View File
@@ -10,6 +10,9 @@ on:
pull_request_review:
types: [submitted]
permissions:
contents: read
jobs:
claude:
if: |
.github/workflows/coverage_comment.yml
+3
View File
@@ -6,6 +6,9 @@ on:
- "Tests"
types: [completed]
permissions:
contents: read
jobs:
comment:
if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
.github/workflows/docker_release.yml
+3
View File
@@ -17,6 +17,9 @@ on:
env:
DOCKER_REPO_NAME: deepset/haystack
permissions:
contents: read
jobs:
build-and-push:
name: Build base image
.github/workflows/docs-website-test-docs-snippets.yml
+3
View File
@@ -15,6 +15,9 @@ env:
HATCH_VERSION: "1.16.5"
PYTHON_VERSION: "3.11"
permissions:
contents: read
jobs:
test-docs-snippets:
runs-on: ubuntu-latest
.github/workflows/docs_search_sync.yml
+3
View File
@@ -10,6 +10,9 @@ concurrency:
group: docs-search-sync
cancel-in-progress: false
permissions:
contents: read
jobs:
docs-search-sync:
runs-on: ubuntu-latest
.github/workflows/docstring_labeler.yml
+33 -24
View File
@@ -1,15 +1,24 @@
name: Add label on docstrings edit
name: Detect docstrings edit
# This workflow runs in the *untrusted* pull_request context: it has a read-only
# token and no access to secrets, so checking out the PR head here is safe.
# It only computes whether docstrings changed and hands the result to the
# "Apply docstrings label" workflow (workflow_run), which holds the write token
# but never checks out untrusted code. See:
# https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
on:
pull_request_target:
pull_request:
paths:
- "haystack/**/*.py"
permissions:
contents: read
env:
PYTHON_VERSION: "3.11"
jobs:
label:
detect:
runs-on: ubuntu-slim
steps:
@@ -18,42 +27,42 @@ jobs:
with:
ref: ${{ github.base_ref }}
- name: Copy file
# We copy our script after base ref checkout so we keep executing
# the same version even after checking out the HEAD ref.
# This is done to prevent executing malicious code in forks' PRs.
run: cp .github/utils/docstrings_checksum.py "${{ runner.temp }}/docstrings_checksum.py"
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Get docstrings
- name: Get base docstrings
id: base-docstrings
run: |
CHECKSUM=$(python "${{ runner.temp }}/docstrings_checksum.py" --root "${{ github.workspace }}")
CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
- name: Checkout HEAD commit
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.ref }}
# This must be set to correctly checkout a fork
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.sha }}
- name: Get docstrings
- name: Get HEAD docstrings
id: head-docstrings
run: |
CHECKSUM=$(python "${{ runner.temp }}/docstrings_checksum.py" --root "${{ github.workspace }}")
CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
- name: Check if we should label
id: run-check
run: echo "should_run=${{ steps.base-docstrings.outputs.checksum != steps.head-docstrings.outputs.checksum }}" >> "$GITHUB_OUTPUT"
- name: Write result
# The follow-up workflow reads these values; it never checks out the PR code.
run: |
mkdir -p result
echo "${{ github.event.pull_request.number }}" > result/pr_number
if [ "${{ steps.base-docstrings.outputs.checksum }}" != "${{ steps.head-docstrings.outputs.checksum }}" ]; then
echo "true" > result/should_label
else
echo "false" > result/should_label
fi
- name: Add label
if: ${{ steps.run-check.outputs.should_run == 'true' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh pr edit ${{ github.event.pull_request.html_url }} --add-label "type:documentation"
- name: Upload result
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: docstring-label-result
path: result/
retention-days: 1
.github/workflows/docstring_labeler_apply.yml
+42
View File
@@ -0,0 +1,42 @@
name: Apply docstrings label
# Runs after "Detect docstrings edit" completes. This workflow holds the write
# token but only consumes the data artifact produced by the untrusted run — it
# never checks out PR code, so it is safe under the workflow_run trigger.
on:
workflow_run:
workflows: ["Detect docstrings edit"]
types:
- completed
permissions:
contents: read
jobs:
label:
runs-on: ubuntu-slim
if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
permissions:
contents: read
pull-requests: write
actions: read
steps:
- name: Download result
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: docstring-label-result
path: result
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Add label
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
SHOULD_LABEL=$(cat result/should_label)
PR_NUMBER=$(cat result/pr_number)
if [ "$SHOULD_LABEL" = "true" ]; then
gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label "type:documentation"
else
echo "Docstrings unchanged, nothing to label."
fi
.github/workflows/docusaurus_sync.yml
+3
View File
@@ -14,6 +14,9 @@ env:
HATCH_VERSION: "1.16.5"
PYTHON_VERSION: "3.11"
permissions:
contents: read
jobs:
sync:
runs-on: ubuntu-slim
.github/workflows/e2e.yml
+3 -1
View File
@@ -1,4 +1,3 @@
# If you change this name also do it in ci_metrics.yml
name: end-to-end
on:
@@ -22,6 +21,9 @@ env:
# see https://github.com/deepset-ai/haystack/issues/9552
HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
permissions:
contents: read
jobs:
run:
timeout-minutes: 60
.github/workflows/github_release.yml
+7 -1
View File
@@ -7,9 +7,15 @@ on:
- "v2.[0-9]+.[0-9]+*"
# Ignore release versions tagged with -rc0 suffix
- "!v2.[0-9]+.[0-9]-rc0"
permissions:
contents: read
jobs:
generate-notes:
runs-on: ubuntu-latest
permissions:
contents: write # ncipollo/release-action creates the GitHub release
steps:
- name: Checkout
@@ -54,7 +60,7 @@ jobs:
reno report --no-show-source --ignore-cache --earliest-version "$EARLIEST_VERSION" -o relnotes.rst
- name: Convert to Markdown
uses: docker://pandoc/core:3.8
uses: docker://pandoc/core:3.8@sha256:c7127705b9d84c1b4acbbe411a86e3a6d67ac57870654dc378b6fe636ab747df
with:
args: "--from rst --to gfm --syntax-highlighting=none --wrap=none relnotes.rst -o relnotes.md"
.github/workflows/labeler.yml
+3 -1
View File
@@ -4,11 +4,13 @@ on:
permissions:
contents: read
pull-requests: write
jobs:
triage:
runs-on: ubuntu-slim
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
with:
.github/workflows/license_compliance.yml
+3
View File
@@ -12,6 +12,9 @@ on:
env:
PYTHON_VERSION: "3.10"
permissions:
contents: read
jobs:
license_check_direct:
name: Direct dependencies only
.github/workflows/nightly_testpypi_release.yml
+3
View File
@@ -9,6 +9,9 @@ on:
env:
HATCH_VERSION: "1.16.5"
permissions:
contents: read
jobs:
nightly-release:
runs-on: ubuntu-latest
.github/workflows/project.yml
+3
View File
@@ -5,6 +5,9 @@ on:
types:
- opened
permissions:
contents: read
jobs:
add-to-project:
name: Add new issues to project for triage
.github/workflows/promote_unstable_docs.yml
+3
View File
@@ -11,6 +11,9 @@ on:
env:
PYTHON_VERSION: "3.10"
permissions:
contents: read
jobs:
promote:
runs-on: ubuntu-slim
.github/workflows/push_release_notes_to_website.yml
+3
View File
@@ -12,6 +12,9 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ inputs.version }}
permissions:
contents: read
jobs:
push-release-notes-to-website:
.github/workflows/pypi_release.yml
+3
View File
@@ -10,6 +10,9 @@ on:
env:
HATCH_VERSION: "1.16.5"
permissions:
contents: read
jobs:
release-on-pypi:
runs-on: ubuntu-latest
.github/workflows/release.yml
+26
View File
@@ -13,8 +13,34 @@ concurrency:
group: release
cancel-in-progress: false
permissions:
contents: read
jobs:
authorize:
# Releasing acts as HaystackBot (a ruleset-bypass actor), so the ruleset
# cannot enforce *who* started the release. Gate on the triggering user's
# role instead: only maintainers/admins may run this workflow, even though
# workflow_dispatch itself is available to anyone with write access.
runs-on: ubuntu-slim
steps:
- name: Verify the triggering user may release
env:
GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
ACTOR: ${{ github.triggering_actor }}
run: |
ROLE=$(gh api "repos/${{ github.repository }}/collaborators/${ACTOR}/permission" --jq '.role_name')
echo "::notice::${ACTOR} has repository role '${ROLE}'"
case "$ROLE" in
admin|maintain)
echo "✅ ${ACTOR} is authorized to trigger a release." ;;
*)
echo "::error::${ACTOR} has role '${ROLE}'. A release can only be triggered by a user with the 'maintain' or 'admin' role."
exit 1 ;;
esac
parse-validate-version:
needs: ["authorize"]
runs-on: ubuntu-slim
outputs:
version: ${{ steps.parse-validate.outputs.version }}
.github/workflows/release_notes.yml
+3
View File
@@ -15,6 +15,9 @@ on:
- "!.github/**/*.py"
- "releasenotes/notes/*.yaml"
permissions:
contents: read
jobs:
reno:
runs-on: ubuntu-slim
.github/workflows/release_notes_skipper.yml
+3
View File
@@ -15,6 +15,9 @@ on:
- "!.github/**/*.py"
- "releasenotes/notes/*.yaml"
permissions:
contents: read
jobs:
reno:
runs-on: ubuntu-slim
.github/workflows/slow.yml
+4 -2
View File
@@ -1,4 +1,3 @@
# If you change this name also do it in ci_metrics.yml
name: Slow Integration Tests
# The workflow will always run, but the actual tests will only execute when:
@@ -34,6 +33,9 @@ on:
- labeled
- unlabeled
permissions:
contents: read
jobs:
check-if-changed:
# This job checks if the relevant files have been changed.
@@ -147,7 +149,7 @@ jobs:
- name: Run Tika
if: matrix.os == 'ubuntu-latest'
run: |
docker run -d -p 9998:9998 apache/tika:2.9.0.0
docker run -d -p 9998:9998 apache/tika:2.9.0.0@sha256:92d055a84e9efbedb5541fb38d635a44a16a875d88d1014c183af9eb93a09aaf
- name: Install Whisper dependencies
shell: bash
.github/workflows/stale.yml
+6
View File
@@ -3,9 +3,15 @@ on:
schedule:
- cron: '30 1 * * *'
permissions:
contents: read
jobs:
makestale:
runs-on: ubuntu-slim
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
with:
.github/workflows/tests.yml
+4 -2
View File
@@ -1,4 +1,3 @@
# If you change this name also do it in ci_metrics.yml
name: Tests
# The workflow will always run, but the actual tests will only execute when:
@@ -35,6 +34,9 @@ env:
PYTHON_VERSION: "3.10"
HATCH_VERSION: "1.16.5"
permissions:
contents: read
jobs:
check-if-changed:
# This job checks if the relevant files have been changed.
@@ -85,7 +87,7 @@ jobs:
run: hatch run fmt-check
- name: Check presence of license header
run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check
run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye:v6.5.1@sha256:0ebd72353053aa6d3f0cb44a6897786386347bd4636ffc9592f8e47c81ea2b9d check
check-imports:
needs: format
.github/workflows/v3_pypi_release.yml
+3
View File
@@ -6,6 +6,9 @@ on:
env:
HATCH_VERSION: "1.16.5"
permissions:
contents: read
jobs:
v3-release:
runs-on: ubuntu-latest
.github/workflows/workflows_linting.yml
+3
View File
@@ -5,6 +5,9 @@ on:
paths:
- ".github/workflows/**"
permissions:
contents: read
jobs:
lint-workflows:
runs-on: ubuntu-slim
README.md
+1 -1
View File
@@ -5,7 +5,7 @@
| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| CI/CD | [![Tests](https://github.com/deepset-ai/haystack/actions/workflows/tests.yml/badge.svg)](https://github.com/deepset-ai/haystack/actions/workflows/tests.yml) [![types - Mypy](https://img.shields.io/badge/types-Mypy-blue.svg)](https://github.com/python/mypy) [![Coverage badge](https://raw.githubusercontent.com/deepset-ai/haystack/python-coverage-comment-action-data/badge.svg)](https://htmlpreview.github.io/?https://github.com/deepset-ai/haystack/blob/python-coverage-comment-action-data/htmlcov/index.html) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) |
| Docs | [![Website](https://img.shields.io/website?label=documentation&up_message=online&url=https%3A%2F%2Fdocs.haystack.deepset.ai)](https://docs.haystack.deepset.ai) |
| Package | [![PyPI](https://img.shields.io/pypi/v/haystack-ai)](https://pypi.org/project/haystack-ai/) ![PyPI - Downloads](https://img.shields.io/pypi/dm/haystack-ai?color=blue&logo=pypi&logoColor=gold) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/haystack-ai?logo=python&logoColor=gold) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/haystack-ai.svg)](https://anaconda.org/conda-forge/haystack-ai) [![GitHub](https://img.shields.io/github/license/deepset-ai/haystack?color=blue)](LICENSE) [![License Compliance](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml/badge.svg)](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml) |
| Package | [![PyPI](https://img.shields.io/pypi/v/haystack-ai)](https://pypi.org/project/haystack-ai/) ![PyPI - Downloads](https://img.shields.io/pypi/dm/haystack-ai?color=blue&logo=pypi&logoColor=gold) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/haystack-ai?logo=python&logoColor=gold) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/haystack-ai.svg)](https://anaconda.org/conda-forge/haystack-ai) [![GitHub](https://img.shields.io/github/license/deepset-ai/haystack?color=blue)](LICENSE) [![License Compliance](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml/badge.svg)](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/13067/badge)](https://www.bestpractices.dev/projects/13067)|
| Meta | [![Discord](https://img.shields.io/discord/993534733298450452?logo=discord)](https://discord.com/invite/qZxjM4bAHU) [![Twitter Follow](https://img.shields.io/twitter/follow/haystack_ai)](https://twitter.com/haystack_ai) |
</div>
docker/Dockerfile.base
+1 -1
View File
@@ -9,7 +9,7 @@ ARG haystack_version
RUN apt-get update && \
apt-get install -y --no-install-recommends git
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
COPY --from=ghcr.io/astral-sh/uv:0.11.17@sha256:03bdc89bb9798628846e60c3a9ad19006c8c3c724ccd2985a33145c039a0577b /uv /uvx /bin/
# Shallow clone Haystack repo, we'll install from the local sources
RUN git clone --depth=1 --branch=${haystack_version} https://github.com/deepset-ai/haystack.git /opt/haystack
docker/docker-bake.hcl
+2 -2
View File
@@ -32,8 +32,8 @@ target "base" {
equal("${IS_STABLE}", "true") ? "${IMAGE_NAME}:stable" : ""
])}"
args = {
build_image = "python:3.12-slim"
base_image = "python:3.12-slim"
build_image = "python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203"
base_image = "python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203"
haystack_version = "${HAYSTACK_VERSION}"
}
platforms = ["linux/amd64", "linux/arm64"]
docs-website/docs/pipeline-components/extractors/llmmetadataextractor.mdx
+2 -2
View File
@@ -14,7 +14,7 @@ Extracts metadata from documents using a Large Language Model. The metadata is e
| | |
| --- | --- |
| **Most common position in a pipeline** | After [PreProcessors](../preprocessors.mdx) in an indexing pipeline |
| **Mandatory init variables** | `prompt`: The prompt to instruct the LLM on how to extract metadata from the document <br /> <br />`chat_generator`: A Chat Generator instance which represents the LLM configured to return a JSON object |
| **Mandatory init variables** | `prompt`: The prompt to instruct the LLM on how to extract metadata from the document. It must contain exactly one variable, called `document`. <br /> <br />`chat_generator`: A Chat Generator instance which represents the LLM configured to return a JSON object |
| **Mandatory run variables** | `documents`: A list of documents |
| **Output variables** | `documents`: A list of documents |
| **API reference** | [Extractors](/reference/extractors-api) |
@@ -27,7 +27,7 @@ Extracts metadata from documents using a Large Language Model. The metadata is e
The `LLMMetadataExtractor` extraction relies on an LLM and a prompt to perform the metadata extraction. At initialization time, it expects an LLM, a Haystack Generator, and a prompt describing the metadata extraction process.
The prompt should have a variable called `document` that will point to a single document in the list of documents. So, to access the content of the document, you can use `{{ document.content }}` in the prompt.
The prompt must have exactly one variable, called `document`, that points to a single document in the list of documents. So, to access the content of the document, you can use `{{ document.content }}` in the prompt. The component raises a `ValueError` at initialization if the prompt has no variables, more than one variable, or a variable with a different name.
At runtime, it expects a list of documents and will run the LLM on each document in the list, extracting metadata from the document. The metadata will be added to the document's metadata field.
docs-website/docs/pipeline-components/preprocessors.mdx
+1
View File
@@ -25,5 +25,6 @@ Use the PreProcessors to prepare your data normalize white spaces, remove header
| [MarkdownHeaderSplitter](preprocessors/markdownheadersplitter.mdx) | Splits documents at ATX-style Markdown headers (#), with optional secondary splitting. Preserves header hierarchy as metadata. |
| [PresidioDocumentCleaner](preprocessors/presidiodocumentcleaner.mdx) | Replaces PII in Document text with entity type placeholders using Microsoft Presidio. |
| [PresidioTextCleaner](preprocessors/presidiotextcleaner.mdx) | Replaces PII in plain strings — useful for sanitizing user queries before they reach an LLM. |
| [PythonCodeSplitter](preprocessors/pythoncodesplitter.mdx) | Splits Python source documents into syntax-aware chunks using AST units such as imports, functions, class headers, methods, and statements. |
| [RecursiveSplitter](preprocessors/recursivesplitter.mdx) | Splits text into smaller chunks, it does so by recursively applying a list of separators <br />to the text, applied in the order they are provided. |
| [TextCleaner](preprocessors/textcleaner.mdx) | Removes regexes, punctuation, and numbers, as well as converts text to lowercase. Useful to clean up text data before evaluation. |
docs-website/docs/pipeline-components/preprocessors/pythoncodesplitter.mdx
+179
View File
@@ -0,0 +1,179 @@
---
title: "PythonCodeSplitter"
id: pythoncodesplitter
slug: "/pythoncodesplitter"
description: "Split Python source documents into syntax-aware chunks using Python's AST, with metadata for line ranges, classes, decorators, and docstrings."
---
# PythonCodeSplitter
`PythonCodeSplitter` splits Python source code documents into syntax-aware chunks. It is designed for Python files and keeps code units such as imports, functions, classes, and methods together where possible.
<div className="key-value-table">
| | |
| --- | --- |
| **Most common position in a pipeline** | In indexing pipelines after [Converters](../converters.mdx), before [Embedders](../embedders.mdx) or [`DocumentWriter`](../writers/documentwriter.mdx) |
| **Mandatory run variables** | `documents`: A list of Python source code documents |
| **Output variables** | `documents`: A list of Python source code documents split into syntax-aware chunks |
| **API reference** | [PreProcessors](/reference/preprocessors-api) |
| **GitHub link** | https://github.com/deepset-ai/haystack/blob/main/haystack/components/preprocessors/python_code_splitter.py |
| **Package name** | `haystack-ai` |
</div>
## Overview
`PythonCodeSplitter` expects each input document's `content` to be valid Python source code. It parses the source with Python's `ast` module and creates ordered split units for:
- Module docstrings
- Consecutive import blocks
- Top-level functions
- Class headers
- Methods and nested classes
- Remaining top-level statements
The splitter merges these units in source order toward `max_effective_lines`. Effective lines are calculated from character length with `ceil(len(source) / expected_chars_per_line)`, so long lines count as more than one line.
Functions and methods are kept whole by the primary AST split. If one syntactic unit is larger than `oversized_factor * max_effective_lines`, the splitter falls back to a line-based secondary split using [`DocumentSplitter`](documentsplitter.mdx). This oversized fallback is the only case where chunks can overlap; the primary AST split does not add overlap.
By default, `preserve_class_definition=True`. When a chunk contains class members without the original class header, the splitter prefixes the bare class signature so the chunk still carries the class context.
If `strip_docstrings=True`, function, method, and class docstrings are removed from chunk content and stored in `meta["docstrings"]`. Module docstrings stay in the chunk content because they are their own top-level unit.
### Per-chunk metadata
Each output document carries the metadata below. All fields from the parent document's `meta` (except `split_id`) are also propagated.
| Field | Description |
| --- | --- |
| `source_id` | ID of the originating document |
| `split_id` | Sequential index of this chunk within its source document |
| `start_line` | First line of the chunk in the original source (1-indexed). Oversized secondary chunks keep the originating unit's range. |
| `end_line` | Last line of the chunk in the original source (1-indexed). Oversized secondary chunks keep the originating unit's range. |
| `unit_kinds` | List of syntactic unit kinds included in this chunk, such as `imports`, `function`, `class_header`, or `method` |
| `include_classes` | *(when applicable)* Ordered list of class names whose members appear in this chunk |
| `decorators` | *(when applicable)* Ordered list of decorator strings found on included functions, methods, or classes |
| `docstrings` | *(when `strip_docstrings=True`)* List of stripped docstring strings in source order |
| `secondary_split` | `True` if this chunk was produced by the oversized fallback splitter |
| `secondary_split_index` | Index of this piece within the secondary split sequence |
| `secondary_split_total` | Total number of pieces produced by the secondary split |
Documents with `None` content raise `ValueError`, documents with non-string content raise `TypeError`, and invalid Python source raises `SyntaxError`. Empty documents are skipped.
## Configuration
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `min_effective_lines` | `int` | `20` | Minimum effective lines per chunk. While a chunk is below this value, the splitter keeps merging in the next unit. |
| `max_effective_lines` | `int` | `100` | Target effective lines per chunk. Units are merged greedily toward this value. |
| `expected_chars_per_line` | `int` | `45` | Character count used to estimate effective lines via `ceil(len(source) / expected_chars_per_line)`. |
| `oversized_factor` | `int` | `3` | Multiplier that triggers secondary line-based splitting for oversized syntactic units. |
| `strip_docstrings` | `bool` | `False` | Moves function, method, and class docstrings from content into `meta["docstrings"]`. |
| `preserve_class_definition` | `bool` | `True` | Prefixes class signatures on chunks that contain class members without the class header. |
| `secondary_split_overlap` | `int` | `5` | Line overlap used only by the oversized secondary split. |
| `secondary_split_length` | `int \| None` | `None` | Line length for the oversized secondary split. Defaults to `max_effective_lines` when `None`. |
## Usage
### On its own
```python
import textwrap
from haystack import Document
from haystack.components.preprocessors import PythonCodeSplitter
source = textwrap.dedent(
'''
"""Math utilities."""
from math import pi
class Circle:
"""A circle."""
def __init__(self, radius: float) -> None:
self.radius = radius
def area(self) -> float:
return pi * self.radius * self.radius
'''
).lstrip()
splitter = PythonCodeSplitter(
min_effective_lines=4,
max_effective_lines=12,
strip_docstrings=True,
)
result = splitter.run(
documents=[Document(content=source, meta={"file_name": "geometry.py"})],
)
for chunk in result["documents"]:
print(chunk.meta["start_line"], chunk.meta["end_line"], chunk.meta.get("include_classes"))
```
### With docstring stripping for RAG
Set `strip_docstrings=True` when docstrings are verbose. The docstring text is moved out of the chunk content into `meta["docstrings"]`, keeping the stored chunk compact. Pass `meta_fields_to_embed=["docstrings"]` to your embedder so the docstring text still influences retrieval even though it is no longer in the chunk content.
```python
from haystack import Document
from haystack.components.preprocessors import PythonCodeSplitter
source = '''
"""Example module."""
from math import pi
class Circle:
"""A circle defined by its radius."""
def __init__(self, r: float) -> None:
"""Store the radius."""
self.r = r
def area(self) -> float:
"""Return the area of the circle."""
return pi * self.r * self.r
'''
splitter = PythonCodeSplitter(
min_effective_lines=20,
max_effective_lines=100,
strip_docstrings=True,
)
result = splitter.run(documents=[Document(content=source, meta={"file_name": "my_module.py"})])
for chunk in result["documents"]:
print(chunk.content)
print(chunk.meta.get("docstrings"))
```
### In a pipeline
This pipeline converts Python files to documents, splits them with `PythonCodeSplitter`, and writes the chunks to an in-memory document store.
```python
from pathlib import Path
from haystack import Pipeline
from haystack.components.converters.txt import TextFileToDocument
from haystack.components.preprocessors import PythonCodeSplitter
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
document_store = InMemoryDocumentStore()
p = Pipeline()
p.add_component("converter", TextFileToDocument())
p.add_component("splitter", PythonCodeSplitter(max_effective_lines=80))
p.add_component("writer", DocumentWriter(document_store=document_store))
p.connect("converter.documents", "splitter.documents")
p.connect("splitter.documents", "writer.documents")
files = list(Path("path/to/your/project").glob("**/*.py"))
p.run({"converter": {"sources": files}})
```
docs-website/docs/pipeline-components/retrievers.mdx
+1
View File
@@ -165,6 +165,7 @@ For details on how to initialize and use a Retriever in a pipeline, see the docu
| [ChromaQueryTextRetriever](retrievers/chromaqueryretriever.mdx) | A Retriever compatible with the Chroma Document Store that uses the Chroma query API. |
| [ElasticsearchEmbeddingRetriever](retrievers/elasticsearchembeddingretriever.mdx) | An embedding-based Retriever compatible with the Elasticsearch Document Store. |
| [ElasticsearchBM25Retriever](retrievers/elasticsearchbm25retriever.mdx) | A keyword-based Retriever that fetches Documents matching a query from the Elasticsearch Document Store. |
| [ElasticsearchSQLRetriever](retrievers/elasticsearchsqlretriever.mdx) | Executes raw Elasticsearch SQL queries against an Elasticsearch Document Store and returns the raw JSON response. |
| [InMemoryBM25Retriever](retrievers/inmemorybm25retriever.mdx) | A keyword-based Retriever compatible with the InMemoryDocumentStore. |
| [InMemoryEmbeddingRetriever](retrievers/inmemoryembeddingretriever.mdx) | An embedding-based Retriever compatible with the InMemoryDocumentStore. |
| [FilterRetriever](retrievers/filterretriever.mdx) | A special Retriever to be used with any Document Store to get the Documents that match specific filters. |
docs-website/docs/pipeline-components/retrievers/elasticsearchsqlretriever.mdx
+113
View File
@@ -0,0 +1,113 @@
---
title: ElasticsearchSQLRetriever
id: elasticsearchsqlretriever
slug: /elasticsearchsqlretriever
description: Executes raw Elasticsearch SQL queries against an Elasticsearch Document Store and returns the raw JSON response.
---
# ElasticsearchSQLRetriever
Executes raw Elasticsearch SQL queries against an Elasticsearch Document Store and returns the raw JSON response.
| | |
| --------------------------------------- | ------------------------------------------------------------------------------------------------ |
| **Most common position in a pipeline** | Standalone, or anywhere you need to fetch metadata, aggregations, or other structured data |
| **Mandatory init variables** | `document_store`: An instance of `ElasticsearchDocumentStore` |
| **Mandatory run variables** | `query`: An Elasticsearch SQL query string |
| **Output variables** | `result`: A dictionary with the raw JSON response from the Elasticsearch SQL API |
| **API reference** | [Elasticsearch](https://docs.haystack.deepset.ai/reference/integrations-elasticsearch) |
| **GitHub link** | https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch |
| **Package name** | `elasticsearch-haystack` |
## Overview
`ElasticsearchSQLRetriever` lets you run [Elasticsearch SQL](https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-sql.html) queries directly against an `ElasticsearchDocumentStore`. Instead of matching a query against documents like the `ElasticsearchBM25Retriever` or `ElasticsearchEmbeddingRetriever`, it executes a SQL statement and returns the **raw JSON response** from the Elasticsearch SQL API.
This is useful when you need structured access to your index at runtime, for example to fetch specific fields, filter on metadata, or compute aggregations such as counts and averages.
Unlike the other Elasticsearch retrievers, this component does not return a list of `Document` objects. The output is a single `result` dictionary, where `result["result"]` holds the raw Elasticsearch response. For a typical query, the response contains:
- `result["result"]["columns"]`: metadata describing each returned column.
- `result["result"]["rows"]`: the data rows.
The component accepts two optional parameters at initialization:
- `raise_on_failure`: if `True` (the default), an exception is raised when the SQL API call fails. If `False`, the error is logged as a warning and an empty dictionary is returned.
- `fetch_size`: the number of results to fetch per page. If not set, the default fetch size configured in Elasticsearch is used.
## Installation
Install Elasticsearch and then start an instance. Haystack supports Elasticsearch 8.
If you have Docker set up, we recommend pulling the Docker image and running it.
```bash
docker pull docker.elastic.co/elasticsearch/elasticsearch:8.11.1
docker run -p 9200:9200 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" -e "xpack.security.enabled=false" elasticsearch:8.11.1
```
As an alternative, you can go to [Elasticsearch integration GitHub](https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch) and start a Docker container running Elasticsearch using the provided `docker-compose.yml`:
```bash
docker compose up
```
Once you have a running Elasticsearch instance, install the `elasticsearch-haystack` integration:
```bash
pip install elasticsearch-haystack
```
## Usage
### On its own
Write a few documents to an index, then run a SQL query against it. The example below selects the `content` field from the index and reads the returned columns and rows:
```python
from haystack import Document
from haystack_integrations.components.retrievers.elasticsearch import (
ElasticsearchSQLRetriever,
)
from haystack_integrations.document_stores.elasticsearch import (
ElasticsearchDocumentStore,
)
from haystack.document_stores.types import DuplicatePolicy
document_store = ElasticsearchDocumentStore(hosts="http://localhost:9200/", index="my_index")
documents = [
Document(content="There are over 7,000 languages spoken around the world today."),
Document(
content="Elephants have been observed to behave in a way that indicates a high level of self-awareness, such as recognizing themselves in mirrors.",
),
Document(
content="In certain parts of the world, like the Maldives, Puerto Rico, and San Diego, you can witness the phenomenon of bioluminescent waves.",
),
]
# DuplicatePolicy.SKIP is optional, but useful to run the script multiple times without throwing errors
document_store.write_documents(documents=documents, policy=DuplicatePolicy.SKIP)
retriever = ElasticsearchSQLRetriever(document_store=document_store)
output = retriever.run(query='SELECT content FROM "my_index" LIMIT 10')
result = output["result"]
print(result["columns"]) # column metadata, e.g. [{"name": "content", "type": "text"}]
for row in result["rows"]:
print(row)
```
### Running an aggregation query
Because the component returns the raw SQL response, you can use it for aggregations that the document-based retrievers don't support, such as counting documents:
```python
retriever = ElasticsearchSQLRetriever(document_store=document_store)
output = retriever.run(query='SELECT COUNT(*) AS doc_count FROM "my_index"')
result = output["result"]
print(result["rows"]) # e.g. [[3]]
```
To avoid raising an exception on a malformed or failing query, initialize the component with `raise_on_failure=False`. In that case, a failed query logs a warning and returns an empty dictionary instead.
docs-website/docusaurus.config.js
+4 -4
View File
@@ -79,12 +79,12 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
beforeDefaultRemarkPlugins: [require('./src/remark/versionedReferenceLinks')],
versions: {
current: {
label: '2.30-unstable',
label: '2.31-unstable',
path: 'next',
banner: 'unreleased',
},
},
lastVersion: '2.29',
lastVersion: '2.30',
},
theme: {
customCss: require.resolve('./src/css/custom.css'),
@@ -132,12 +132,12 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
exclude: ['**/_templates/**'],
versions: {
current: {
label: '2.30-unstable',
label: '2.31-unstable',
path: 'next',
banner: 'unreleased',
},
},
lastVersion: '2.29',
lastVersion: '2.30',
},
],
[
docs-website/reference/haystack-api/extractors_api.md
+5 -3
View File
@@ -170,8 +170,8 @@ Extracts metadata from documents using a Large Language Model (LLM).
The metadata is extracted by providing a prompt to an LLM that generates the metadata.
This component expects as input a list of documents and a prompt. The prompt should have a variable called
`document` that will point to a single document in the list of documents. So to access the content of the document,
This component expects as input a list of documents and a prompt. The prompt must have exactly one variable, called
`document`, that points to a single document in the list of documents. So to access the content of the document,
you can use `{{ document.content }}` in the prompt.
The component will run the LLM on each document in the list and extract metadata from the document. The metadata
@@ -304,7 +304,9 @@ Initializes the LLMMetadataExtractor.
**Parameters:**
- **prompt** (<code>str</code>) – The prompt to be used for the LLM.
- **prompt** (<code>str</code>) – The prompt to be used for the LLM. It must contain exactly one variable, called `document`,
which points to a single document in the list of documents. For example, to access the content of the
document, use `{{ document.content }}` in the prompt.
- **chat_generator** (<code>ChatGenerator</code>) – a ChatGenerator instance which represents the LLM. In order for the component to work,
the LLM should be configured to return a JSON object. For example, when using the OpenAIChatGenerator, you
should pass `{"response_format": {"type": "json_object"}}` in the `generation_kwargs`.
docs-website/reference/haystack-api/utils_api.md
+4
View File
@@ -917,6 +917,10 @@ For example, given a page_range=['1-3', '5', '8', '10-12'] the function will ret
- <code>list\[int\]</code> – An expanded list of page integers
**Raises:**
- <code>ValueError</code> – If any element is not a valid integer or a range string in the format `'start-end'`.
### expit
```python
docs-website/reference/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.18/integrations-api/pgvector.md
+19 -29
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -212,17 +214,6 @@ result = retriever.run(query="languages")
assert res['retriever']['documents'][0].content == "There are over 7,000 languages spoken around the world today."
#### __init__
```python
@@ -232,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -350,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -440,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -731,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -753,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -782,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -825,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -847,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.18/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.19/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.19/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.20/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.20/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.21/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.21/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.22/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.22/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.23/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.23/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.24/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.24/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.25/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.25/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.26/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.26/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.27/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.27/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.28/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.28/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.29/integrations-api/pgvector.md
+19 -18
View File
@@ -65,9 +65,11 @@ __init__(
Literal["cosine_similarity", "inner_product", "l2_distance"] | None
) = None,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
```
Initialize the PgvectorEmbeddingRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -221,9 +223,11 @@ __init__(
filters: dict[str, Any] | None = None,
top_k: int = 10,
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE
)
) -> None
````
Initialize the PgvectorKeywordRetriever.
**Parameters:**
- **document_store** (<code>PgvectorDocumentStore</code>) – An instance of `PgvectorDocumentStore`.
@@ -339,10 +343,11 @@ __init__(
hnsw_index_name: str = "haystack_hnsw_index",
hnsw_ef_search: int | None = None,
keyword_index_name: str = "haystack_keyword_index"
)
) -> None
```
Creates a new PgvectorDocumentStore instance.
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
A specific table to store Haystack documents will be created if it doesn't exist yet.
@@ -429,17 +434,18 @@ Deserializes the component from a dictionary.
#### delete_table
```python
delete_table()
delete_table() -> None
```
Deletes the table used to store Haystack documents.
The name of the schema (`schema_name`) and the name of the table (`table_name`)
are defined when initializing the `PgvectorDocumentStore`.
#### delete_table_async
```python
delete_table_async()
delete_table_async() -> None
```
Async method to delete the table used to store Haystack documents.
@@ -720,8 +726,9 @@ count_unique_metadata_by_filter(
) -> dict[str, int]
```
Returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -742,8 +749,9 @@ count_unique_metadata_by_filter_async(
) -> dict[str, int]
```
Asynchronously returns the count of unique values for each specified metadata field,
considering only documents that match the provided filters.
Asynchronously returns the count of unique values for each specified metadata field.
Considers only documents that match the provided filters.
**Parameters:**
@@ -771,7 +779,6 @@ Example return:
```python
{
'content': {'type': 'text'},
'category': {'type': 'text'},
'status': {'type': 'text'},
'priority': {'type': 'integer'},
@@ -814,10 +821,7 @@ Returns the minimum and maximum values for a given metadata field.
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_min_max_async
@@ -836,10 +840,7 @@ Asynchronously returns the minimum and maximum values for a given metadata field
- <code>dict\[str, Any\]</code> – A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
For numeric fields (integer, real), returns numeric min/max.
For text fields, returns lexicographic min/max based on database collation.
**Raises:**
- <code>ValueError</code> – If the field doesn't exist or has no values.
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
#### get_metadata_field_unique_values
docs-website/reference_versioned_docs/version-2.29/integrations-api/supabase.md
+104
View File
@@ -6,6 +6,110 @@ slug: "/integrations-supabase"
---
## haystack_integrations.components.downloaders.supabase.supabase_bucket_downloader
### SupabaseBucketDownloader
Downloads files from a Supabase Storage bucket and returns them as ByteStream objects.
Files are downloaded in-memory and returned as `ByteStream` objects ready for further
processing in indexing pipelines (e.g. passing to a `DocumentConverter`).
Example usage:
```python
from haystack_integrations.components.downloaders.supabase import SupabaseBucketDownloader
from haystack.utils import Secret
downloader = SupabaseBucketDownloader(
supabase_url="https://<project-ref>.supabase.co",
supabase_key=Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name="my-documents",
)
result = downloader.run(sources=["reports/report.pdf", "data/notes.txt"])
streams = result["streams"]
```
#### __init__
```python
__init__(
*,
supabase_url: str,
supabase_key: Secret = Secret.from_env_var("SUPABASE_SERVICE_KEY"),
bucket_name: str,
file_extensions: list[str] | None = None
) -> None
```
Creates a new SupabaseBucketDownloader instance.
**Parameters:**
- **supabase_url** (<code>str</code>) – The URL of your Supabase project, e.g. `https://<project-ref>.supabase.co`.
- **supabase_key** (<code>Secret</code>) – The Supabase API key used to authenticate requests. Defaults to the
`SUPABASE_SERVICE_KEY` environment variable. Use the service role key for private buckets.
- **bucket_name** (<code>str</code>) – The name of the Supabase Storage bucket to download files from.
- **file_extensions** (<code>list\[str\] | None</code>) – Optional list of file extensions to filter downloads (e.g. `[".pdf", ".txt"]`).
If `None`, all files are downloaded. Extensions are matched case-insensitively.
#### warm_up
```python
warm_up() -> None
```
Initializes the Supabase client.
Called automatically on the first run(), or can be called explicitly in a pipeline.
#### run
```python
run(sources: list[str]) -> dict[str, list[ByteStream]]
```
Downloads files from the Supabase Storage bucket.
**Parameters:**
- **sources** (<code>list\[str\]</code>) – List of file paths within the bucket to download,
e.g. `["folder/file.pdf", "notes.txt"]`.
**Returns:**
- <code>dict\[str, list\[ByteStream\]\]</code> – A dictionary with:
- `streams`: list of `ByteStream` objects, one per successfully downloaded file.
Each `ByteStream` has `meta["file_path"]` and `meta["bucket_name"]` set.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> SupabaseBucketDownloader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>SupabaseBucketDownloader</code> – Deserialized component.
## haystack_integrations.components.retrievers.supabase.embedding_retriever
### SupabasePgvectorEmbeddingRetriever
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_agents_api.md
+475
View File
@@ -0,0 +1,475 @@
---
title: "Agents"
id: experimental-agents-api
description: "Tool-using agents with provider-agnostic chat model support."
slug: "/experimental-agents-api"
---
<a id="haystack_experimental.components.agents.agent"></a>
## Module haystack\_experimental.components.agents.agent
<a id="haystack_experimental.components.agents.agent.Agent"></a>
### Agent
A Haystack component that implements a tool-using agent with provider-agnostic chat model support.
NOTE: This class extends Haystack's Agent component to add support for human-in-the-loop confirmation strategies.
The component processes messages and executes tools until an exit condition is met.
The exit condition can be triggered either by a direct text response or by invoking a specific designated tool.
Multiple exit conditions can be specified.
When you call an Agent without tools, it acts as a ChatGenerator, produces one response, then exits.
### Usage example
```python
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.dataclasses import ChatMessage
from haystack.tools.tool import Tool
from haystack_experimental.components.agents import Agent
from haystack_experimental.components.agents.human_in_the_loop import (
HumanInTheLoopStrategy,
AlwaysAskPolicy,
NeverAskPolicy,
SimpleConsoleUI,
)
calculator_tool = Tool(name="calculator", description="A tool for performing mathematical calculations.", ...)
search_tool = Tool(name="search", description="A tool for searching the web.", ...)
agent = Agent(
chat_generator=OpenAIChatGenerator(),
tools=[calculator_tool, search_tool],
confirmation_strategies={
calculator_tool.name: HumanInTheLoopStrategy(
confirmation_policy=NeverAskPolicy(), confirmation_ui=SimpleConsoleUI()
),
search_tool.name: HumanInTheLoopStrategy(
confirmation_policy=AlwaysAskPolicy(), confirmation_ui=SimpleConsoleUI()
),
},
)
# Run the agent
result = agent.run(
messages=[ChatMessage.from_user("Find information about Haystack")]
)
assert "messages" in result # Contains conversation history
```
<a id="haystack_experimental.components.agents.agent.Agent.__init__"></a>
#### Agent.\_\_init\_\_
```python
def __init__(*,
chat_generator: ChatGenerator,
tools: ToolsType | None = None,
system_prompt: str | None = None,
exit_conditions: list[str] | None = None,
state_schema: dict[str, Any] | None = None,
max_agent_steps: int = 100,
streaming_callback: StreamingCallbackT | None = None,
raise_on_tool_invocation_failure: bool = False,
confirmation_strategies: dict[str, ConfirmationStrategy]
| None = None,
tool_invoker_kwargs: dict[str, Any] | None = None,
chat_message_store: ChatMessageStore | None = None,
memory_store: MemoryStore | None = None) -> None
```
Initialize the agent component.
**Arguments**:
- `chat_generator`: An instance of the chat generator that your agent should use. It must support tools.
- `tools`: List of Tool objects or a Toolset that the agent can use.
- `system_prompt`: System prompt for the agent.
- `exit_conditions`: List of conditions that will cause the agent to return.
Can include "text" if the agent should return when it generates a message without tool calls,
or tool names that will cause the agent to return once the tool was executed. Defaults to ["text"].
- `state_schema`: The schema for the runtime state used by the tools.
- `max_agent_steps`: Maximum number of steps the agent will run before stopping. Defaults to 100.
If the agent exceeds this number of steps, it will stop and return the current state.
- `streaming_callback`: A callback that will be invoked when a response is streamed from the LLM.
The same callback can be configured to emit tool results when a tool is called.
- `raise_on_tool_invocation_failure`: Should the agent raise an exception when a tool invocation fails?
If set to False, the exception will be turned into a chat message and passed to the LLM.
- `tool_invoker_kwargs`: Additional keyword arguments to pass to the ToolInvoker.
- `chat_message_store`: The ChatMessageStore that the agent can use to store
and retrieve chat messages history.
- `memory_store`: The memory store that the agent can use to store and retrieve memories.
**Raises**:
- `TypeError`: If the chat_generator does not support tools parameter in its run method.
- `ValueError`: If the exit_conditions are not valid.
<a id="haystack_experimental.components.agents.agent.Agent.run"></a>
#### Agent.run
```python
def run(messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
*,
generation_kwargs: dict[str, Any] | None = None,
break_point: AgentBreakpoint | None = None,
snapshot: AgentSnapshot | None = None,
system_prompt: str | None = None,
tools: ToolsType | list[str] | None = None,
confirmation_strategy_context: dict[str, Any] | None = None,
chat_message_store_kwargs: dict[str, Any] | None = None,
memory_store_kwargs: dict[str, Any] | None = None,
**kwargs: Any) -> dict[str, Any]
```
Process messages and execute tools until an exit condition is met.
**Arguments**:
- `messages`: List of Haystack ChatMessage objects to process.
- `streaming_callback`: A callback that will be invoked when a response is streamed from the LLM.
The same callback can be configured to emit tool results when a tool is called.
- `generation_kwargs`: Additional keyword arguments for LLM. These parameters will
override the parameters passed during component initialization.
- `break_point`: An AgentBreakpoint, can be a Breakpoint for the "chat_generator" or a ToolBreakpoint
for "tool_invoker".
- `snapshot`: A dictionary containing a snapshot of a previously saved agent execution. The snapshot contains
the relevant information to restart the Agent execution from where it left off.
- `system_prompt`: System prompt for the agent. If provided, it overrides the default system prompt.
- `tools`: Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
When passing tool names, tools are selected from the Agent's originally configured tools.
- `confirmation_strategy_context`: Optional dictionary for passing request-scoped resources
to confirmation strategies. Useful in web/server environments to provide per-request
objects (e.g., WebSocket connections, async queues, Redis pub/sub clients) that strategies
can use for non-blocking user interaction.
- `chat_message_store_kwargs`: Optional dictionary of keyword arguments to pass to the ChatMessageStore.
For example, it can include the `chat_history_id` and `last_k` parameters for retrieving chat history.
- `memory_store_kwargs`: Optional dictionary of keyword arguments to pass to the MemoryStore.
It can include:
- `user_id`: The user ID to search and add memories from.
- `run_id`: The run ID to search and add memories from.
- `agent_id`: The agent ID to search and add memories from.
- `search_criteria`: A dictionary of containing kwargs for the `search_memories` method.
This can include:
- `filters`: A dictionary of filters to search for memories.
- `query`: The query to search for memories.
Note: If you pass this, the user query passed to the agent will be
ignored for memory retrieval.
- `top_k`: The number of memories to return.
- `include_memory_metadata`: Whether to include the memory metadata in the ChatMessage.
- `kwargs`: Additional data to pass to the State schema used by the Agent.
The keys must match the schema defined in the Agent's `state_schema`.
**Raises**:
- `RuntimeError`: If the Agent component wasn't warmed up before calling `run()`.
- `BreakpointException`: If an agent breakpoint is triggered.
**Returns**:
A dictionary with the following keys:
- "messages": List of all messages exchanged during the agent's run.
- "last_message": The last message exchanged during the agent's run.
- Any additional keys defined in the `state_schema`.
<a id="haystack_experimental.components.agents.agent.Agent.run_async"></a>
#### Agent.run\_async
```python
async def run_async(messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
*,
generation_kwargs: dict[str, Any] | None = None,
break_point: AgentBreakpoint | None = None,
snapshot: AgentSnapshot | None = None,
system_prompt: str | None = None,
tools: ToolsType | list[str] | None = None,
confirmation_strategy_context: dict[str, Any]
| None = None,
chat_message_store_kwargs: dict[str, Any] | None = None,
memory_store_kwargs: dict[str, Any] | None = None,
**kwargs: Any) -> dict[str, Any]
```
Asynchronously process messages and execute tools until the exit condition is met.
This is the asynchronous version of the `run` method. It follows the same logic but uses
asynchronous operations where possible, such as calling the `run_async` method of the ChatGenerator
if available.
**Arguments**:
- `messages`: List of Haystack ChatMessage objects to process.
- `streaming_callback`: An asynchronous callback that will be invoked when a response is streamed from the
LLM. The same callback can be configured to emit tool results when a tool is called.
- `generation_kwargs`: Additional keyword arguments for LLM. These parameters will
override the parameters passed during component initialization.
- `break_point`: An AgentBreakpoint, can be a Breakpoint for the "chat_generator" or a ToolBreakpoint
for "tool_invoker".
- `snapshot`: A dictionary containing a snapshot of a previously saved agent execution. The snapshot contains
the relevant information to restart the Agent execution from where it left off.
- `system_prompt`: System prompt for the agent. If provided, it overrides the default system prompt.
- `tools`: Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
- `confirmation_strategy_context`: Optional dictionary for passing request-scoped resources
to confirmation strategies. Useful in web/server environments to provide per-request
objects (e.g., WebSocket connections, async queues, Redis pub/sub clients) that strategies
can use for non-blocking user interaction.
- `chat_message_store_kwargs`: Optional dictionary of keyword arguments to pass to the ChatMessageStore.
For example, it can include the `chat_history_id` and `last_k` parameters for retrieving chat history.
- `kwargs`: Additional data to pass to the State schema used by the Agent.
- `memory_store_kwargs`: Optional dictionary of keyword arguments to pass to the MemoryStore.
It can include:
- `user_id`: The user ID to search and add memories from.
- `run_id`: The run ID to search and add memories from.
- `agent_id`: The agent ID to search and add memories from.
- `search_criteria`: A dictionary of containing kwargs for the `search_memories` method.
This can include:
- `filters`: A dictionary of filters to search for memories.
- `query`: The query to search for memories.
Note: If you pass this, the user query passed to the agent will be
ignored for memory retrieval.
- `top_k`: The number of memories to return.
- `include_memory_metadata`: Whether to include the memory metadata in the ChatMessage.
- `kwargs`: Additional data to pass to the State schema used by the Agent.
The keys must match the schema defined in the Agent's `state_schema`.
**Raises**:
- `RuntimeError`: If the Agent component wasn't warmed up before calling `run_async()`.
- `BreakpointException`: If an agent breakpoint is triggered.
**Returns**:
A dictionary with the following keys:
- "messages": List of all messages exchanged during the agent's run.
- "last_message": The last message exchanged during the agent's run.
- Any additional keys defined in the `state_schema`.
<a id="haystack_experimental.components.agents.agent.Agent.to_dict"></a>
#### Agent.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serialize the component to a dictionary.
**Returns**:
Dictionary with serialized data
<a id="haystack_experimental.components.agents.agent.Agent.from_dict"></a>
#### Agent.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Agent"
```
Deserialize the agent from a dictionary.
**Arguments**:
- `data`: Dictionary to deserialize from
**Returns**:
Deserialized agent
<a id="haystack_experimental.components.agents.human_in_the_loop.breakpoint"></a>
## Module haystack\_experimental.components.agents.human\_in\_the\_loop.breakpoint
<a id="haystack_experimental.components.agents.human_in_the_loop.breakpoint.get_tool_calls_and_descriptions_from_snapshot"></a>
#### get\_tool\_calls\_and\_descriptions\_from\_snapshot
```python
def get_tool_calls_and_descriptions_from_snapshot(
agent_snapshot: AgentSnapshot,
breakpoint_tool_only: bool = True
) -> tuple[list[dict], dict[str, str]]
```
Extract tool calls and tool descriptions from an AgentSnapshot.
By default, only the tool call that caused the breakpoint is processed and its arguments are reconstructed.
This is useful for scenarios where you want to present the relevant tool call and its description
to a human for confirmation before execution.
**Arguments**:
- `agent_snapshot`: The AgentSnapshot from which to extract tool calls and descriptions.
- `breakpoint_tool_only`: If True, only the tool call that caused the breakpoint is returned. If False, all tool
calls are returned.
**Returns**:
A tuple containing a list of tool call dictionaries and a dictionary of tool descriptions
<a id="haystack_experimental.components.agents.human_in_the_loop.errors"></a>
## Module haystack\_experimental.components.agents.human\_in\_the\_loop.errors
<a id="haystack_experimental.components.agents.human_in_the_loop.errors.HITLBreakpointException"></a>
### HITLBreakpointException
Exception raised when a tool execution is paused by a ConfirmationStrategy (e.g. BreakpointConfirmationStrategy).
<a id="haystack_experimental.components.agents.human_in_the_loop.errors.HITLBreakpointException.__init__"></a>
#### HITLBreakpointException.\_\_init\_\_
```python
def __init__(message: str,
tool_name: str,
snapshot_file_path: str,
tool_call_id: str | None = None) -> None
```
Initialize the HITLBreakpointException.
**Arguments**:
- `message`: The exception message.
- `tool_name`: The name of the tool whose execution is paused.
- `snapshot_file_path`: The file path to the saved pipeline snapshot.
- `tool_call_id`: Optional unique identifier for the tool call. This can be used to track and correlate
the decision with a specific tool invocation.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies"></a>
## Module haystack\_experimental.components.agents.human\_in\_the\_loop.strategies
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy"></a>
### BreakpointConfirmationStrategy
Confirmation strategy that raises a tool breakpoint exception to pause execution and gather user feedback.
This strategy is designed for scenarios where immediate user interaction is not possible.
When a tool execution requires confirmation, it raises an `HITLBreakpointException`, which is caught by the Agent.
The Agent then serialize its current state, including the tool call details. This information can then be used to
notify a user to review and confirm the tool execution.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy.__init__"></a>
#### BreakpointConfirmationStrategy.\_\_init\_\_
```python
def __init__(snapshot_file_path: str) -> None
```
Initialize the BreakpointConfirmationStrategy.
**Arguments**:
- `snapshot_file_path`: The path to the directory that the snapshot should be saved.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy.run"></a>
#### BreakpointConfirmationStrategy.run
```python
def run(
*,
tool_name: str,
tool_description: str,
tool_params: dict[str, Any],
tool_call_id: str | None = None,
confirmation_strategy_context: dict[str, Any] | None = None
) -> ToolExecutionDecision
```
Run the breakpoint confirmation strategy for a given tool and its parameters.
**Arguments**:
- `tool_name`: The name of the tool to be executed.
- `tool_description`: The description of the tool.
- `tool_params`: The parameters to be passed to the tool.
- `tool_call_id`: Optional unique identifier for the tool call. This can be used to track and correlate the decision with a
specific tool invocation.
- `confirmation_strategy_context`: Optional dictionary for passing request-scoped resources. Not used by this strategy but included for
interface compatibility.
**Raises**:
- `HITLBreakpointException`: Always raises an `HITLBreakpointException` exception to signal that user confirmation is required.
**Returns**:
This method does not return; it always raises an exception.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy.run_async"></a>
#### BreakpointConfirmationStrategy.run\_async
```python
async def run_async(
*,
tool_name: str,
tool_description: str,
tool_params: dict[str, Any],
tool_call_id: str | None = None,
confirmation_strategy_context: dict[str, Any] | None = None
) -> ToolExecutionDecision
```
Async version of run. Calls the sync run() method.
**Arguments**:
- `tool_name`: The name of the tool to be executed.
- `tool_description`: The description of the tool.
- `tool_params`: The parameters to be passed to the tool.
- `tool_call_id`: Optional unique identifier for the tool call.
- `confirmation_strategy_context`: Optional dictionary for passing request-scoped resources.
**Raises**:
- `HITLBreakpointException`: Always raises an `HITLBreakpointException` exception to signal that user confirmation is required.
**Returns**:
This method does not return; it always raises an exception.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy.to_dict"></a>
#### BreakpointConfirmationStrategy.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serializes the BreakpointConfirmationStrategy to a dictionary.
<a id="haystack_experimental.components.agents.human_in_the_loop.strategies.BreakpointConfirmationStrategy.from_dict"></a>
#### BreakpointConfirmationStrategy.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "BreakpointConfirmationStrategy"
```
Deserializes the BreakpointConfirmationStrategy from a dictionary.
**Arguments**:
- `data`: Dictionary to deserialize from.
**Returns**:
Deserialized BreakpointConfirmationStrategy.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_chatmessage_store_api.md
+181
View File
@@ -0,0 +1,181 @@
---
title: "ChatMessage Store"
id: experimental-chatmessage-store-api
description: "Storage for the chat messages."
slug: "/experimental-chatmessage-store-api"
---
<a id="haystack_experimental.chat_message_stores.in_memory"></a>
## Module haystack\_experimental.chat\_message\_stores.in\_memory
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore"></a>
### InMemoryChatMessageStore
Stores chat messages in-memory.
The `chat_history_id` parameter is used as a unique identifier for each conversation or chat session.
It acts as a namespace that isolates messages from different sessions. Each `chat_history_id` value corresponds to a
separate list of `ChatMessage` objects stored in memory.
Typical usage involves providing a unique `chat_history_id` (for example, a session ID or conversation ID)
whenever you write, read, or delete messages. This ensures that chat messages from different
conversations do not overlap.
Usage example:
```python
from haystack.dataclasses import ChatMessage
from haystack_experimental.chat_message_stores.in_memory import InMemoryChatMessageStore
message_store = InMemoryChatMessageStore()
messages = [
ChatMessage.from_assistant("Hello, how can I help you?"),
ChatMessage.from_user("Hi, I have a question about Python. What is a Protocol?"),
]
message_store.write_messages(chat_history_id="user_456_session_123", messages=messages)
retrieved_messages = message_store.retrieve_messages(chat_history_id="user_456_session_123")
print(retrieved_messages)
```
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.__init__"></a>
#### InMemoryChatMessageStore.\_\_init\_\_
```python
def __init__(skip_system_messages: bool = True,
last_k: int | None = 10) -> None
```
Create an InMemoryChatMessageStore.
**Arguments**:
- `skip_system_messages`: Whether to skip storing system messages. Defaults to True.
- `last_k`: The number of last messages to retrieve. Defaults to 10 messages if not specified.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.to_dict"></a>
#### InMemoryChatMessageStore.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns**:
Dictionary with serialized data.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.from_dict"></a>
#### InMemoryChatMessageStore.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "InMemoryChatMessageStore"
```
Deserializes the component from a dictionary.
**Arguments**:
- `data`: The dictionary to deserialize from.
**Returns**:
The deserialized component.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.count_messages"></a>
#### InMemoryChatMessageStore.count\_messages
```python
def count_messages(chat_history_id: str) -> int
```
Returns the number of chat messages stored in this store.
**Arguments**:
- `chat_history_id`: The chat history id for which to count messages.
**Returns**:
The number of messages.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.write_messages"></a>
#### InMemoryChatMessageStore.write\_messages
```python
def write_messages(chat_history_id: str, messages: list[ChatMessage]) -> int
```
Writes chat messages to the ChatMessageStore.
**Arguments**:
- `chat_history_id`: The chat history id under which to store the messages.
- `messages`: A list of ChatMessages to write.
**Raises**:
- `ValueError`: If messages is not a list of ChatMessages.
**Returns**:
The number of messages written.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.retrieve_messages"></a>
#### InMemoryChatMessageStore.retrieve\_messages
```python
def retrieve_messages(chat_history_id: str,
last_k: int | None = None) -> list[ChatMessage]
```
Retrieves all stored chat messages.
**Arguments**:
- `chat_history_id`: The chat history id from which to retrieve messages.
- `last_k`: The number of last messages to retrieve. If unspecified, the last_k parameter passed
to the constructor will be used.
**Raises**:
- `ValueError`: If last_k is not None and is less than 0.
**Returns**:
A list of chat messages.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.delete_messages"></a>
#### InMemoryChatMessageStore.delete\_messages
```python
def delete_messages(chat_history_id: str) -> None
```
Deletes all stored chat messages.
**Arguments**:
- `chat_history_id`: The chat history id from which to delete messages.
<a id="haystack_experimental.chat_message_stores.in_memory.InMemoryChatMessageStore.delete_all_messages"></a>
#### InMemoryChatMessageStore.delete\_all\_messages
```python
def delete_all_messages() -> None
```
Deletes all stored chat messages from all chat history ids.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_generators_api.md
+152
View File
@@ -0,0 +1,152 @@
---
title: "Generators"
id: experimental-generators-api
description: "Enables text generation using LLMs."
slug: "/experimental-generators-api"
---
<a id="haystack_experimental.components.generators.chat.openai"></a>
## Module haystack\_experimental.components.generators.chat.openai
<a id="haystack_experimental.components.generators.chat.openai.OpenAIChatGenerator"></a>
### OpenAIChatGenerator
An OpenAI chat-based text generator component that supports hallucination risk scoring.
This is based on the paper
[LLMs are Bayesian, in Expectation, not in Realization](https://arxiv.org/abs/2507.11768).
## Usage Example:
```python
from haystack.dataclasses import ChatMessage
from haystack_experimental.utils.hallucination_risk_calculator.dataclasses import HallucinationScoreConfig
from haystack_experimental.components.generators.chat.openai import OpenAIChatGenerator
# Evidence-based Example
llm = OpenAIChatGenerator(model="gpt-4o")
rag_result = llm.run(
messages=[
ChatMessage.from_user(
text="Task: Answer strictly based on the evidence provided below.
"
"Question: Who won the Nobel Prize in Physics in 2019?
"
"Evidence:
"
"- Nobel Prize press release (2019): James Peebles (1/2); Michel Mayor & Didier Queloz (1/2).
"
"Constraints: If evidence is insufficient or conflicting, refuse."
)
],
hallucination_score_config=HallucinationScoreConfig(skeleton_policy="evidence_erase"),
)
print(f"Decision: {rag_result['replies'][0].meta['hallucination_decision']}")
print(f"Risk bound: {rag_result['replies'][0].meta['hallucination_risk']:.3f}")
print(f"Rationale: {rag_result['replies'][0].meta['hallucination_rationale']}")
print(f"Answer:
{rag_result['replies'][0].text}")
print("---")
```
<a id="haystack_experimental.components.generators.chat.openai.OpenAIChatGenerator.run"></a>
#### OpenAIChatGenerator.run
```python
@component.output_types(replies=list[ChatMessage])
def run(
messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
generation_kwargs: dict[str, Any] | None = None,
*,
tools: ToolsType | None = None,
tools_strict: bool | None = None,
hallucination_score_config: HallucinationScoreConfig | None = None
) -> dict[str, list[ChatMessage]]
```
Invokes chat completion based on the provided messages and generation parameters.
**Arguments**:
- `messages`: A list of ChatMessage instances representing the input messages.
- `streaming_callback`: A callback function that is called when a new token is received from the stream.
- `generation_kwargs`: Additional keyword arguments for text generation. These parameters will
override the parameters passed during component initialization.
For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
- `tools`: A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
If set, it will override the `tools` parameter provided during initialization.
- `tools_strict`: Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
the schema provided in the `parameters` field of the tool definition, but this may increase latency.
If set, it will override the `tools_strict` parameter set during component initialization.
- `hallucination_score_config`: If provided, the generator will evaluate the hallucination risk of its responses using
the OpenAIPlanner and annotate each response with hallucination metrics.
This involves generating multiple samples and analyzing their consistency, which may increase
latency and cost. Use this option when you need to assess the reliability of the generated content
in scenarios where accuracy is critical.
For details, see the [research paper](https://arxiv.org/abs/2507.11768)
**Returns**:
A dictionary with the following key:
- `replies`: A list containing the generated responses as ChatMessage instances. If hallucination
scoring is enabled, each message will include additional metadata:
- `hallucination_decision`: "ANSWER" if the model decided to answer, "REFUSE" if it abstained.
- `hallucination_risk`: The EDFL hallucination risk bound.
- `hallucination_rationale`: The rationale behind the hallucination decision.
<a id="haystack_experimental.components.generators.chat.openai.OpenAIChatGenerator.run_async"></a>
#### OpenAIChatGenerator.run\_async
```python
@component.output_types(replies=list[ChatMessage])
async def run_async(
messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
generation_kwargs: dict[str, Any] | None = None,
*,
tools: ToolsType | None = None,
tools_strict: bool | None = None,
hallucination_score_config: HallucinationScoreConfig | None = None
) -> dict[str, list[ChatMessage]]
```
Asynchronously invokes chat completion based on the provided messages and generation parameters.
This is the asynchronous version of the `run` method. It has the same parameters and return values
but can be used with `await` in async code.
**Arguments**:
- `messages`: A list of ChatMessage instances representing the input messages.
- `streaming_callback`: A callback function that is called when a new token is received from the stream.
Must be a coroutine.
- `generation_kwargs`: Additional keyword arguments for text generation. These parameters will
override the parameters passed during component initialization.
For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
- `tools`: A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
If set, it will override the `tools` parameter provided during initialization.
- `tools_strict`: Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
the schema provided in the `parameters` field of the tool definition, but this may increase latency.
If set, it will override the `tools_strict` parameter set during component initialization.
- `hallucination_score_config`: If provided, the generator will evaluate the hallucination risk of its responses using
the OpenAIPlanner and annotate each response with hallucination metrics.
This involves generating multiple samples and analyzing their consistency, which may increase
latency and cost. Use this option when you need to assess the reliability of the generated content
in scenarios where accuracy is critical.
For details, see the [research paper](https://arxiv.org/abs/2507.11768)
**Returns**:
A dictionary with the following key:
- `replies`: A list containing the generated responses as ChatMessage instances. If hallucination
scoring is enabled, each message will include additional metadata:
- `hallucination_decision`: "ANSWER" if the model decided to answer, "REFUSE" if it abstained.
- `hallucination_risk`: The EDFL hallucination risk bound.
- `hallucination_rationale`: The rationale behind the hallucination decision.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_mem0_memory_store_api.md
+218
View File
@@ -0,0 +1,218 @@
---
title: "Mem0 Memory Store"
id: experimental-mem0-memory-store-api
description: "Storage for the memories using Mem0 as the backend."
slug: "/experimental-mem0-memory-store-api"
---
<a id="haystack_experimental.memory_stores.mem0.memory_store"></a>
## Module haystack\_experimental.memory\_stores.mem0.memory\_store
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore"></a>
### Mem0MemoryStore
A memory store implementation using Mem0 as the backend.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.__init__"></a>
#### Mem0MemoryStore.\_\_init\_\_
```python
def __init__(*, api_key: Secret = Secret.from_env_var("MEM0_API_KEY"))
```
Initialize the Mem0 memory store.
**Arguments**:
- `api_key`: The Mem0 API key. You can also set it using `MEM0_API_KEY` environment variable.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.to_dict"></a>
#### Mem0MemoryStore.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serialize the store configuration to a dictionary.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.from_dict"></a>
#### Mem0MemoryStore.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Mem0MemoryStore"
```
Deserialize the store from a dictionary.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.add_memories"></a>
#### Mem0MemoryStore.add\_memories
```python
def add_memories(*,
messages: list[ChatMessage],
infer: bool = True,
user_id: str | None = None,
run_id: str | None = None,
agent_id: str | None = None,
async_mode: bool = False,
**kwargs: Any) -> list[dict[str, Any]]
```
Add ChatMessage memories to Mem0.
**Arguments**:
- `messages`: List of ChatMessage objects with memory metadata
- `infer`: Whether to infer facts from the messages. If False, the whole message will
be added as a memory.
- `user_id`: The user ID to to store and retrieve memories from the memory store.
- `run_id`: The run ID to to store and retrieve memories from the memory store.
- `agent_id`: The agent ID to to store and retrieve memories from the memory store.
If you want Mem0 to store chat messages from the assistant, you need to set the agent_id.
- `async_mode`: Whether to add memories asynchronously.
If True, the method will return immediately and the memories will be added in the background.
- `kwargs`: Additional keyword arguments to pass to the Mem0 client.add method.
Note: ChatMessage.meta in the list of messages will be ignored because Mem0 doesn't allow
passing metadata for each message in the list. You can pass metadata for the whole memory
by passing the `metadata` keyword argument to the method.
**Returns**:
List of objects with the memory_id and the memory
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.search_memories"></a>
#### Mem0MemoryStore.search\_memories
```python
def search_memories(*,
query: str | None = None,
filters: dict[str, Any] | None = None,
top_k: int = 5,
user_id: str | None = None,
run_id: str | None = None,
agent_id: str | None = None,
include_memory_metadata: bool = False,
**kwargs: Any) -> list[ChatMessage]
```
Search for memories in Mem0.
If filters are not provided, at least one of user_id, run_id, or agent_id must be set.
If filters are provided, the search will be scoped to the provided filters and the other ids will be ignored.
**Arguments**:
- `query`: Text query to search for. If not provided, all memories will be returned.
- `filters`: Haystack filters to apply on search. For more details on Haystack filters, see https://docs.haystack.deepset.ai/docs/metadata-filtering
- `top_k`: Maximum number of results to return
- `user_id`: The user ID to to store and retrieve memories from the memory store.
- `run_id`: The run ID to to store and retrieve memories from the memory store.
- `agent_id`: The agent ID to to store and retrieve memories from the memory store.
If you want Mem0 to store chat messages from the assistant, you need to set the agent_id.
- `include_memory_metadata`: Whether to include the mem0 related metadata for the
retrieved memory in the ChatMessage.
If True, the metadata will include the mem0 related metadata i.e. memory_id, score, etc.
in the `mem0_memory_metadata` key.
If False, the `ChatMessage.meta` will only contain the user defined metadata.
- `kwargs`: Additional keyword arguments to pass to the Mem0 client.
If query is passed, the kwargs will be passed to the Mem0 client.search method.
If query is not passed, the kwargs will be passed to the Mem0 client.get_all method.
**Returns**:
List of ChatMessage memories matching the criteria
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.search_memories_as_single_message"></a>
#### Mem0MemoryStore.search\_memories\_as\_single\_message
```python
def search_memories_as_single_message(*,
query: str | None = None,
filters: dict[str, Any] | None = None,
top_k: int = 5,
user_id: str | None = None,
run_id: str | None = None,
agent_id: str | None = None,
**kwargs: Any) -> ChatMessage
```
Search for memories in Mem0 and return a single ChatMessage object.
If filters are not provided, at least one of user_id, run_id, or agent_id must be set.
If filters are provided, the search will be scoped to the provided filters and the other ids will be ignored.
**Arguments**:
- `query`: Text query to search for. If not provided, all memories will be returned.
- `filters`: Additional filters to apply on search. For more details on mem0 filters, see https://mem0.ai/docs/search/
- `top_k`: Maximum number of results to return
- `user_id`: The user ID to to store and retrieve memories from the memory store.
- `run_id`: The run ID to to store and retrieve memories from the memory store.
- `agent_id`: The agent ID to to store and retrieve memories from the memory store.
If you want Mem0 to store chat messages from the assistant, you need to set the agent_id.
- `kwargs`: Additional keyword arguments to pass to the Mem0 client.
If query is passed, the kwargs will be passed to the Mem0 client.search method.
If query is not passed, the kwargs will be passed to the Mem0 client.get_all method.
**Returns**:
A single ChatMessage object with the memories matching the criteria
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.delete_all_memories"></a>
#### Mem0MemoryStore.delete\_all\_memories
```python
def delete_all_memories(*,
user_id: str | None = None,
run_id: str | None = None,
agent_id: str | None = None,
**kwargs: Any) -> None
```
Delete memory records from Mem0.
At least one of user_id, run_id, or agent_id must be set.
**Arguments**:
- `user_id`: The user ID to delete memories from.
- `run_id`: The run ID to delete memories from.
- `agent_id`: The agent ID to delete memories from.
- `kwargs`: Additional keyword arguments to pass to the Mem0 client.delete_all method.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.delete_memory"></a>
#### Mem0MemoryStore.delete\_memory
```python
def delete_memory(memory_id: str, **kwargs: Any) -> None
```
Delete memory from Mem0.
**Arguments**:
- `memory_id`: The ID of the memory to delete.
- `kwargs`: Additional keyword arguments to pass to the Mem0 client.delete method.
<a id="haystack_experimental.memory_stores.mem0.memory_store.Mem0MemoryStore.normalize_filters"></a>
#### Mem0MemoryStore.normalize\_filters
```python
@staticmethod
def normalize_filters(filters: dict[str, Any]) -> dict[str, Any]
```
Convert Haystack filters to Mem0 filters.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_preprocessors_api.md
+76
View File
@@ -0,0 +1,76 @@
---
title: "Preprocessors"
id: experimental-preprocessors-api
description: "Pipelines wrapped as components."
slug: "/experimental-preprocessors-api"
---
<a id="haystack_experimental.components.preprocessors.md_header_level_inferrer"></a>
## Module haystack\_experimental.components.preprocessors.md\_header\_level\_inferrer
<a id="haystack_experimental.components.preprocessors.md_header_level_inferrer.MarkdownHeaderLevelInferrer"></a>
### MarkdownHeaderLevelInferrer
Infers and rewrites header levels in Markdown text to normalize hierarchy.
First header → Always becomes level 1 (#)
Subsequent headers → Level increases if no content between headers, stays same if content exists
Maximum level → Capped at 6 (######)
### Usage example
```python
from haystack import Document
from haystack_experimental.components.preprocessors import MarkdownHeaderLevelInferrer
# Create a document with uniform header levels
text = "## Title
## Subheader
Section
## Subheader
More Content"
doc = Document(content=text)
# Initialize the inferrer and process the document
inferrer = MarkdownHeaderLevelInferrer()
result = inferrer.run([doc])
# The headers are now normalized with proper hierarchy
print(result["documents"][0].content)
> # Title
## Subheader
Section
## Subheader
More Content
```
<a id="haystack_experimental.components.preprocessors.md_header_level_inferrer.MarkdownHeaderLevelInferrer.__init__"></a>
#### MarkdownHeaderLevelInferrer.\_\_init\_\_
```python
def __init__()
```
Initializes the MarkdownHeaderLevelInferrer.
<a id="haystack_experimental.components.preprocessors.md_header_level_inferrer.MarkdownHeaderLevelInferrer.run"></a>
#### MarkdownHeaderLevelInferrer.run
```python
@component.output_types(documents=list[Document])
def run(documents: list[Document]) -> dict
```
Infers and rewrites the header levels in the content for documents that use uniform header levels.
**Arguments**:
- `documents`: list of Document objects to process.
**Returns**:
dict: a dictionary with the key 'documents' containing the processed Document objects.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_retrievers_api.md
+124
View File
@@ -0,0 +1,124 @@
---
title: "Retrievers"
id: experimental-retrievers-api
description: "Sweep through Document Stores and return a set of candidate documents that are relevant to the query."
slug: "/experimental-retrievers-api"
---
<a id="haystack_experimental.components.retrievers.chat_message_retriever"></a>
## Module haystack\_experimental.components.retrievers.chat\_message\_retriever
<a id="haystack_experimental.components.retrievers.chat_message_retriever.ChatMessageRetriever"></a>
### ChatMessageRetriever
Retrieves chat messages from the underlying ChatMessageStore.
Usage example:
```python
from haystack.dataclasses import ChatMessage
from haystack_experimental.components.retrievers import ChatMessageRetriever
from haystack_experimental.chat_message_stores.in_memory import InMemoryChatMessageStore
messages = [
ChatMessage.from_assistant("Hello, how can I help you?"),
ChatMessage.from_user("Hi, I have a question about Python. What is a Protocol?"),
]
message_store = InMemoryChatMessageStore()
message_store.write_messages(chat_history_id="user_456_session_123", messages=messages)
retriever = ChatMessageRetriever(message_store)
result = retriever.run(chat_history_id="user_456_session_123")
print(result["messages"])
```
<a id="haystack_experimental.components.retrievers.chat_message_retriever.ChatMessageRetriever.__init__"></a>
#### ChatMessageRetriever.\_\_init\_\_
```python
def __init__(chat_message_store: ChatMessageStore, last_k: int | None = 10)
```
Create the ChatMessageRetriever component.
**Arguments**:
- `chat_message_store`: An instance of a ChatMessageStore.
- `last_k`: The number of last messages to retrieve. Defaults to 10 messages if not specified.
<a id="haystack_experimental.components.retrievers.chat_message_retriever.ChatMessageRetriever.to_dict"></a>
#### ChatMessageRetriever.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns**:
Dictionary with serialized data.
<a id="haystack_experimental.components.retrievers.chat_message_retriever.ChatMessageRetriever.from_dict"></a>
#### ChatMessageRetriever.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "ChatMessageRetriever"
```
Deserializes the component from a dictionary.
**Arguments**:
- `data`: The dictionary to deserialize from.
**Returns**:
The deserialized component.
<a id="haystack_experimental.components.retrievers.chat_message_retriever.ChatMessageRetriever.run"></a>
#### ChatMessageRetriever.run
```python
@component.output_types(messages=list[ChatMessage])
def run(
chat_history_id: str,
*,
last_k: int | None = None,
current_messages: list[ChatMessage] | None = None
) -> dict[str, list[ChatMessage]]
```
Run the ChatMessageRetriever
**Arguments**:
- `chat_history_id`: A unique identifier for the chat session or conversation whose messages should be retrieved.
Each `chat_history_id` corresponds to a distinct chat history stored in the underlying ChatMessageStore.
For example, use a session ID or conversation ID to isolate messages from different chat sessions.
- `last_k`: The number of last messages to retrieve. This parameter takes precedence over the last_k
parameter passed to the ChatMessageRetriever constructor. If unspecified, the last_k parameter passed
to the constructor will be used.
- `current_messages`: A list of incoming chat messages to combine with the retrieved messages. System messages from this list
are prepended before the retrieved history, while all other messages (e.g., user messages) are appended
after. This is useful for including new conversational context alongside stored history so the output
can be directly used as input to a ChatGenerator or an Agent. If not provided, only the stored messages
will be returned.
**Raises**:
- `ValueError`: If last_k is not None and is less than 0.
**Returns**:
A dictionary with the following key:
- `messages` - The retrieved chat messages combined with any provided current messages.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_summarizer_api.md
+198
View File
@@ -0,0 +1,198 @@
---
title: "Summarizers"
id: experimental-summarizers-api
description: "Components that summarize texts into concise versions."
slug: "/experimental-summarizers-api"
---
<a id="haystack_experimental.components.summarizers.llm_summarizer"></a>
## Module haystack\_experimental.components.summarizers.llm\_summarizer
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer"></a>
### LLMSummarizer
Summarizes text using a language model.
It's inspired by code from the OpenAI blog post: https://cookbook.openai.com/examples/summarizing_long_documents
Example
```python
from haystack_experimental.components.summarizers.summarizer import Summarizer
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack import Document
text = ("Machine learning is a subset of artificial intelligence that provides systems "
"the ability to automatically learn and improve from experience without being "
"explicitly programmed. The process of learning begins with observations or data. "
"Supervised learning algorithms build a mathematical model of sample data, known as "
"training data, in order to make predictions or decisions. Unsupervised learning "
"algorithms take a set of data that contains only inputs and find structure in the data. "
"Reinforcement learning is an area of machine learning where an agent learns to behave "
"in an environment by performing actions and seeing the results. Deep learning uses "
"artificial neural networks to model complex patterns in data. Neural networks consist "
"of layers of connected nodes, each performing a simple computation.")
doc = Document(content=text)
chat_generator = OpenAIChatGenerator(model="gpt-4")
summarizer = Summarizer(chat_generator=chat_generator)
summarizer.run(documents=[doc])
```
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.__init__"></a>
#### LLMSummarizer.\_\_init\_\_
```python
def __init__(chat_generator: ChatGenerator,
system_prompt: str
| None = "Rewrite this text in summarized form.",
summary_detail: float = 0,
minimum_chunk_size: int | None = 500,
chunk_delimiter: str = ".",
summarize_recursively: bool = False,
split_overlap: int = 0)
```
Initialize the Summarizer component.
:param chat_generator: A ChatGenerator instance to use for summarization.
:param system_prompt: The prompt to instruct the LLM to summarise text, if not given defaults to:
"Rewrite this text in summarized form."
:param summary_detail: The level of detail for the summary (0-1), defaults to 0.
This parameter controls the trade-off between conciseness and completeness by adjusting how many
chunks the text is divided into. At detail=0, the text is processed as a single chunk (or very few
chunks), producing the most concise summary. At detail=1, the text is split into the maximum number
of chunks allowed by minimum_chunk_size, enabling more granular analysis and detailed summaries.
The formula uses linear interpolation: num_chunks = 1 + detail * (max_chunks - 1), where max_chunks
is determined by dividing the document length by minimum_chunk_size.
:param minimum_chunk_size: The minimum token count per chunk, defaults to 500
:param chunk_delimiter: The character used to determine separator priority.
"." uses sentence-based splitting, "
" uses paragraph-based splitting, defaults to "."
:param summarize_recursively: Whether to use previous summaries as context, defaults to False.
:param split_overlap: Number of tokens to overlap between consecutive chunks, defaults to 0.
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.warm_up"></a>
#### LLMSummarizer.warm\_up
```python
def warm_up()
```
Warm up the chat generator and document splitter components.
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.to_dict"></a>
#### LLMSummarizer.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns**:
Dictionary with serialized data.
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.from_dict"></a>
#### LLMSummarizer.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "LLMSummarizer"
```
Deserializes the component from a dictionary.
**Arguments**:
- `data`: Dictionary with serialized data.
**Returns**:
An instance of the component.
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.num_tokens"></a>
#### LLMSummarizer.num\_tokens
```python
def num_tokens(text: str) -> int
```
Estimates the token count for a given text.
Uses the RecursiveDocumentSplitter's tokenization logic for consistency.
**Arguments**:
- `text`: The text to tokenize
**Returns**:
The estimated token count
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.summarize"></a>
#### LLMSummarizer.summarize
```python
def summarize(text: str,
detail: float,
minimum_chunk_size: int,
summarize_recursively: bool = False) -> str
```
Summarizes text by splitting it into optimally-sized chunks and processing each with an LLM.
**Arguments**:
- `text`: Text to summarize
- `detail`: Detail level (0-1) where 0 is most concise and 1 is most detailed
- `minimum_chunk_size`: Minimum token count per chunk
- `summarize_recursively`: Whether to use previous summaries as context
**Raises**:
- `ValueError`: If detail is not between 0 and 1
**Returns**:
The textual content summarized by the LLM.
<a id="haystack_experimental.components.summarizers.llm_summarizer.LLMSummarizer.run"></a>
#### LLMSummarizer.run
```python
@component.output_types(summary=list[Document])
def run(*,
documents: list[Document],
detail: float | None = None,
minimum_chunk_size: int | None = None,
summarize_recursively: bool | None = None,
system_prompt: str | None = None) -> dict[str, list[Document]]
```
Run the summarizer on a list of documents.
**Arguments**:
- `documents`: List of documents to summarize
- `detail`: The level of detail for the summary (0-1), defaults to 0 overwriting the component's default.
- `minimum_chunk_size`: The minimum token count per chunk, defaults to 500 overwriting the
component's default.
- `system_prompt`: If given it will overwrite prompt given at init time or the default one.
- `summarize_recursively`: Whether to use previous summaries as context, defaults to False overwriting the
component's default.
**Raises**:
- `RuntimeError`: If the component wasn't warmed up.
docs-website/reference_versioned_docs/version-2.30/experiments-api/experimental_writers_api.md
+105
View File
@@ -0,0 +1,105 @@
---
title: "Writers"
id: experimental-writers-api
description: "Writers for Haystack."
slug: "/experimental-writers-api"
---
<a id="haystack_experimental.components.writers.chat_message_writer"></a>
## Module haystack\_experimental.components.writers.chat\_message\_writer
<a id="haystack_experimental.components.writers.chat_message_writer.ChatMessageWriter"></a>
### ChatMessageWriter
Writes chat messages to an underlying ChatMessageStore.
Usage example:
```python
from haystack.dataclasses import ChatMessage
from haystack_experimental.components.writers import ChatMessageWriter
from haystack_experimental.chat_message_stores.in_memory import InMemoryChatMessageStore
messages = [
ChatMessage.from_assistant("Hello, how can I help you?"),
ChatMessage.from_user("I have a question about Python."),
]
message_store = InMemoryChatMessageStore()
writer = ChatMessageWriter(message_store)
writer.run(chat_history_id="user_456_session_123", messages=messages)
```
<a id="haystack_experimental.components.writers.chat_message_writer.ChatMessageWriter.__init__"></a>
#### ChatMessageWriter.\_\_init\_\_
```python
def __init__(chat_message_store: ChatMessageStore) -> None
```
Create a ChatMessageWriter component.
**Arguments**:
- `chat_message_store`: The ChatMessageStore where the chat messages are to be written.
<a id="haystack_experimental.components.writers.chat_message_writer.ChatMessageWriter.to_dict"></a>
#### ChatMessageWriter.to\_dict
```python
def to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns**:
Dictionary with serialized data.
<a id="haystack_experimental.components.writers.chat_message_writer.ChatMessageWriter.from_dict"></a>
#### ChatMessageWriter.from\_dict
```python
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "ChatMessageWriter"
```
Deserializes the component from a dictionary.
**Arguments**:
- `data`: The dictionary to deserialize from.
**Raises**:
- `DeserializationError`: If the message store is not properly specified in the serialization data or its type cannot be imported.
**Returns**:
The deserialized component.
<a id="haystack_experimental.components.writers.chat_message_writer.ChatMessageWriter.run"></a>
#### ChatMessageWriter.run
```python
@component.output_types(messages_written=int)
def run(chat_history_id: str, messages: list[ChatMessage]) -> dict[str, int]
```
Run the ChatMessageWriter on the given input data.
**Arguments**:
- `chat_history_id`: A unique identifier for the chat session or conversation whose messages should be retrieved.
Each `chat_history_id` corresponds to a distinct chat history stored in the underlying ChatMessageStore.
For example, use a session ID or conversation ID to isolate messages from different chat sessions.
- `messages`: A list of chat messages to write to the store.
**Returns**:
- `messages_written`: Number of messages written to the ChatMessageStore.
docs-website/reference_versioned_docs/version-2.30/haystack-api/agents_api.md
+463
View File
@@ -0,0 +1,463 @@
---
title: "Agents"
id: agents-api
description: "Tool-using agents with provider-agnostic chat model support."
slug: "/agents-api"
---
## agent
### Agent
A tool-using Agent powered by a large language model.
The Agent processes messages and calls tools until it meets an exit condition.
You can set one or more exit conditions to control when it stops.
For example, it can stop after generating a response or after calling a tool.
Without tools, the Agent works like a standard LLM that generates text. It produces one response and then stops.
### Usage examples
This is an example agent that:
1. Searches for tipping customs in France.
1. Uses a calculator to compute tips based on its findings.
1. Returns the final answer with its context.
```python
from haystack.components.agents import Agent
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.generators.utils import print_streaming_chunk
from haystack.dataclasses import ChatMessage
from haystack.tools import tool
from typing import Annotated, Literal
# Tool functions - in practice, these would have real implementations
@tool
def search(query: Annotated[str, "The search query"]) -> str:
'''Search for information on the web.'''
# Placeholder: would call actual search API
return "In France, a 15% service charge is typically included, but leaving 5-10% extra is appreciated."
@tool
def calculator(
operation: Annotated[Literal["multiply", "percentage"], "The mathematical operation to perform"],
a: Annotated[float, "First number"],
b: Annotated[float, "Second number"],
) -> float:
'''Perform mathematical calculations.'''
if operation == "multiply":
return a * b
elif operation == "percentage":
return (a / 100) * b
return 0
agent = Agent(
system_prompt=(
"You are a helpful assistant. Use the 'search' tool to find information "
"about a user's question and the 'calculator' tool to perform math."
),
chat_generator=OpenAIChatGenerator(),
tools=[search, calculator],
streaming_callback=print_streaming_chunk,
)
result = agent.run(
messages=[ChatMessage.from_user("Calculate the appropriate tip for an €85 meal in France")]
)
# Access the final response from the Agent
# print(result["last_message"].text)
```
#### Using a `user_prompt` template with variables
You can define a reusable `user_prompt` with Jinja2 template variables so the Agent can be invoked
with different inputs without manually constructing `ChatMessage` objects each time.
This is especially useful when embedding the Agent in a pipeline.
```python
from haystack.components.agents import Agent
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.tools import tool
from typing import Annotated
@tool
def translate(
text: Annotated[str, "The text to translate"],
target_language: Annotated[str, "The language to translate to"],
) -> str:
"""Translate text to a target language."""
# Placeholder: would call an actual translation API
return f"[Translated '{text}' to {target_language}]"
agent = Agent(
chat_generator=OpenAIChatGenerator(),
tools=[translate],
system_prompt="You are a helpful translation assistant.",
user_prompt="""{% message role="user"%}
Translate the following document to {{ language }}: {{ document }}
{% endmessage %}""",
required_variables=["language", "document"],
)
# The template variables 'language' and 'document' become inputs to the run method
result = agent.run(
messages=[],
language="French",
document="The weather is lovely today and the sun is shining.",
)
print(result["last_message"].text)
```
#### __init__
```python
__init__(
*,
chat_generator: ChatGenerator,
tools: ToolsType | None = None,
system_prompt: str | None = None,
user_prompt: str | None = None,
required_variables: list[str] | Literal["*"] | None = None,
exit_conditions: list[str] | None = None,
state_schema: dict[str, Any] | None = None,
max_agent_steps: int = 100,
streaming_callback: StreamingCallbackT | None = None,
raise_on_tool_invocation_failure: bool = False,
tool_invoker_kwargs: dict[str, Any] | None = None,
confirmation_strategies: (
dict[str | tuple[str, ...], ConfirmationStrategy] | None
) = None
) -> None
```
Initialize the agent component.
**Parameters:**
- **chat_generator** (<code>ChatGenerator</code>) – An instance of the chat generator that your agent should use. It must support tools.
- **tools** (<code>ToolsType | None</code>) – A list of Tool and/or Toolset objects, or a single Toolset that the agent can use.
- **system_prompt** (<code>str | None</code>) – System prompt for the agent. Can be a plain string or a Jinja2 string template.
For details on the supported template syntax, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/chatpromptbuilder#string-templates).
- **user_prompt** (<code>str | None</code>) – User prompt for the agent, defined as a Jinja2 string template. If provided, this is
appended to the messages provided at runtime.
For details on the supported template syntax, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/chatpromptbuilder#string-templates).
- **required_variables** (<code>list\[str\] | Literal['\*'] | None</code>) – Lists the variables that must be provided as inputs to `user_prompt` or `system_prompt`.
If a required variable is not provided at run time, an exception is raised.
If set to `"*"`, all variables found in the prompts are required. Optional.
- **exit_conditions** (<code>list\[str\] | None</code>) – List of conditions that will cause the agent to return.
Can include "text" if the agent should return when it generates a message without tool calls,
or tool names that will cause the agent to return once the tool was executed. Defaults to ["text"].
- **state_schema** (<code>dict\[str, Any\] | None</code>) – A dictionary defining the agent's runtime state. Each key maps to a type config
with `"type"` (required) and an optional `"handler"` for merging values across tool calls.
Tools can read from and write to state keys using `inputs_from_state` and `outputs_to_state`.
- **max_agent_steps** (<code>int</code>) – Maximum number of steps the agent will run before stopping. Defaults to 100.
If the agent exceeds this number of steps, it will stop and return the current state.
- **streaming_callback** (<code>StreamingCallbackT | None</code>) – A callback that will be invoked when a response is streamed from the LLM.
The same callback can be configured to emit tool results when a tool is called.
- **raise_on_tool_invocation_failure** (<code>bool</code>) – Should the agent raise an exception when a tool invocation fails?
If set to False, the exception will be turned into a chat message and passed to the LLM.
- **tool_invoker_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments to pass to the ToolInvoker.
- **confirmation_strategies** (<code>dict\[str | tuple\[str, ...\], ConfirmationStrategy\] | None</code>) – A dictionary mapping tool names to ConfirmationStrategy instances.
**Raises:**
- <code>TypeError</code> – If the chat_generator does not support tools parameter in its run method.
- <code>ValueError</code> – If the exit_conditions are not valid.
- <code>ValueError</code> – If any `user_prompt` variable overlaps with the `state_schema` or `run` method parameters.
#### warm_up
```python
warm_up() -> None
```
Warm up the Agent.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serialize the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> Agent
```
Deserialize the agent from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>Agent</code> – Deserialized agent.
#### run
```python
run(
messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
*,
generation_kwargs: dict[str, Any] | None = None,
break_point: AgentBreakpoint | None = None,
snapshot: AgentSnapshot | None = None,
system_prompt: str | None = None,
user_prompt: str | None = None,
tools: ToolsType | list[str] | None = None,
snapshot_callback: SnapshotCallback | None = None,
confirmation_strategy_context: dict[str, Any] | None = None,
**kwargs: Any
) -> dict[str, Any]
```
Process messages and execute tools until an exit condition is met.
**Parameters:**
- **messages** (<code>list\[ChatMessage\]</code>) – List of Haystack ChatMessage objects to process.
- **streaming_callback** (<code>StreamingCallbackT | None</code>) – A callback that will be invoked when a response is streamed from the LLM.
The same callback can be configured to emit tool results when a tool is called.
- **generation_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments for LLM. These parameters will
override the parameters passed during component initialization.
- **break_point** (<code>AgentBreakpoint | None</code>) – An AgentBreakpoint, can be a Breakpoint for the "chat_generator" or a ToolBreakpoint
for "tool_invoker".
- **snapshot** (<code>AgentSnapshot | None</code>) – An `AgentSnapshot` object containing the state of a previously saved agent execution,
used to restart the agent from where it left off.
- **system_prompt** (<code>str | None</code>) – System prompt for the agent. If provided, it overrides the default system prompt.
- **user_prompt** (<code>str | None</code>) – User prompt for the agent. If provided, it overrides the default user prompt and is
appended to the messages provided at runtime.
- **tools** (<code>ToolsType | list\[str\] | None</code>) – Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
When passing tool names, tools are selected from the Agent's originally configured tools.
- **snapshot_callback** (<code>SnapshotCallback | None</code>) – Optional callback function that is invoked when a pipeline snapshot is created.
The callback receives a `PipelineSnapshot` object and can return an optional string.
If provided, the callback is used instead of the default file-saving behavior.
- **confirmation_strategy_context** (<code>dict\[str, Any\] | None</code>) – Optional dictionary for passing request-scoped resources
to confirmation strategies. Useful in web/server environments to provide per-request
objects (e.g., WebSocket connections, async queues, Redis pub/sub clients) that strategies
can use for non-blocking user interaction.
- **kwargs** (<code>Any</code>) – Additional data to pass to the State schema used by the Agent.
The keys must match the schema defined in the Agent's `state_schema`.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- "messages": List of all messages exchanged during the agent's run.
- "last_message": The last message exchanged during the agent's run.
- Any additional keys defined in the `state_schema`.
**Raises:**
- <code>BreakpointException</code> – If an agent breakpoint is triggered.
#### run_async
```python
run_async(
messages: list[ChatMessage],
streaming_callback: StreamingCallbackT | None = None,
*,
generation_kwargs: dict[str, Any] | None = None,
break_point: AgentBreakpoint | None = None,
snapshot: AgentSnapshot | None = None,
system_prompt: str | None = None,
user_prompt: str | None = None,
tools: ToolsType | list[str] | None = None,
snapshot_callback: SnapshotCallback | None = None,
confirmation_strategy_context: dict[str, Any] | None = None,
**kwargs: Any
) -> dict[str, Any]
```
Asynchronously process messages and execute tools until the exit condition is met.
This is the asynchronous version of the `run` method. It follows the same logic but uses
asynchronous operations where possible, such as calling the `run_async` method of the ChatGenerator
if available.
**Parameters:**
- **messages** (<code>list\[ChatMessage\]</code>) – List of Haystack ChatMessage objects to process.
- **streaming_callback** (<code>StreamingCallbackT | None</code>) – An asynchronous callback that will be invoked when a response is streamed from the
LLM. The same callback can be configured to emit tool results when a tool is called.
- **generation_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments for LLM. These parameters will
override the parameters passed during component initialization.
- **break_point** (<code>AgentBreakpoint | None</code>) – An AgentBreakpoint, can be a Breakpoint for the "chat_generator" or a ToolBreakpoint
for "tool_invoker".
- **snapshot** (<code>AgentSnapshot | None</code>) – An `AgentSnapshot` object containing the state of a previously saved agent execution,
used to restart the agent from where it left off.
- **system_prompt** (<code>str | None</code>) – System prompt for the agent. If provided, it overrides the default system prompt.
- **user_prompt** (<code>str | None</code>) – User prompt for the agent. If provided, it overrides the default user prompt and is
appended to the messages provided at runtime.
- **tools** (<code>ToolsType | list\[str\] | None</code>) – Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
- **snapshot_callback** (<code>SnapshotCallback | None</code>) – Optional callback function that is invoked when a pipeline snapshot is created.
The callback receives a `PipelineSnapshot` object and can return an optional string.
If provided, the callback is used instead of the default file-saving behavior.
- **confirmation_strategy_context** (<code>dict\[str, Any\] | None</code>) – Optional dictionary for passing request-scoped resources
to confirmation strategies. Useful in web/server environments to provide per-request
objects (e.g., WebSocket connections, async queues, Redis pub/sub clients) that strategies
can use for non-blocking user interaction.
- **kwargs** (<code>Any</code>) – Additional data to pass to the State schema used by the Agent.
The keys must match the schema defined in the Agent's `state_schema`.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- "messages": List of all messages exchanged during the agent's run.
- "last_message": The last message exchanged during the agent's run.
- Any additional keys defined in the `state_schema`.
**Raises:**
- <code>BreakpointException</code> – If an agent breakpoint is triggered.
## state/state
### State
State is a container for storing shared information during the execution of an Agent and its tools.
For instance, State can be used to store documents, context, and intermediate results.
Internally it wraps a `_data` dictionary defined by a `schema`. Each schema entry has:
```json
"parameter_name": {
"type": SomeType, # expected type
"handler": Optional[Callable[[Any, Any], Any]] # merge/update function
}
```
Handlers control how values are merged when using the `set()` method:
- For list types: defaults to `merge_lists` (concatenates lists)
- For other types: defaults to `replace_values` (overwrites existing value)
A `messages` field with type `list[ChatMessage]` is automatically added to the schema.
This makes it possible for the Agent to read from and write to the same context.
### Usage example
```python
from haystack.components.agents.state import State
my_state = State(
schema={"gh_repo_name": {"type": str}, "user_name": {"type": str}},
data={"gh_repo_name": "my_repo", "user_name": "my_user_name"}
)
```
#### __init__
```python
__init__(schema: dict[str, Any], data: dict[str, Any] | None = None) -> None
```
Initialize a State object with a schema and optional data.
**Parameters:**
- **schema** (<code>dict\[str, Any\]</code>) – Dictionary mapping parameter names to their type and handler configs.
Type must be a valid Python type, and handler must be a callable function or None.
If handler is None, the default handler for the type will be used. The default handlers are:
- For list types: `haystack.agents.state.state_utils.merge_lists`
- For all other types: `haystack.agents.state.state_utils.replace_values`
- **data** (<code>dict\[str, Any\] | None</code>) – Optional dictionary of initial data to populate the state
#### get
```python
get(key: str, default: Any = None) -> Any
```
Retrieve a value from the state by key.
**Parameters:**
- **key** (<code>str</code>) – Key to look up in the state
- **default** (<code>Any</code>) – Value to return if key is not found
**Returns:**
- <code>Any</code> – Value associated with key or default if not found
#### set
```python
set(
key: str,
value: Any,
handler_override: Callable[[Any, Any], Any] | None = None,
) -> None
```
Set or merge a value in the state according to schema rules.
Value is merged or overwritten according to these rules:
- if handler_override is given, use that
- else use the handler defined in the schema for 'key'
**Parameters:**
- **key** (<code>str</code>) – Key to store the value under
- **value** (<code>Any</code>) – Value to store or merge
- **handler_override** (<code>Callable\\[[Any, Any\], Any\] | None</code>) – Optional function to override the default merge behavior
#### data
```python
data: dict[str, Any]
```
All current data of the state.
#### has
```python
has(key: str) -> bool
```
Check if a key exists in the state.
**Parameters:**
- **key** (<code>str</code>) – Key to check for existence
**Returns:**
- <code>bool</code> – True if key exists in state, False otherwise
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Convert the State object to a dictionary.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> State
```
Convert a dictionary back to a State object.
docs-website/reference_versioned_docs/version-2.30/haystack-api/audio_api.md
+243
View File
@@ -0,0 +1,243 @@
---
title: "Audio"
id: audio-api
description: "Transcribes audio files."
slug: "/audio-api"
---
## whisper_local
### LocalWhisperTranscriber
Transcribes audio files using OpenAI's Whisper model on your local machine.
For the supported audio formats, languages, and other parameters, see the
[Whisper API documentation](https://platform.openai.com/docs/guides/speech-to-text) and the official Whisper
[GitHub repository](https://github.com/openai/whisper).
### Usage example
<!-- test-ignore -->
```python
from haystack.components.audio import LocalWhisperTranscriber
whisper = LocalWhisperTranscriber(model="small")
transcription = whisper.run(sources=["test/test_files/audio/answer.wav"])
```
#### __init__
```python
__init__(
model: WhisperLocalModel = "large",
device: ComponentDevice | None = None,
whisper_params: dict[str, Any] | None = None,
) -> None
```
Creates an instance of the LocalWhisperTranscriber component.
**Parameters:**
- **model** (<code>WhisperLocalModel</code>) – The name of the model to use. Set to one of the following models:
"tiny", "base", "small", "medium", "large" (default).
For details on the models and their modifications, see the
[Whisper documentation](https://github.com/openai/whisper?tab=readme-ov-file#available-models-and-languages).
- **device** (<code>ComponentDevice | None</code>) – The device for loading the model. If `None`, automatically selects the default device.
#### warm_up
```python
warm_up() -> None
```
Loads the model in memory.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> LocalWhisperTranscriber
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>LocalWhisperTranscriber</code> – The deserialized component.
#### run
```python
run(
sources: list[str | Path | ByteStream],
whisper_params: dict[str, Any] | None = None,
) -> dict[str, Any]
```
Transcribes a list of audio files into a list of documents.
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – A list of paths or binary streams to transcribe.
- **whisper_params** (<code>dict\[str, Any\] | None</code>) – For the supported audio formats, languages, and other parameters, see the
[Whisper API documentation](https://platform.openai.com/docs/guides/speech-to-text) and the official Whisper
[GitHup repo](https://github.com/openai/whisper).
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `documents`: A list of documents where each document is a transcribed audio file. The content of
the document is the transcription text, and the document's metadata contains the values returned by
the Whisper model, such as the alignment data and the path to the audio file used
for the transcription.
#### transcribe
```python
transcribe(
sources: list[str | Path | ByteStream], **kwargs: Any
) -> list[Document]
```
Transcribes the audio files into a list of Documents, one for each input file.
For the supported audio formats, languages, and other parameters, see the
[Whisper API documentation](https://platform.openai.com/docs/guides/speech-to-text) and the official Whisper
[github repo](https://github.com/openai/whisper).
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – A list of paths or binary streams to transcribe.
**Returns:**
- <code>list\[Document\]</code> – A list of Documents, one for each file.
## whisper_remote
### RemoteWhisperTranscriber
Transcribes audio files using the OpenAI's Whisper API.
The component requires an OpenAI API key, see the
[OpenAI documentation](https://platform.openai.com/docs/api-reference/authentication) for more details.
For the supported audio formats, languages, and other parameters, see the
[Whisper API documentation](https://platform.openai.com/docs/guides/speech-to-text).
### Usage example
```python
from haystack.components.audio import RemoteWhisperTranscriber
whisper = RemoteWhisperTranscriber(model="whisper-1")
transcription = whisper.run(sources=["test/test_files/audio/answer.wav"])
```
#### __init__
```python
__init__(
api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
model: str = "whisper-1",
api_base_url: str | None = None,
organization: str | None = None,
http_client_kwargs: dict[str, Any] | None = None,
**kwargs: Any
) -> None
```
Creates an instance of the RemoteWhisperTranscriber component.
**Parameters:**
- **api_key** (<code>Secret</code>) – OpenAI API key.
You can set it with an environment variable `OPENAI_API_KEY`, or pass with this parameter
during initialization.
- **model** (<code>str</code>) – Name of the model to use. Currently accepts only `whisper-1`.
- **organization** (<code>str | None</code>) – Your OpenAI organization ID. See OpenAI's documentation on
[Setting Up Your Organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
- **api_base_url** (<code>str | None</code>) – An optional URL to use as the API base. For details, see the
OpenAI [documentation](https://platform.openai.com/docs/api-reference/audio).
- **http_client_kwargs** (<code>dict\[str, Any\] | None</code>) – A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
- **kwargs** (<code>Any</code>) – Other optional parameters for the model. These are sent directly to the OpenAI
endpoint. See OpenAI [documentation](https://platform.openai.com/docs/api-reference/audio) for more details.
Some of the supported parameters are:
- `language`: The language of the input audio.
Provide the input language in ISO-639-1 format
to improve transcription accuracy and latency.
- `prompt`: An optional text to guide the model's
style or continue a previous audio segment.
The prompt should match the audio language.
- `response_format`: The format of the transcript
output. This component only supports `json`.
- `temperature`: The sampling temperature, between 0
and 1. Higher values like 0.8 make the output more
random, while lower values like 0.2 make it more
focused and deterministic. If set to 0, the model
uses log probability to automatically increase the
temperature until certain thresholds are hit.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> RemoteWhisperTranscriber
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>RemoteWhisperTranscriber</code> – The deserialized component.
#### run
```python
run(sources: list[str | Path | ByteStream]) -> dict[str, Any]
```
Transcribes the list of audio files into a list of documents.
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – A list of file paths or `ByteStream` objects containing the audio files to transcribe.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `documents`: A list of documents, one document for each file.
The content of each document is the transcribed text.
docs-website/reference_versioned_docs/version-2.30/haystack-api/builders_api.md
+540
View File
@@ -0,0 +1,540 @@
---
title: "Builders"
id: builders-api
description: "Extract the output of a Generator to an Answer format, and build prompts."
slug: "/builders-api"
---
## answer_builder
### AnswerBuilder
Converts a query and Generator replies into a `GeneratedAnswer` object.
AnswerBuilder parses Generator replies using custom regular expressions.
Check out the usage example below to see how it works.
Optionally, it can also take documents and metadata from the Generator to add to the `GeneratedAnswer` object.
AnswerBuilder works with both non-chat and chat Generators.
### Usage example
```python
from haystack.components.builders import AnswerBuilder
builder = AnswerBuilder(pattern="Answer: (.*)")
builder.run(query="What's the answer?", replies=["This is an argument. Answer: This is the answer."])
```
### Usage example with documents and reference pattern
```python
from haystack import Document
from haystack.components.builders import AnswerBuilder
replies = ["The capital of France is Paris [2]."]
docs = [
Document(content="Berlin is the capital of Germany."),
Document(content="Paris is the capital of France."),
Document(content="Rome is the capital of Italy."),
]
builder = AnswerBuilder(reference_pattern="\[(\d+)\]", return_only_referenced_documents=False)
result = builder.run(query="What is the capital of France?", replies=replies, documents=docs)["answers"][0]
print(f"Answer: {result.data}")
print("References:")
for doc in result.documents:
if doc.meta["referenced"]:
print(f"[{doc.meta['source_index']}] {doc.content}")
print("Other sources:")
for doc in result.documents:
if not doc.meta["referenced"]:
print(f"[{doc.meta['source_index']}] {doc.content}")
# >> Answer: The capital of France is Paris
# >> References:
# >> [2] Paris is the capital of France.
# >> Other sources:
# >> [1] Berlin is the capital of Germany.
# >> [3] Rome is the capital of Italy.
```
#### __init__
```python
__init__(
pattern: str | None = None,
reference_pattern: str | None = None,
last_message_only: bool = False,
*,
return_only_referenced_documents: bool = True
) -> None
```
Creates an instance of the AnswerBuilder component.
**Parameters:**
- **pattern** (<code>str | None</code>) – The regular expression pattern to extract the answer text from the Generator.
If not specified, the entire response is used as the answer.
The regular expression can have one capture group at most.
If present, the capture group text
is used as the answer. If no capture group is present, the whole match is used as the answer.
Examples:
`[^\n]+$` finds "this is an answer" in a string "this is an argument.\\nthis is an answer".
`Answer: (.*)` finds "this is an answer" in a string "this is an argument. Answer: this is an answer".
- **reference_pattern** (<code>str | None</code>) – The regular expression pattern used for parsing the document references.
If not specified, no parsing is done, and all documents are returned.
References need to be specified as indices of the input documents and start at [1].
Example: `\[(\d+)\]` finds "1" in a string "this is an answer[1]".
If this parameter is provided, documents metadata will contain a "referenced" key with a boolean value.
- **last_message_only** (<code>bool</code>) – If False (default value), all messages are used as the answer.
If True, only the last message is used as the answer.
- **return_only_referenced_documents** (<code>bool</code>) – To be used in conjunction with `reference_pattern`.
If True (default value), only the documents that were actually referenced in `replies` are returned.
If False, all documents are returned.
If `reference_pattern` is not provided, this parameter has no effect, and all documents are returned.
#### run
```python
run(
query: str,
replies: list[str] | list[ChatMessage],
meta: list[dict[str, Any]] | None = None,
documents: list[Document] | None = None,
pattern: str | None = None,
reference_pattern: str | None = None,
) -> dict[str, Any]
```
Turns the output of a Generator into `GeneratedAnswer` objects using regular expressions.
**Parameters:**
- **query** (<code>str</code>) – The input query used as the Generator prompt.
- **replies** (<code>list\[str\] | list\[ChatMessage\]</code>) – The output of the Generator. Can be a list of strings or a list of `ChatMessage` objects.
- **meta** (<code>list\[dict\[str, Any\]\] | None</code>) – The metadata returned by the Generator. If not specified, the generated answer will contain no metadata.
- **documents** (<code>list\[Document\] | None</code>) – The documents used as the Generator inputs. If specified, they are added to
the `GeneratedAnswer` objects.
The Document copies inside the returned `GeneratedAnswer.documents` each include a "source_index" key,
representing the document's 1-based position in the input list. The original input documents are
not modified.
When `reference_pattern` is provided:
- "referenced" key is added to the Document copies inside `GeneratedAnswer.documents`, indicating if
the document was referenced in the output.
- `return_only_referenced_documents` init parameter controls if all or only referenced documents are
returned.
- **pattern** (<code>str | None</code>) – The regular expression pattern to extract the answer text from the Generator.
If not specified, the entire response is used as the answer.
The regular expression can have one capture group at most.
If present, the capture group text
is used as the answer. If no capture group is present, the whole match is used as the answer.
Examples:
`[^\n]+$` finds "this is an answer" in a string "this is an argument.\\nthis is an answer".
`Answer: (.*)` finds "this is an answer" in a string
"this is an argument. Answer: this is an answer".
- **reference_pattern** (<code>str | None</code>) – The regular expression pattern used for parsing the document references.
If not specified, no parsing is done, and all documents are returned.
References need to be specified as indices of the input documents and start at [1].
Example: `\[(\d+)\]` finds "1" in a string "this is an answer[1]".
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `answers`: The answers received from the output of the Generator.
## chat_prompt_builder
### ChatPromptBuilder
Renders a chat prompt from a template using Jinja2 syntax.
A template can be a list of `ChatMessage` objects, or a special string, as shown in the usage examples.
It constructs prompts using static or dynamic templates, which you can update for each pipeline run.
Template variables in the template are optional unless specified otherwise.
If an optional variable isn't provided, it defaults to an empty string. Use `variable` and `required_variables`
to define input types and required variables.
### Usage examples
#### Static ChatMessage prompt template
```python
template = [ChatMessage.from_user("Translate to {{ target_language }}. Context: {{ snippet }}; Translation:")]
builder = ChatPromptBuilder(template=template)
builder.run(target_language="spanish", snippet="I can't speak spanish.")
```
#### Overriding static ChatMessage template at runtime
```python
template = [ChatMessage.from_user("Translate to {{ target_language }}. Context: {{ snippet }}; Translation:")]
builder = ChatPromptBuilder(template=template)
builder.run(target_language="spanish", snippet="I can't speak spanish.")
msg = "Translate to {{ target_language }} and summarize. Context: {{ snippet }}; Summary:"
summary_template = [ChatMessage.from_user(msg)]
builder.run(target_language="spanish", snippet="I can't speak spanish.", template=summary_template)
```
#### Dynamic ChatMessage prompt template
```python
from haystack.components.builders import ChatPromptBuilder
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.dataclasses import ChatMessage
from haystack import Pipeline
# no parameter init, we don't use any runtime template variables
prompt_builder = ChatPromptBuilder()
llm = OpenAIChatGenerator(model="gpt-5-mini")
pipe = Pipeline()
pipe.add_component("prompt_builder", prompt_builder)
pipe.add_component("llm", llm)
pipe.connect("prompt_builder.prompt", "llm.messages")
location = "Berlin"
language = "English"
system_message = ChatMessage.from_system("You are an assistant giving information to tourists in {{language}}")
messages = [system_message, ChatMessage.from_user("Tell me about {{location}}")]
res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "language": language},
"template": messages}})
print(res)
# >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
# "Berlin is the capital city of Germany and one of the most vibrant
# and diverse cities in Europe. Here are some key things to know...Enjoy your time exploring the vibrant and dynamic
# capital of Germany!")], _name=None, _meta={'model': 'gpt-5-mini',
# 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 27, 'completion_tokens': 681, 'total_tokens':
# 708}})]}}
messages = [system_message, ChatMessage.from_user("What's the weather forecast for {{location}} in the next {{day_count}} days?")]
res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "day_count": "5"},
"template": messages}})
print(res)
# >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
# "Here is the weather forecast for Berlin in the next 5
# days:\n\nDay 1: Mostly cloudy with a high of 22°C (72°F) and...so it's always a good idea to check for updates
# closer to your visit.")], _name=None, _meta={'model': 'gpt-5-mini',
# 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 37, 'completion_tokens': 201,
# 'total_tokens': 238}})]}}
```
#### String prompt template
```python
from haystack.components.builders import ChatPromptBuilder
from haystack.dataclasses.image_content import ImageContent
template = """
{% message role="system" %}
You are a helpful assistant.
{% endmessage %}
{% message role="user" %}
Hello! I am {{user_name}}. What's the difference between the following images?
{% for image in images %}
{{ image | templatize_part }}
{% endfor %}
{% endmessage %}
"""
images = [ImageContent.from_file_path("test/test_files/images/apple.jpg"),
ImageContent.from_file_path("test/test_files/images/haystack-logo.png")]
builder = ChatPromptBuilder(template=template)
builder.run(user_name="John", images=images)
```
#### __init__
```python
__init__(
template: list[ChatMessage] | str | None = None,
required_variables: list[str] | Literal["*"] | None = None,
variables: list[str] | None = None,
) -> None
```
Constructs a ChatPromptBuilder component.
**Parameters:**
- **template** (<code>list\[ChatMessage\] | str | None</code>) – A list of `ChatMessage` objects or a string template. The component looks for Jinja2 template syntax and
renders the prompt with the provided variables. Provide the template in either
the `init` method`or the`run\` method.
- **required_variables** (<code>list\[str\] | Literal['\*'] | None</code>) – List variables that must be provided as input to ChatPromptBuilder.
If a variable listed as required is not provided, an exception is raised.
If set to `"*"`, all variables found in the prompt are required. Optional.
- **variables** (<code>list\[str\] | None</code>) – List input variables to use in prompt templates instead of the ones inferred from the
`template` parameter. For example, to use more variables during prompt engineering than the ones present
in the default template, you can provide them here.
#### run
```python
run(
template: list[ChatMessage] | str | None = None,
template_variables: dict[str, Any] | None = None,
**kwargs: Any
) -> dict[str, list[ChatMessage]]
```
Renders the prompt template with the provided variables.
It applies the template variables to render the final prompt. You can provide variables with pipeline kwargs.
To overwrite the default template, you can set the `template` parameter.
To overwrite pipeline kwargs, you can set the `template_variables` parameter.
**Parameters:**
- **template** (<code>list\[ChatMessage\] | str | None</code>) – An optional list of `ChatMessage` objects or string template to overwrite ChatPromptBuilder's default
template.
If `None`, the default template provided at initialization is used.
- **template_variables** (<code>dict\[str, Any\] | None</code>) – An optional dictionary of template variables to overwrite the pipeline variables.
- **kwargs** (<code>Any</code>) – Pipeline variables used for rendering the prompt.
**Returns:**
- <code>dict\[str, list\[ChatMessage\]\]</code> – A dictionary with the following keys:
- `prompt`: The updated list of `ChatMessage` objects after rendering the templates.
**Raises:**
- <code>ValueError</code> – If `chat_messages` is empty or contains elements that are not instances of `ChatMessage`.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Returns a dictionary representation of the component.
**Returns:**
- <code>dict\[str, Any\]</code> – Serialized dictionary representation of the component.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> ChatPromptBuilder
```
Deserialize this component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize and create the component.
**Returns:**
- <code>ChatPromptBuilder</code> – The deserialized component.
## prompt_builder
### PromptBuilder
Renders a prompt filling in any variables so that it can send it to a Generator.
The prompt uses Jinja2 template syntax.
The variables in the default template are used as PromptBuilder's input and are all optional.
If they're not provided, they're replaced with an empty string in the rendered prompt.
To try out different prompts, you can replace the prompt template at runtime by
providing a template for each pipeline run invocation.
### Usage examples
#### On its own
This example uses PromptBuilder to render a prompt template and fill it with `target_language`
and `snippet`. PromptBuilder returns a prompt with the string "Translate the following context to Spanish.
Context: I can't speak Spanish.; Translation:".
```python
from haystack.components.builders import PromptBuilder
template = "Translate the following context to {{ target_language }}. Context: {{ snippet }}; Translation:"
builder = PromptBuilder(template=template)
builder.run(target_language="spanish", snippet="I can't speak spanish.")
```
#### In a Pipeline
This is an example of a RAG pipeline where PromptBuilder renders a custom prompt template and fills it
with the contents of the retrieved documents and a query. The rendered prompt is then sent to a Generator.
```python
from haystack import Pipeline, Document
from haystack.utils import Secret
from haystack.components.generators import OpenAIGenerator
from haystack.components.builders.prompt_builder import PromptBuilder
# in a real world use case documents could come from a retriever, web, or any other source
documents = [Document(content="Joe lives in Berlin"), Document(content="Joe is a software engineer")]
prompt_template = """
Given these documents, answer the question.
Documents:
{% for doc in documents %}
{{ doc.content }}
{% endfor %}
Question: {{query}}
Answer:
"""
p = Pipeline()
p.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder")
p.add_component(instance=OpenAIGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), name="llm")
p.connect("prompt_builder", "llm")
question = "Where does Joe live?"
result = p.run({"prompt_builder": {"documents": documents, "query": question}})
print(result)
```
#### Changing the template at runtime (prompt engineering)
You can change the prompt template of an existing pipeline, like in this example:
```python
documents = [
Document(content="Joe lives in Berlin", meta={"name": "doc1"}),
Document(content="Joe is a software engineer", meta={"name": "doc1"}),
]
new_template = """
You are a helpful assistant.
Given these documents, answer the question.
Documents:
{% for doc in documents %}
Document {{ loop.index }}:
Document name: {{ doc.meta['name'] }}
{{ doc.content }}
{% endfor %}
Question: {{ query }}
Answer:
"""
p.run({
"prompt_builder": {
"documents": documents,
"query": question,
"template": new_template,
},
})
```
To replace the variables in the default template when testing your prompt,
pass the new variables in the `variables` parameter.
#### Overwriting variables at runtime
To overwrite the values of variables, use `template_variables` during runtime:
```python
language_template = """
You are a helpful assistant.
Given these documents, answer the question.
Documents:
{% for doc in documents %}
Document {{ loop.index }}:
Document name: {{ doc.meta['name'] }}
{{ doc.content }}
{% endfor %}
Question: {{ query }}
Please provide your answer in {{ answer_language | default('English') }}
Answer:
"""
p.run({
"prompt_builder": {
"documents": documents,
"query": question,
"template": language_template,
"template_variables": {"answer_language": "German"},
},
})
```
Note that `language_template` introduces variable `answer_language` which is not bound to any pipeline variable.
If not set otherwise, it will use its default value 'English'.
This example overwrites its value to 'German'.
Use `template_variables` to overwrite pipeline variables (such as documents) as well.
#### __init__
```python
__init__(
template: str,
required_variables: list[str] | Literal["*"] | None = None,
variables: list[str] | None = None,
) -> None
```
Constructs a PromptBuilder component.
**Parameters:**
- **template** (<code>str</code>) – A prompt template that uses Jinja2 syntax to add variables. For example:
`"Summarize this document: {{ documents[0].content }}\nSummary:"`
It's used to render the prompt.
The variables in the default template are input for PromptBuilder and are all optional,
unless explicitly specified.
If an optional variable is not provided, it's replaced with an empty string in the rendered prompt.
- **required_variables** (<code>list\[str\] | Literal['\*'] | None</code>) – List variables that must be provided as input to PromptBuilder.
If a variable listed as required is not provided, an exception is raised.
If set to `"*"`, all variables found in the prompt are required. Optional.
- **variables** (<code>list\[str\] | None</code>) – List input variables to use in prompt templates instead of the ones inferred from the
`template` parameter. For example, to use more variables during prompt engineering than the ones present
in the default template, you can provide them here.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Returns a dictionary representation of the component.
**Returns:**
- <code>dict\[str, Any\]</code> – Serialized dictionary representation of the component.
#### run
```python
run(
template: str | None = None,
template_variables: dict[str, Any] | None = None,
**kwargs: Any
) -> dict[str, Any]
```
Renders the prompt template with the provided variables.
It applies the template variables to render the final prompt. You can provide variables via pipeline kwargs.
In order to overwrite the default template, you can set the `template` parameter.
In order to overwrite pipeline kwargs, you can set the `template_variables` parameter.
**Parameters:**
- **template** (<code>str | None</code>) – An optional string template to overwrite PromptBuilder's default template. If None, the default template
provided at initialization is used.
- **template_variables** (<code>dict\[str, Any\] | None</code>) – An optional dictionary of template variables to overwrite the pipeline variables.
- **kwargs** (<code>Any</code>) – Pipeline variables used for rendering the prompt.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `prompt`: The updated prompt text after rendering the prompt template.
**Raises:**
- <code>ValueError</code> – If any of the required template variables is not provided.
docs-website/reference_versioned_docs/version-2.30/haystack-api/cachings_api.md
+114
View File
@@ -0,0 +1,114 @@
---
title: "Caching"
id: caching-api
description: "Checks if any document coming from the given URL is already present in the store."
slug: "/caching-api"
---
## cache_checker
### CacheChecker
Checks for the presence of documents in a Document Store based on a specified field in each document's metadata.
If matching documents are found, they are returned as "hits". If not found in the cache, the items
are returned as "misses".
### Usage example
```python
from haystack import Document
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.caching.cache_checker import CacheChecker
docstore = InMemoryDocumentStore()
documents = [
Document(content="doc1", meta={"url": "https://example.com/1"}),
Document(content="doc2", meta={"url": "https://example.com/2"}),
Document(content="doc3", meta={"url": "https://example.com/1"}),
Document(content="doc4", meta={"url": "https://example.com/2"}),
]
docstore.write_documents(documents)
checker = CacheChecker(docstore, cache_field="url")
results = checker.run(items=["https://example.com/1", "https://example.com/5"])
assert results == {"hits": [documents[0], documents[2]], "misses": ["https://example.com/5"]}
```
#### __init__
```python
__init__(document_store: DocumentStore, cache_field: str) -> None
```
Creates a CacheChecker component.
**Parameters:**
- **document_store** (<code>DocumentStore</code>) – Document Store to check for the presence of specific documents.
- **cache_field** (<code>str</code>) – Name of the document's metadata field
to check for cache hits.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> CacheChecker
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>CacheChecker</code> – Deserialized component.
#### run
```python
run(items: list[Any]) -> dict[str, Any]
```
Checks if any document associated with the specified cache field is already present in the store.
**Parameters:**
- **items** (<code>list\[Any\]</code>) – Values to be checked against the cache field.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with two keys:
- `hits` - Documents that matched with at least one of the items.
- `misses` - Items that were not present in any documents.
#### run_async
```python
run_async(items: list[Any]) -> dict[str, Any]
```
Asynchronously checks if any document associated with the specified cache field is already present in the store.
**Parameters:**
- **items** (<code>list\[Any\]</code>) – Values to be checked against the cache field.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with two keys:
- `hits` - Documents that matched with at least one of the items.
- `misses` - Items that were not present in any documents.
docs-website/reference_versioned_docs/version-2.30/haystack-api/classifiers_api.md
+247
View File
@@ -0,0 +1,247 @@
---
title: "Classifiers"
id: classifiers-api
description: "Classify documents based on the provided labels."
slug: "/classifiers-api"
---
## document_language_classifier
### DocumentLanguageClassifier
Classifies the language of each document and adds it to its metadata.
Provide a list of languages during initialization. If the document's text doesn't match any of the
specified languages, the metadata value is set to "unmatched".
To route documents based on their language, use the MetadataRouter component after DocumentLanguageClassifier.
For routing plain text, use the TextLanguageRouter component instead.
### Usage example
```python
from haystack import Document, Pipeline
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.classifiers import DocumentLanguageClassifier
from haystack.components.routers import MetadataRouter
from haystack.components.writers import DocumentWriter
docs = [Document(id="1", content="This is an English document"),
Document(id="2", content="Este es un documento en español")]
document_store = InMemoryDocumentStore()
p = Pipeline()
p.add_component(instance=DocumentLanguageClassifier(languages=["en"]), name="language_classifier")
p.add_component(
instance=MetadataRouter(rules={
"en": {
"field": "meta.language",
"operator": "==",
"value": "en"
}
}),
name="router")
p.add_component(instance=DocumentWriter(document_store=document_store), name="writer")
p.connect("language_classifier.documents", "router.documents")
p.connect("router.en", "writer.documents")
p.run({"language_classifier": {"documents": docs}})
written_docs = document_store.filter_documents()
assert len(written_docs) == 1
assert written_docs[0] == Document(id="1", content="This is an English document", meta={"language": "en"})
```
#### __init__
```python
__init__(languages: list[str] | None = None) -> None
```
Initializes the DocumentLanguageClassifier component.
**Parameters:**
- **languages** (<code>list\[str\] | None</code>) – A list of ISO language codes.
See the supported languages in [`langdetect` documentation](https://github.com/Mimino666/langdetect#languages).
If not specified, defaults to ["en"].
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Classifies the language of each document and adds it to its metadata.
If the document's text doesn't match any of the languages specified at initialization,
sets the metadata value to "unmatched".
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of documents for language classification.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: A list of documents with an added `language` metadata field.
**Raises:**
- <code>TypeError</code> – if the input is not a list of Documents.
## zero_shot_document_classifier
### TransformersZeroShotDocumentClassifier
Performs zero-shot classification of documents based on given labels and adds the predicted label to their metadata.
The component uses a Hugging Face pipeline for zero-shot classification.
Provide the model and the set of labels to be used for categorization during initialization.
Additionally, you can configure the component to allow multiple labels to be true.
Classification is run on the document's content field by default. If you want it to run on another field, set the
`classification_field` to one of the document's metadata fields.
Available models for the task of zero-shot-classification include:
\- `valhalla/distilbart-mnli-12-3`
\- `cross-encoder/nli-distilroberta-base`
\- `cross-encoder/nli-deberta-v3-xsmall`
### Usage example
The following is a pipeline that classifies documents based on predefined classification labels
retrieved from a search pipeline:
```python
from haystack import Document
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.core.pipeline import Pipeline
from haystack.components.classifiers import TransformersZeroShotDocumentClassifier
documents = [Document(id="0", content="Today was a nice day!"),
Document(id="1", content="Yesterday was a bad day!")]
document_store = InMemoryDocumentStore()
retriever = InMemoryBM25Retriever(document_store=document_store)
document_classifier = TransformersZeroShotDocumentClassifier(
model="cross-encoder/nli-deberta-v3-xsmall",
labels=["positive", "negative"],
)
document_store.write_documents(documents)
pipeline = Pipeline()
pipeline.add_component(instance=retriever, name="retriever")
pipeline.add_component(instance=document_classifier, name="document_classifier")
pipeline.connect("retriever", "document_classifier")
queries = ["How was your day today?", "How was your day yesterday?"]
expected_predictions = ["positive", "negative"]
for idx, query in enumerate(queries):
result = pipeline.run({"retriever": {"query": query, "top_k": 1}})
assert result["document_classifier"]["documents"][0].to_dict()["id"] == str(idx)
assert (result["document_classifier"]["documents"][0].to_dict()["classification"]["label"]
== expected_predictions[idx])
```
#### __init__
```python
__init__(
model: str,
labels: list[str],
multi_label: bool = False,
classification_field: str | None = None,
device: ComponentDevice | None = None,
token: Secret | None = Secret.from_env_var(
["HF_API_TOKEN", "HF_TOKEN"], strict=False
),
huggingface_pipeline_kwargs: dict[str, Any] | None = None,
) -> None
```
Initializes the TransformersZeroShotDocumentClassifier.
See the Hugging Face [website](https://huggingface.co/models?pipeline_tag=zero-shot-classification&sort=downloads&search=nli)
for the full list of zero-shot classification models (NLI) models.
**Parameters:**
- **model** (<code>str</code>) – The name or path of a Hugging Face model for zero shot document classification.
- **labels** (<code>list\[str\]</code>) – The set of possible class labels to classify each document into, for example,
["positive", "negative"]. The labels depend on the selected model.
- **multi_label** (<code>bool</code>) – Whether or not multiple candidate labels can be true.
If `False`, the scores are normalized such that
the sum of the label likelihoods for each sequence is 1. If `True`, the labels are considered
independent and probabilities are normalized for each candidate by doing a softmax of the entailment
score vs. the contradiction score.
- **classification_field** (<code>str | None</code>) – Name of document's meta field to be used for classification.
If not set, `Document.content` is used by default.
- **device** (<code>ComponentDevice | None</code>) – The device on which the model is loaded. If `None`, the default device is automatically
selected. If a device/device map is specified in `huggingface_pipeline_kwargs`, it overrides this parameter.
- **token** (<code>Secret | None</code>) – The Hugging Face token to use as HTTP bearer authorization.
Check your HF token in your [account settings](https://huggingface.co/settings/tokens).
- **huggingface_pipeline_kwargs** (<code>dict\[str, Any\] | None</code>) – Dictionary containing keyword arguments used to initialize the
Hugging Face pipeline for text classification.
#### warm_up
```python
warm_up() -> None
```
Initializes the component.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> TransformersZeroShotDocumentClassifier
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>TransformersZeroShotDocumentClassifier</code> – Deserialized component.
#### run
```python
run(documents: list[Document], batch_size: int = 1) -> dict[str, Any]
```
Classifies the documents based on the provided labels and adds them to their metadata.
The classification results are stored in the `classification` dict within
each document's metadata. If `multi_label` is set to `True`, the scores for each label are available under
the `details` key within the `classification` dictionary.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – Documents to process.
- **batch_size** (<code>int</code>) – Batch size used for processing the content in each document.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following key:
- `documents`: A list of documents with an added metadata field called `classification`.
docs-website/reference_versioned_docs/version-2.30/haystack-api/connectors_api.md
+275
View File
@@ -0,0 +1,275 @@
---
title: "Connectors"
id: connectors-api
description: "Various connectors to integrate with external services."
slug: "/connectors-api"
---
## openapi
### OpenAPIConnector
OpenAPIConnector enables direct invocation of REST endpoints defined in an OpenAPI specification.
The OpenAPIConnector serves as a bridge between Haystack pipelines and any REST API that follows
the OpenAPI(formerly Swagger) specification. It dynamically interprets the API specification and
provides an interface for executing API operations. It is usually invoked by passing input
arguments to it from a Haystack pipeline run method or by other components in a pipeline that
pass input arguments to this component.
Example:
<!-- test-ignore -->
```python
from haystack.utils import Secret
from haystack.components.connectors.openapi import OpenAPIConnector
serper_dev_token = Secret.from_env_var("SERPERDEV_API_KEY")
def my_custom_config_factory():
# Create and return a custom configuration for the OpenAPIClient
pass
connector = OpenAPIConnector(
openapi_spec="https://bit.ly/serperdev_openapi",
credentials=serper_dev_token,
service_kwargs={"config_factory": my_custom_config_factory()}
)
response = connector.run(
operation_id="search",
arguments={"q": "Who was Nikola Tesla?"}
)
```
Note:
- The `service_kwargs` argument is optional, it can be used to pass additional options to the OpenAPIClient.
#### __init__
```python
__init__(
openapi_spec: str,
credentials: Secret | None = None,
service_kwargs: dict[str, Any] | None = None,
) -> None
```
Initialize the OpenAPIConnector with a specification and optional credentials.
**Parameters:**
- **openapi_spec** (<code>str</code>) – URL, file path, or raw string of the OpenAPI specification
- **credentials** (<code>Secret | None</code>) – Optional API key or credentials for the service wrapped in a Secret
- **service_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments passed to OpenAPIClient.from_spec()
For example, you can pass a custom config_factory or other configuration options.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serialize this component to a dictionary.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> OpenAPIConnector
```
Deserialize this component from a dictionary.
#### run
```python
run(
operation_id: str, arguments: dict[str, Any] | None = None
) -> dict[str, Any]
```
Invokes a REST endpoint specified in the OpenAPI specification.
**Parameters:**
- **operation_id** (<code>str</code>) – The operationId from the OpenAPI spec to invoke
- **arguments** (<code>dict\[str, Any\] | None</code>) – Optional parameters for the endpoint (query, path, or body parameters)
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary containing the service response
## openapi_service
### patch_request
```python
patch_request(
self: Operation,
base_url: str,
*,
data: Any | None = None,
parameters: dict[str, Any] | None = None,
raw_response: bool = False,
security: dict[str, str] | None = None,
session: Any | None = None,
verify: bool | str = True
) -> Any | None
```
Sends an HTTP request as described by this path.
**Parameters:**
- **base_url** (<code>str</code>) – The URL to append this operation's path to when making
the call.
- **data** (<code>Any | None</code>) – The request body to send.
- **parameters** (<code>dict\[str, Any\] | None</code>) – The parameters used to create the path.
- **raw_response** (<code>bool</code>) – If true, return the raw response instead of validating
and extrapolating it.
- **security** (<code>dict\[str, str\] | None</code>) – The security scheme to use, and the values it needs to
process successfully.
- **session** (<code>Any | None</code>) – A persistent request session.
- **verify** (<code>bool | str</code>) – If we should do an SSL verification on the request or not.
In case str was provided, will use that as the CA.
**Returns:**
- <code>Any | None</code> – The response data, either raw or processed depending on raw_response flag.
### OpenAPIServiceConnector
A component which connects the Haystack framework to OpenAPI services.
The `OpenAPIServiceConnector` component connects the Haystack framework to OpenAPI services, enabling it to call
operations as defined in the OpenAPI specification of the service.
It integrates with `ChatMessage` dataclass, where the `ToolCall` entries in messages are used to determine the
method to be called and the parameters to be passed. The method name and parameters are then used to invoke the
method on the OpenAPI service. The response from the service is returned as a `ChatMessage`.
Before using this component, users usually resolve service endpoint parameters with a help of
`OpenAPIServiceToFunctions` component.
The example below demonstrates how to use the `OpenAPIServiceConnector` to invoke a method on a https://serper.dev/
service specified via OpenAPI specification.
Note, however, that `OpenAPIServiceConnector` is usually not meant to be used directly, but rather as part of a
pipeline that includes the `OpenAPIServiceToFunctions` component and a Chat Generator component using an LLM
with tool calling capabilities. In the example below we use the tool call payload directly, but in a
real-world scenario, the tool calls would usually be generated by the Chat Generator component.
You need to define the `serper_token` variable with your Serper.dev API token for the example to work.
Can be through the `SERPERDEV_API_KEY` environment variable or by directly assigning the token string to the
variable in the code.
Usage example:
<!-- test-ignore -->
```python
import json
import httpx
from haystack.components.connectors import OpenAPIServiceConnector
from haystack.dataclasses import ChatMessage, ToolCall
from haystack.utils import Secret
tool_call = ToolCall(
tool_name="search",
arguments={"q": "Why was Sam Altman ousted from OpenAI?"},
)
message = ChatMessage.from_assistant(tool_calls=[tool_call])
serper_token = Secret.from_env_var("SERPERDEV_API_KEY").resolve_value()
serperdev_openapi_spec = json.loads(httpx.get("https://bit.ly/serper_dev_spec", follow_redirects=True).text)
service_connector = OpenAPIServiceConnector()
result = service_connector.run(
messages=[message],
service_openapi_spec=serperdev_openapi_spec,
service_credentials=serper_token,
)
print(result)
# {'service_response': ChatMessage(_role=<ChatRole.USER: 'user'>, _content=[TextContent(text=
# '{"searchParameters": {"q": "Why was Sam Altman ousted from OpenAI?",
# "type": "search", "engine": "google"}, "answerBox": {"snippet": "Concerns over AI safety and OpenAI's role
# in protecting were at the center of Altman's brief ouster from the company."...
```
#### __init__
```python
__init__(ssl_verify: bool | str | None = None) -> None
```
Initializes the OpenAPIServiceConnector instance
**Parameters:**
- **ssl_verify** (<code>[bool | str | None</code>) – Decide if to use SSL verification to the requests or not,
in case a string is passed, will be used as the CA.
#### run
```python
run(
messages: list[ChatMessage],
service_openapi_spec: dict[str, Any],
service_credentials: dict | str | None = None,
) -> dict[str, list[ChatMessage]]
```
Processes a list of chat messages to invoke a method on an OpenAPI service.
It parses the last message in the list, expecting it to contain tool calls.
**Parameters:**
- **messages** (<code>list\[ChatMessage\]</code>) – A list of `ChatMessage` objects containing the messages to be processed. The last message
should contain the tool calls.
- **service_openapi_spec** (<code>dict\[str, Any\]</code>) – The OpenAPI JSON specification object of the service to be invoked. All the refs
should already be resolved.
- **service_credentials** (<code>dict | str | None</code>) – The credentials to be used for authentication with the service.
Currently, only the http and apiKey OpenAPI security schemes are supported.
**Returns:**
- <code>dict\[str, list\[ChatMessage\]\]</code> – A dictionary with the following keys:
- `service_response`: a list of `ChatMessage` objects, each containing the response from the service. The
response is in JSON format, and the `content` attribute of the `ChatMessage` contains
the JSON string.
**Raises:**
- <code>ValueError</code> – If the last message is not from the assistant or if it does not contain tool calls.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> OpenAPIServiceConnector
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>OpenAPIServiceConnector</code> – The deserialized component.
docs-website/reference_versioned_docs/version-2.30/haystack-api/converters_api.md
+1950
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/data_classes_api.md
+1349
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/document_stores_api.md
+593
View File
@@ -0,0 +1,593 @@
---
title: "Document Stores"
id: document-stores-api
description: "Stores your texts and meta data and provides them to the Retriever at query time."
slug: "/document-stores-api"
---
## document_store
### BM25DocumentStats
A dataclass for managing document statistics for BM25 retrieval.
**Parameters:**
- **freq_token** (<code>dict\[str, int\]</code>) – A Counter of token frequencies in the document.
- **doc_len** (<code>int</code>) – Number of tokens in the document.
### InMemoryDocumentStore
Stores data in-memory. It's ephemeral and cannot be saved to disk.
#### __init__
```python
__init__(
bm25_tokenization_regex: str = "(?u)\\b\\w+\\b",
bm25_algorithm: Literal["BM25Okapi", "BM25L", "BM25Plus"] = "BM25L",
bm25_parameters: dict | None = None,
embedding_similarity_function: Literal[
"dot_product", "cosine"
] = "dot_product",
index: str | None = None,
async_executor: ThreadPoolExecutor | None = None,
return_embedding: bool = True,
) -> None
```
Initializes the DocumentStore.
**Parameters:**
- **bm25_tokenization_regex** (<code>str</code>) – The regular expression used to tokenize the text for BM25 retrieval.
- **bm25_algorithm** (<code>Literal['BM25Okapi', 'BM25L', 'BM25Plus']</code>) – The BM25 algorithm to use. One of "BM25Okapi", "BM25L", or "BM25Plus".
- **bm25_parameters** (<code>dict | None</code>) – Parameters for BM25 implementation in a dictionary format.
For example: `{'k1':1.5, 'b':0.75, 'epsilon':0.25}`
You can learn more about these parameters by visiting https://github.com/dorianbrown/rank_bm25.
- **embedding_similarity_function** (<code>Literal['dot_product', 'cosine']</code>) – The similarity function used to compare Documents embeddings.
One of "dot_product" (default) or "cosine". To choose the most appropriate function, look for information
about your embedding model.
- **index** (<code>str | None</code>) – A specific index to store the documents. If not specified, a random UUID is used.
Using the same index allows you to store documents across multiple InMemoryDocumentStore instances.
- **async_executor** (<code>ThreadPoolExecutor | None</code>) – Optional ThreadPoolExecutor to use for async calls. If not provided, a single-threaded
executor will be initialized and used.
- **return_embedding** (<code>bool</code>) – Whether to return the embedding of the retrieved Documents. Default is True.
#### shutdown
```python
shutdown() -> None
```
Explicitly shutdown the executor if we own it.
#### storage
```python
storage: dict[str, Document]
```
Utility property that returns the storage used by this instance of InMemoryDocumentStore.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> InMemoryDocumentStore
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>InMemoryDocumentStore</code> – The deserialized component.
#### save_to_disk
```python
save_to_disk(path: str) -> None
```
Write the database and its data to disk as a JSON file.
**Parameters:**
- **path** (<code>str</code>) – The path to the JSON file.
#### load_from_disk
```python
load_from_disk(path: str) -> InMemoryDocumentStore
```
Load the database and its data from disk as a JSON file.
**Parameters:**
- **path** (<code>str</code>) – The path to the JSON file.
**Returns:**
- <code>InMemoryDocumentStore</code> – The loaded InMemoryDocumentStore.
#### count_documents
```python
count_documents() -> int
```
Returns the number of documents present in the DocumentStore.
#### filter_documents
```python
filter_documents(filters: dict[str, Any] | None = None) -> list[Document]
```
Returns the documents that match the filters provided.
**Parameters:**
- **filters** (<code>dict\[str, Any\] | None</code>) – The filters to apply. For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
**Returns:**
- <code>list\[Document\]</code> – A list of Documents that match the given filters.
#### write_documents
```python
write_documents(
documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
) -> int
```
Refer to the DocumentStore.write_documents() protocol documentation.
If `policy` is set to `DuplicatePolicy.NONE` defaults to `DuplicatePolicy.FAIL`.
#### delete_documents
```python
delete_documents(document_ids: list[str]) -> None
```
Deletes all documents with matching document_ids from the DocumentStore.
**Parameters:**
- **document_ids** (<code>list\[str\]</code>) – The document_ids to delete.
#### delete_all_documents
```python
delete_all_documents() -> None
```
Deletes all documents in the document store.
#### update_by_filter
```python
update_by_filter(filters: dict[str, Any], meta: dict[str, Any]) -> int
```
Updates the metadata of all documents that match the provided filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply to select documents for updating.
For filter syntax, see filter_documents.
- **meta** (<code>dict\[str, Any\]</code>) – The metadata fields to update. These will be merged with existing metadata.
**Returns:**
- <code>int</code> – The number of documents updated.
**Raises:**
- <code>ValueError</code> – if filters have invalid syntax.
#### delete_by_filter
```python
delete_by_filter(filters: dict[str, Any]) -> int
```
Deletes all documents that match the provided filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply to select documents for deletion.
For filter syntax, see filter_documents.
**Returns:**
- <code>int</code> – The number of documents deleted.
**Raises:**
- <code>ValueError</code> – if filters have invalid syntax.
#### count_documents_by_filter
```python
count_documents_by_filter(filters: dict[str, Any]) -> int
```
Returns the number of documents that match the provided filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply.
For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
**Returns:**
- <code>int</code> – The number of documents that match the filters.
#### count_unique_metadata_by_filter
```python
count_unique_metadata_by_filter(
filters: dict[str, Any], metadata_fields: list[str]
) -> dict[str, int]
```
Returns the number of unique values for each specified metadata field from documents matching the filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply.
For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
- **metadata_fields** (<code>list\[str\]</code>) – List of field names to count unique values for.
Field names can include or omit the "meta." prefix.
**Returns:**
- <code>dict\[str, int\]</code> – A dictionary mapping each metadata field name (without "meta." prefix)
to the count of its unique values among the filtered documents.
#### get_metadata_fields_info
```python
get_metadata_fields_info() -> dict[str, dict[str, str]]
```
Returns information about the metadata fields present in the stored documents.
Types are inferred from the stored values (keyword, int, float, boolean).
**Returns:**
- <code>dict\[str, dict\[str, str\]\]</code> – A dictionary mapping each metadata field name to a dict with a "type" key.
#### get_metadata_field_min_max
```python
get_metadata_field_min_max(metadata_field: str) -> dict[str, Any]
```
Returns the minimum and maximum values for the given metadata field across all documents.
**Parameters:**
- **metadata_field** (<code>str</code>) – The metadata field name. Can include or omit the "meta." prefix.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with "min" and "max" keys. Returns `{"min": None, "max": None}`
if the field is missing or has no values.
#### get_metadata_field_unique_values
```python
get_metadata_field_unique_values(
metadata_field: str, search_term: str | None = None
) -> tuple[list[str], int]
```
Returns unique values for a metadata field, optionally filtered by a search term in content.
**Parameters:**
- **metadata_field** (<code>str</code>) – The metadata field name. Can include or omit the "meta." prefix.
- **search_term** (<code>str | None</code>) – If set, only documents whose content contains this term (case-insensitive)
are considered.
**Returns:**
- <code>tuple\[list\[str\], int\]</code> – A tuple of (list of unique values, total count of unique values).
#### bm25_retrieval
```python
bm25_retrieval(
query: str,
filters: dict[str, Any] | None = None,
top_k: int = 10,
scale_score: bool = False,
) -> list[Document]
```
Retrieves documents that are most relevant to the query using BM25 algorithm.
**Parameters:**
- **query** (<code>str</code>) – The query string.
- **filters** (<code>dict\[str, Any\] | None</code>) – A dictionary with filters to narrow down the search space.
- **top_k** (<code>int</code>) – The number of top documents to retrieve. Default is 10.
- **scale_score** (<code>bool</code>) – Whether to scale the scores of the retrieved documents. Default is False.
**Returns:**
- <code>list\[Document\]</code> – A list of the top_k documents most relevant to the query.
#### embedding_retrieval
```python
embedding_retrieval(
query_embedding: list[float],
filters: dict[str, Any] | None = None,
top_k: int = 10,
scale_score: bool = False,
return_embedding: bool | None = False,
) -> list[Document]
```
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
**Parameters:**
- **query_embedding** (<code>list\[float\]</code>) – Embedding of the query.
- **filters** (<code>dict\[str, Any\] | None</code>) – A dictionary with filters to narrow down the search space.
- **top_k** (<code>int</code>) – The number of top documents to retrieve. Default is 10.
- **scale_score** (<code>bool</code>) – Whether to scale the scores of the retrieved Documents. Default is False.
- **return_embedding** (<code>bool | None</code>) – Whether to return the embedding of the retrieved Documents.
If not provided, the value of the `return_embedding` parameter set at component
initialization will be used. Default is False.
**Returns:**
- <code>list\[Document\]</code> – A list of the top_k documents most relevant to the query.
**Raises:**
- <code>ValueError</code> – if filters have invalid syntax.
#### count_documents_async
```python
count_documents_async() -> int
```
Returns the number of documents present in the DocumentStore.
#### filter_documents_async
```python
filter_documents_async(filters: dict[str, Any] | None = None) -> list[Document]
```
Returns the documents that match the filters provided.
**Parameters:**
- **filters** (<code>dict\[str, Any\] | None</code>) – The filters to apply. For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
**Returns:**
- <code>list\[Document\]</code> – A list of Documents that match the given filters.
#### write_documents_async
```python
write_documents_async(
documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
) -> int
```
Refer to the DocumentStore.write_documents() protocol documentation.
If `policy` is set to `DuplicatePolicy.NONE` defaults to `DuplicatePolicy.FAIL`.
#### delete_documents_async
```python
delete_documents_async(document_ids: list[str]) -> None
```
Deletes all documents with matching document_ids from the DocumentStore.
**Parameters:**
- **document_ids** (<code>list\[str\]</code>) – The document_ids to delete.
#### update_by_filter_async
```python
update_by_filter_async(filters: dict[str, Any], meta: dict[str, Any]) -> int
```
Updates the metadata of all documents that match the provided filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply to select documents for updating.
For filter syntax, see filter_documents.
- **meta** (<code>dict\[str, Any\]</code>) – The metadata fields to update. These will be merged with existing metadata.
**Returns:**
- <code>int</code> – The number of documents updated.
#### count_documents_by_filter_async
```python
count_documents_by_filter_async(filters: dict[str, Any]) -> int
```
Returns the number of documents that match the provided filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply.
For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
**Returns:**
- <code>int</code> – The number of documents that match the filters.
#### count_unique_metadata_by_filter_async
```python
count_unique_metadata_by_filter_async(
filters: dict[str, Any], metadata_fields: list[str]
) -> dict[str, int]
```
Returns the number of unique values for each specified metadata field from documents matching the filters.
**Parameters:**
- **filters** (<code>dict\[str, Any\]</code>) – The filters to apply.
For a detailed specification of the filters, refer to the
[documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering).
- **metadata_fields** (<code>list\[str\]</code>) – List of field names to count unique values for.
Field names can include or omit the "meta." prefix.
**Returns:**
- <code>dict\[str, int\]</code> – A dictionary mapping each metadata field name (without "meta." prefix)
to the count of its unique values among the filtered documents.
#### get_metadata_fields_info_async
```python
get_metadata_fields_info_async() -> dict[str, dict[str, str]]
```
Returns information about the metadata fields present in the stored documents.
Types are inferred from the stored values (keyword, int, float, boolean).
**Returns:**
- <code>dict\[str, dict\[str, str\]\]</code> – A dictionary mapping each metadata field name to a dict with a "type" key.
#### get_metadata_field_min_max_async
```python
get_metadata_field_min_max_async(metadata_field: str) -> dict[str, Any]
```
Returns the minimum and maximum values for the given metadata field across all documents.
**Parameters:**
- **metadata_field** (<code>str</code>) – The metadata field name. Can include or omit the "meta." prefix.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with "min" and "max" keys. Returns `{"min": None, "max": None}`
if the field is missing or has no values.
#### get_metadata_field_unique_values_async
```python
get_metadata_field_unique_values_async(
metadata_field: str, search_term: str | None = None
) -> tuple[list[str], int]
```
Returns unique values for a metadata field, optionally filtered by a search term in content.
**Parameters:**
- **metadata_field** (<code>str</code>) – The metadata field name. Can include or omit the "meta." prefix.
- **search_term** (<code>str | None</code>) – If set, only documents whose content contains this term (case-insensitive)
are considered.
**Returns:**
- <code>tuple\[list\[str\], int\]</code> – A tuple of (list of unique values, total count of unique values).
#### delete_all_documents_async
```python
delete_all_documents_async() -> None
```
Deletes all documents in the document store.
#### bm25_retrieval_async
```python
bm25_retrieval_async(
query: str,
filters: dict[str, Any] | None = None,
top_k: int = 10,
scale_score: bool = False,
) -> list[Document]
```
Retrieves documents that are most relevant to the query using BM25 algorithm.
**Parameters:**
- **query** (<code>str</code>) – The query string.
- **filters** (<code>dict\[str, Any\] | None</code>) – A dictionary with filters to narrow down the search space.
- **top_k** (<code>int</code>) – The number of top documents to retrieve. Default is 10.
- **scale_score** (<code>bool</code>) – Whether to scale the scores of the retrieved documents. Default is False.
**Returns:**
- <code>list\[Document\]</code> – A list of the top_k documents most relevant to the query.
#### embedding_retrieval_async
```python
embedding_retrieval_async(
query_embedding: list[float],
filters: dict[str, Any] | None = None,
top_k: int = 10,
scale_score: bool = False,
return_embedding: bool = False,
) -> list[Document]
```
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
**Parameters:**
- **query_embedding** (<code>list\[float\]</code>) – Embedding of the query.
- **filters** (<code>dict\[str, Any\] | None</code>) – A dictionary with filters to narrow down the search space.
- **top_k** (<code>int</code>) – The number of top documents to retrieve. Default is 10.
- **scale_score** (<code>bool</code>) – Whether to scale the scores of the retrieved Documents. Default is False.
- **return_embedding** (<code>bool</code>) – Whether to return the embedding of the retrieved Documents. Default is False.
**Returns:**
- <code>list\[Document\]</code> – A list of the top_k documents most relevant to the query.
docs-website/reference_versioned_docs/version-2.30/haystack-api/document_writers_api.md
+129
View File
@@ -0,0 +1,129 @@
---
title: "Document Writers"
id: document-writers-api
description: "Writes Documents to a DocumentStore."
slug: "/document-writers-api"
---
## document_writer
### DocumentWriter
Writes documents to a DocumentStore.
### Usage example
```python
from haystack import Document
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
docs = [
Document(content="Python is a popular programming language"),
]
doc_store = InMemoryDocumentStore()
writer = DocumentWriter(document_store=doc_store)
writer.run(docs)
```
#### __init__
```python
__init__(
document_store: DocumentStore,
policy: DuplicatePolicy = DuplicatePolicy.NONE,
) -> None
```
Create a DocumentWriter component.
**Parameters:**
- **document_store** (<code>DocumentStore</code>) – The instance of the document store where you want to store your documents.
- **policy** (<code>DuplicatePolicy</code>) – The policy to apply when a Document with the same ID already exists in the DocumentStore.
- `DuplicatePolicy.NONE`: Default policy, relies on the DocumentStore settings.
- `DuplicatePolicy.SKIP`: Skips documents with the same ID and doesn't write them to the DocumentStore.
- `DuplicatePolicy.OVERWRITE`: Overwrites documents with the same ID.
- `DuplicatePolicy.FAIL`: Raises an error if a Document with the same ID is already in the DocumentStore.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> DocumentWriter
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>DocumentWriter</code> – The deserialized component.
**Raises:**
- <code>DeserializationError</code> – If the document store is not properly specified in the serialization data or its type cannot be imported.
#### run
```python
run(
documents: list[Document], policy: DuplicatePolicy | None = None
) -> dict[str, int]
```
Run the DocumentWriter on the given input data.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of documents to write to the document store.
- **policy** (<code>DuplicatePolicy | None</code>) – The policy to use when encountering duplicate documents.
**Returns:**
- <code>dict\[str, int\]</code> – Number of documents written to the document store.
**Raises:**
- <code>ValueError</code> – If the specified document store is not found.
#### run_async
```python
run_async(
documents: list[Document], policy: DuplicatePolicy | None = None
) -> dict[str, int]
```
Asynchronously run the DocumentWriter on the given input data.
This is the asynchronous version of the `run` method. It has the same parameters and return values
but can be used with `await` in async code.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of documents to write to the document store.
- **policy** (<code>DuplicatePolicy | None</code>) – The policy to use when encountering duplicate documents.
**Returns:**
- <code>dict\[str, int\]</code> – Number of documents written to the document store.
**Raises:**
- <code>ValueError</code> – If the specified document store is not found.
- <code>TypeError</code> – If the specified document store does not implement `write_documents_async`.
docs-website/reference_versioned_docs/version-2.30/haystack-api/embedders_api.md
+1588
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/evaluation_api.md
+108
View File
@@ -0,0 +1,108 @@
---
title: "Evaluation"
id: evaluation-api
description: "Represents the results of evaluation."
slug: "/evaluation-api"
---
## eval_run_result
### EvaluationRunResult
Contains the inputs and the outputs of an evaluation pipeline and provides methods to inspect them.
#### __init__
```python
__init__(
run_name: str,
inputs: dict[str, list[Any]],
results: dict[str, dict[str, Any]],
) -> None
```
Initialize a new evaluation run result.
**Parameters:**
- **run_name** (<code>str</code>) – Name of the evaluation run.
- **inputs** (<code>dict\[str, list\[Any\]\]</code>) – Dictionary containing the inputs used for the run. Each key is the name of the input and its value is a list
of input values. The length of the lists should be the same.
- **results** (<code>dict\[str, dict\[str, Any\]\]</code>) – Dictionary containing the results of the evaluators used in the evaluation pipeline. Each key is the name
of the metric and its value is dictionary with the following keys:
- 'score': The aggregated score for the metric.
- 'individual_scores': A list of scores for each input sample.
#### aggregated_report
```python
aggregated_report(
output_format: Literal["json", "csv", "df"] = "json",
csv_file: str | None = None,
) -> Union[dict[str, list[Any]], DataFrame, str]
```
Generates a report with aggregated scores for each metric.
**Parameters:**
- **output_format** (<code>Literal['json', 'csv', 'df']</code>) – The output format for the report, "json", "csv", or "df", default to "json".
- **csv_file** (<code>str | None</code>) – Filepath to save CSV output if `output_format` is "csv", must be provided.
**Returns:**
- <code>Union\[dict\[str, list\[Any\]\], DataFrame, str\]</code> – JSON or DataFrame with aggregated scores, in case the output is set to a CSV file, a message confirming the
successful write or an error message.
#### detailed_report
```python
detailed_report(
output_format: Literal["json", "csv", "df"] = "json",
csv_file: str | None = None,
) -> Union[dict[str, list[Any]], DataFrame, str]
```
Generates a report with detailed scores for each metric.
**Parameters:**
- **output_format** (<code>Literal['json', 'csv', 'df']</code>) – The output format for the report, "json", "csv", or "df", default to "json".
- **csv_file** (<code>str | None</code>) – Filepath to save CSV output if `output_format` is "csv", must be provided.
**Returns:**
- <code>Union\[dict\[str, list\[Any\]\], DataFrame, str\]</code> – JSON or DataFrame with the detailed scores, in case the output is set to a CSV file, a message confirming
the successful write or an error message.
#### comparative_detailed_report
```python
comparative_detailed_report(
other: EvaluationRunResult,
keep_columns: list[str] | None = None,
output_format: Literal["json", "csv", "df"] = "json",
csv_file: str | None = None,
) -> Union[str, DataFrame, None]
```
Generates a report with detailed scores for each metric from two evaluation runs for comparison.
**Parameters:**
- **other** (<code>EvaluationRunResult</code>) – Results of another evaluation run to compare with.
- **keep_columns** (<code>list\[str\] | None</code>) – List of common column names to keep from the inputs of the evaluation runs to compare.
- **output_format** (<code>Literal['json', 'csv', 'df']</code>) – The output format for the report, "json", "csv", or "df", default to "json".
- **csv_file** (<code>str | None</code>) – Filepath to save CSV output if `output_format` is "csv", must be provided.
**Returns:**
- <code>Union\[str, DataFrame, None\]</code> – JSON or DataFrame with a comparison of the detailed scores, in case the output is set to a CSV file,
a message confirming the successful write or an error message.
**Raises:**
- <code>TypeError</code> – If `other` is not an EvaluationRunResult instance, or if the detailed reports are not
dictionaries.
- <code>ValueError</code> – If the `other` parameter is missing required attributes.
docs-website/reference_versioned_docs/version-2.30/haystack-api/evaluators_api.md
+1057
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/extractors_api.md
+693
View File
@@ -0,0 +1,693 @@
---
title: "Extractors"
id: extractors-api
description: "Components to extract specific elements from textual data."
slug: "/extractors-api"
---
## image/llm_document_content_extractor
### LLMDocumentContentExtractor
Extracts textual content and optionally metadata from image-based documents using a vision-enabled LLM.
One prompt and one LLM call per document. The component converts each document to an image via
DocumentToImageContent and sends it to the ChatGenerator. The prompt must not contain Jinja variables.
Response handling:
- If the LLM returns a **plain string** (non-JSON or not a JSON object), it is written to the document's content.
- If the LLM returns a **JSON object with only the key** `document_content`, that value is written to content.
- If the LLM returns a **JSON object with multiple keys**, the value of `document_content` (if present) is
written to content and all other keys are merged into the document's metadata.
The ChatGenerator can be configured to return JSON (e.g. `response_format={"type": "json_object"}`
in `generation_kwargs`).
Documents that fail extraction are returned in `failed_documents` with `content_extraction_error` in metadata.
### Usage example
```python
from haystack import Document
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.extractors.image import LLMDocumentContentExtractor
prompt = """
Extract the content from the provided image.
Format everything as markdown. Return only the extracted content as a JSON object with the key 'document_content'.
No markdown, no code fence, only raw JSON.
Extract metadata about the image like source of the image, date of creation, etc. if you can.
Return this metadata as additional key-value pairs in the same JSON object.
"""
chat_generator = OpenAIChatGenerator(
generation_kwargs={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "entity_extraction",
"schema": {
"type": "object",
"properties": {
"document_content": {"type": "string"},
"author": {"type": "string"},
"date": {"type": "string"},
"document_type": {"type": "string"},
"title": {"type": "string"},
},
"additionalProperties": False,
},
},
}
}
)
extractor = LLMDocumentContentExtractor(
chat_generator=chat_generator,
file_path_meta_field="file_path",
raise_on_failure=False
)
documents = [
Document(content="", meta={"file_path": "test/test_files/images/image_metadata.png"}),
Document(content="", meta={"file_path": "test/test_files/images/apple.jpg", "page_number": 1})
]
result = extractor.run(documents=documents)
updated_documents = result["documents"]
```
#### __init__
```python
__init__(
*,
chat_generator: ChatGenerator,
prompt: str = DEFAULT_PROMPT_TEMPLATE,
file_path_meta_field: str = "file_path",
root_path: str | None = None,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None,
raise_on_failure: bool = False,
max_workers: int = 3
) -> None
```
Initialize the LLMDocumentContentExtractor component.
**Parameters:**
- **chat_generator** (<code>ChatGenerator</code>) – A ChatGenerator that supports vision input. Optionally configured for JSON
(e.g. `response_format={"type": "json_object"}` in `generation_kwargs`).
- **prompt** (<code>str</code>) – Prompt for extraction. Must not contain Jinja variables.
- **file_path_meta_field** (<code>str</code>) – The metadata field in the Document that contains the file path to the image or PDF.
- **root_path** (<code>str | None</code>) – The root directory path where document files are located. If provided, file paths in
document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within (width, height) while keeping aspect ratio.
- **raise_on_failure** (<code>bool</code>) – If True, exceptions from the LLM are raised. If False, failed documents are returned.
- **max_workers** (<code>int</code>) – Maximum number of threads for parallel LLM calls.
#### warm_up
```python
warm_up() -> None
```
Warm up the ChatGenerator if it has a warm_up method.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> LLMDocumentContentExtractor
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary with serialized data.
**Returns:**
- <code>LLMDocumentContentExtractor</code> – An instance of the component.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Run extraction on image-based documents. One LLM call per document.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of image-based documents to process. Each must have a valid file path in its metadata.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with "documents" (successfully processed) and "failed_documents" (with failure metadata).
## llm_metadata_extractor
### LLMMetadataExtractor
Extracts metadata from documents using a Large Language Model (LLM).
The metadata is extracted by providing a prompt to an LLM that generates the metadata.
This component expects as input a list of documents and a prompt. The prompt must have exactly one variable, called
`document`, that points to a single document in the list of documents. So to access the content of the document,
you can use `{{ document.content }}` in the prompt.
The component will run the LLM on each document in the list and extract metadata from the document. The metadata
will be added to the document's metadata field. If the LLM fails to extract metadata from a document, the document
will be added to the `failed_documents` list. The failed documents will have the keys `metadata_extraction_error` and
`metadata_extraction_response` in their metadata. These documents can be re-run with another extractor to
extract metadata by using the `metadata_extraction_response` and `metadata_extraction_error` in the prompt.
```python
from haystack import Document
from haystack.components.extractors.llm_metadata_extractor import LLMMetadataExtractor
from haystack.components.generators.chat import OpenAIChatGenerator
NER_PROMPT = '''
-Goal-
Given text and a list of entity types, identify all entities of those types from the text.
-Steps-
1. Identify all entities. For each identified entity, extract the following information:
- entity: Name of the entity
- entity_type: One of the following types: [organization, product, service, industry]
Format each entity as a JSON like: {"entity": <entity_name>, "entity_type": <entity_type>}
2. Return output in a single list with all the entities identified in steps 1.
-Examples-
######################
Example 1:
entity_types: [organization, person, partnership, financial metric, product, service, industry, investment strategy, market trend]
text: Another area of strength is our co-brand issuance. Visa is the primary network partner for eight of the top
10 co-brand partnerships in the US today and we are pleased that Visa has finalized a multi-year extension of
our successful credit co-branded partnership with Alaska Airlines, a portfolio that benefits from a loyal customer
base and high cross-border usage.
We have also had significant co-brand momentum in CEMEA. First, we launched a new co-brand card in partnership
with Qatar Airways, British Airways and the National Bank of Kuwait. Second, we expanded our strong global
Marriott relationship to launch Qatar's first hospitality co-branded card with Qatar Islamic Bank. Across the
United Arab Emirates, we now have exclusive agreements with all the leading airlines marked by a recent
agreement with Emirates Skywards.
And we also signed an inaugural Airline co-brand agreement in Morocco with Royal Air Maroc. Now newer digital
issuers are equally
------------------------
output:
{"entities": [{"entity": "Visa", "entity_type": "company"}, {"entity": "Alaska Airlines", "entity_type": "company"}, {"entity": "Qatar Airways", "entity_type": "company"}, {"entity": "British Airways", "entity_type": "company"}, {"entity": "National Bank of Kuwait", "entity_type": "company"}, {"entity": "Marriott", "entity_type": "company"}, {"entity": "Qatar Islamic Bank", "entity_type": "company"}, {"entity": "Emirates Skywards", "entity_type": "company"}, {"entity": "Royal Air Maroc", "entity_type": "company"}]}
#############################
-Real Data-
######################
entity_types: [company, organization, person, country, product, service]
text: {{ document.content }}
######################
output:
'''
docs = [
Document(content="deepset was founded in 2018 in Berlin, and is known for its Haystack framework"),
Document(content="Hugging Face is a company that was founded in New York, USA and is known for its Transformers library")
]
chat_generator = OpenAIChatGenerator(
generation_kwargs={
"max_completion_tokens": 500,
"temperature": 0.0,
"seed": 0,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "entity_extraction",
"schema": {
"type": "object",
"properties": {
"entities": {
"type": "array",
"items": {
"type": "object",
"properties": {
"entity": {"type": "string"},
"entity_type": {"type": "string"}
},
"required": ["entity", "entity_type"],
"additionalProperties": False
}
}
},
"required": ["entities"],
"additionalProperties": False
}
}
},
},
max_retries=1,
timeout=60.0,
)
extractor = LLMMetadataExtractor(
prompt=NER_PROMPT,
chat_generator=chat_generator,
expected_keys=["entities"],
raise_on_failure=False,
)
extractor.run(documents=docs)
# >> {'documents': [
# Document(id=.., content: 'deepset was founded in 2018 in Berlin, and is known for its Haystack framework',
# meta: {'entities': [{'entity': 'deepset', 'entity_type': 'company'}, {'entity': 'Berlin', 'entity_type': 'city'},
# {'entity': 'Haystack', 'entity_type': 'product'}]}),
# Document(id=.., content: 'Hugging Face is a company that was founded in New York, USA and is known for its Transformers library',
# meta: {'entities': [
# {'entity': 'Hugging Face', 'entity_type': 'company'}, {'entity': 'New York', 'entity_type': 'city'},
# {'entity': 'USA', 'entity_type': 'country'}, {'entity': 'Transformers', 'entity_type': 'product'}
# ]})
# ]
# 'failed_documents': []
# }
# >>
```
#### __init__
```python
__init__(
prompt: str,
chat_generator: ChatGenerator,
expected_keys: list[str] | None = None,
page_range: list[str | int] | None = None,
raise_on_failure: bool = False,
max_workers: int = 3,
) -> None
```
Initializes the LLMMetadataExtractor.
**Parameters:**
- **prompt** (<code>str</code>) – The prompt to be used for the LLM. It must contain exactly one variable, called `document`,
which points to a single document in the list of documents. For example, to access the content of the
document, use `{{ document.content }}` in the prompt.
- **chat_generator** (<code>ChatGenerator</code>) – a ChatGenerator instance which represents the LLM. In order for the component to work,
the LLM should be configured to return a JSON object. For example, when using the OpenAIChatGenerator, you
should pass `{"response_format": {"type": "json_object"}}` in the `generation_kwargs`.
- **expected_keys** (<code>list\[str\] | None</code>) – The keys expected in the JSON output from the LLM.
- **page_range** (<code>list\[str | int\] | None</code>) – A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
metadata from the first and third pages of each document. It also accepts printable range strings, e.g.:
['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,11, 12.
If None, metadata will be extracted from the entire document for each document in the documents list.
This parameter is optional and can be overridden in the `run` method.
- **raise_on_failure** (<code>bool</code>) – Whether to raise an error on failure during the execution of the Generator or
validation of the JSON output.
- **max_workers** (<code>int</code>) – The maximum number of workers to use in the thread pool executor.
This parameter is used limit the maximum number of requests that should be allowed to run concurrently
when using the `run_async` method.
#### warm_up
```python
warm_up() -> None
```
Warm up the LLM provider component.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> LLMMetadataExtractor
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary with serialized data.
**Returns:**
- <code>LLMMetadataExtractor</code> – An instance of the component.
#### run
```python
run(
documents: list[Document], page_range: list[str | int] | None = None
) -> dict[str, Any]
```
Extract metadata from documents using a Large Language Model.
If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
extracted from the entire document if `page_range` is not provided.
The original documents will be returned updated with the extracted metadata.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of documents to extract metadata from.
- **page_range** (<code>list\[str | int\] | None</code>) – A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
metadata from the first and third pages of each document. It also accepts printable range
strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
11, 12.
If None, metadata will be extracted from the entire document for each document in the
documents list.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the keys:
- "documents": A list of documents that were successfully updated with the extracted metadata.
- "failed_documents": A list of documents that failed to extract metadata. These documents will have
"metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
re-run with the extractor to extract metadata.
#### run_async
```python
run_async(
documents: list[Document], page_range: list[str | int] | None = None
) -> dict[str, Any]
```
Asynchronously extract metadata from documents using a Large Language Model.
If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
extracted from the entire document if `page_range` is not provided.
The original documents will be returned updated with the extracted metadata.
This is the asynchronous version of the `run` method. It has the same parameters
and return values but can be used with `await` in an async code.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of documents to extract metadata from.
- **page_range** (<code>list\[str | int\] | None</code>) – A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
metadata from the first and third pages of each document. It also accepts printable range
strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
11, 12.
If None, metadata will be extracted from the entire document for each document in the
documents list.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the keys:
- "documents": A list of documents that were successfully updated with the extracted metadata.
- "failed_documents": A list of documents that failed to extract metadata. These documents will have
"metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
re-run with the extractor to extract metadata.
## named_entity_extractor
### NamedEntityExtractorBackend
Bases: <code>Enum</code>
NLP backend to use for Named Entity Recognition.
#### from_str
```python
from_str(string: str) -> NamedEntityExtractorBackend
```
Convert a string to a NamedEntityExtractorBackend enum.
### NamedEntityAnnotation
Describes a single NER annotation.
**Parameters:**
- **entity** (<code>str</code>) – Entity label.
- **start** (<code>int</code>) – Start index of the entity in the document.
- **end** (<code>int</code>) – End index of the entity in the document.
- **score** (<code>float | None</code>) – Score calculated by the model.
### NamedEntityExtractor
Annotates named entities in a collection of documents.
The component supports two backends: Hugging Face and spaCy. The
former can be used with any sequence classification model from the
[Hugging Face model hub](https://huggingface.co/models), while the
latter can be used with any [spaCy model](https://spacy.io/models)
that contains an NER component. Annotations are stored as metadata
in the documents.
Usage example:
<!-- test-ignore -->
```python
from haystack import Document
from haystack.components.extractors.named_entity_extractor import NamedEntityExtractor
documents = [
Document(content="I'm Merlin, the happy pig!"),
Document(content="My name is Clara and I live in Berkeley, California."),
]
extractor = NamedEntityExtractor(backend="hugging_face", model="dslim/bert-base-NER")
results = extractor.run(documents=documents)["documents"]
annotations = [NamedEntityExtractor.get_stored_annotations(doc) for doc in results]
print(annotations)
```
#### __init__
```python
__init__(
*,
backend: str | NamedEntityExtractorBackend,
model: str,
pipeline_kwargs: dict[str, Any] | None = None,
device: ComponentDevice | None = None,
token: Secret | None = Secret.from_env_var(
["HF_API_TOKEN", "HF_TOKEN"], strict=False
)
) -> None
```
Create a Named Entity extractor component.
**Parameters:**
- **backend** (<code>str | NamedEntityExtractorBackend</code>) – Backend to use for NER.
- **model** (<code>str</code>) – Name of the model or a path to the model on
the local disk. Dependent on the backend.
- **pipeline_kwargs** (<code>dict\[str, Any\] | None</code>) – Keyword arguments passed to the pipeline. The
pipeline can override these arguments. Dependent on the backend.
- **device** (<code>ComponentDevice | None</code>) – The device on which the model is loaded. If `None`,
the default device is automatically selected. If a
device/device map is specified in `pipeline_kwargs`,
it overrides this parameter (only applicable to the
HuggingFace backend).
- **token** (<code>Secret | None</code>) – The API token to download private models from Hugging Face.
#### warm_up
```python
warm_up() -> None
```
Initialize the component.
**Raises:**
- <code>ComponentError</code> – If the backend fails to initialize successfully.
#### run
```python
run(documents: list[Document], batch_size: int = 1) -> dict[str, Any]
```
Annotate named entities in each document and store the annotations in the document's metadata.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – Documents to process.
- **batch_size** (<code>int</code>) – Batch size used for processing the documents.
**Returns:**
- <code>dict\[str, Any\]</code> – Processed documents.
**Raises:**
- <code>ComponentError</code> – If the backend fails to process a document.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> NamedEntityExtractor
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>NamedEntityExtractor</code> – Deserialized component.
#### initialized
```python
initialized: bool
```
Returns if the extractor is ready to annotate text.
#### get_stored_annotations
```python
get_stored_annotations(
document: Document,
) -> list[NamedEntityAnnotation] | None
```
Returns the document's named entity annotations stored in its metadata, if any.
**Parameters:**
- **document** (<code>Document</code>) – Document whose annotations are to be fetched.
**Returns:**
- <code>list\[NamedEntityAnnotation\] | None</code> – The stored annotations.
## regex_text_extractor
### RegexTextExtractor
Extracts text from chat message or string input using a regex pattern.
RegexTextExtractor parses input text or ChatMessages using a provided regular expression pattern.
It can be configured to search through all messages or only the last message in a list of ChatMessages.
### Usage example
```python
from haystack.components.extractors import RegexTextExtractor
from haystack.dataclasses import ChatMessage
# Using with a string
parser = RegexTextExtractor(regex_pattern='<issue url="(.+)">')
result = parser.run(text_or_messages='<issue url="github.com/hahahaha">hahahah</issue>')
# result: {"captured_text": "github.com/hahahaha"}
# Using with ChatMessages
messages = [ChatMessage.from_user('<issue url="github.com/hahahaha">hahahah</issue>')]
result = parser.run(text_or_messages=messages)
# result: {"captured_text": "github.com/hahahaha"}
```
#### __init__
```python
__init__(regex_pattern: str) -> None
```
Creates an instance of the RegexTextExtractor component.
**Parameters:**
- **regex_pattern** (<code>str</code>) – The regular expression pattern used to extract text.
The pattern should include a capture group to extract the desired text.
Example: `'<issue url="(.+)">'` captures `'github.com/hahahaha'` from `'<issue url="github.com/hahahaha">'`.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> RegexTextExtractor
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>RegexTextExtractor</code> – The deserialized component.
#### run
```python
run(text_or_messages: str | list[ChatMessage]) -> dict[str, str]
```
Extracts text from input using the configured regex pattern.
**Parameters:**
- **text_or_messages** (<code>str | list\[ChatMessage\]</code>) – Either a string or a list of ChatMessage objects to search through.
**Returns:**
- <code>dict\[str, str\]</code> – - `{"captured_text": "matched text"}` if a match is found
- `{"captured_text": ""}` if no match is found
**Raises:**
- <code>TypeError</code> – if receiving a list the last element is not a ChatMessage instance.
docs-website/reference_versioned_docs/version-2.30/haystack-api/fetchers_api.md
+121
View File
@@ -0,0 +1,121 @@
---
title: "Fetchers"
id: fetchers-api
description: "Fetches content from a list of URLs and returns a list of extracted content streams."
slug: "/fetchers-api"
---
## link_content
### LinkContentFetcher
Fetches and extracts content from URLs.
It supports various content types, retries on failures, and automatic user-agent rotation for failed web
requests. Use it as the data-fetching step in your pipelines.
You may need to convert LinkContentFetcher's output into a list of documents. Use HTMLToDocument
converter to do this.
### Usage example
```python
from haystack.components.fetchers.link_content import LinkContentFetcher
fetcher = LinkContentFetcher()
streams = fetcher.run(urls=["https://www.google.com"])["streams"]
assert len(streams) == 1
assert streams[0].meta == {'content_type': 'text/html', 'url': 'https://www.google.com'}
assert streams[0].data
```
For async usage:
```python
import asyncio
from haystack.components.fetchers import LinkContentFetcher
async def fetch_async():
fetcher = LinkContentFetcher()
result = await fetcher.run_async(urls=["https://www.google.com"])
return result["streams"]
streams = asyncio.run(fetch_async())
```
#### __init__
```python
__init__(
raise_on_failure: bool = True,
user_agents: list[str] | None = None,
retry_attempts: int = 2,
timeout: int = 3,
http2: bool = False,
client_kwargs: dict | None = None,
request_headers: dict[str, str] | None = None,
) -> None
```
Initializes the component.
**Parameters:**
- **raise_on_failure** (<code>bool</code>) – If `True`, raises an exception if it fails to fetch a single URL.
For multiple URLs, it logs errors and returns the content it successfully fetched.
- **user_agents** (<code>list\[str\] | None</code>) – [User agents](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent)
for fetching content. If `None`, a default user agent is used.
- **retry_attempts** (<code>int</code>) – The number of times to retry to fetch the URL's content.
- **timeout** (<code>int</code>) – Timeout in seconds for the request.
- **http2** (<code>bool</code>) – Whether to enable HTTP/2 support for requests. Defaults to False.
Requires the 'h2' package to be installed (via `pip install httpx[http2]`).
- **client_kwargs** (<code>dict | None</code>) – Additional keyword arguments to pass to the httpx client.
If `None`, default values are used.
#### run
```python
run(urls: list[str]) -> dict[str, Any]
```
Fetches content from a list of URLs and returns a list of extracted content streams.
Each content stream is a `ByteStream` object containing the extracted content as binary data.
Each ByteStream object in the returned list corresponds to the contents of a single URL.
The content type of each stream is stored in the metadata of the ByteStream object under
the key "content_type". The URL of the fetched content is stored under the key "url".
**Parameters:**
- **urls** (<code>list\[str\]</code>) – A list of URLs to fetch content from.
**Returns:**
- <code>dict\[str, Any\]</code> – `ByteStream` objects representing the extracted content.
**Raises:**
- <code>Exception</code> – If the provided list of URLs contains only a single URL, and `raise_on_failure` is set to
`True`, an exception will be raised in case of an error during content retrieval.
In all other scenarios, any retrieval errors are logged, and a list of successfully retrieved `ByteStream`
objects is returned.
#### run_async
```python
run_async(urls: list[str]) -> dict[str, Any]
```
Asynchronously fetches content from a list of URLs and returns a list of extracted content streams.
This is the asynchronous version of the `run` method with the same parameters and return values.
**Parameters:**
- **urls** (<code>list\[str\]</code>) – A list of URLs to fetch content from.
**Returns:**
- <code>dict\[str, Any\]</code> – `ByteStream` objects representing the extracted content.
docs-website/reference_versioned_docs/version-2.30/haystack-api/generators_api.md
+2443
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/human_in_the_loop_api.md
+362
View File
@@ -0,0 +1,362 @@
---
title: "Human-in-the-Loop"
id: human-in-the-loop-api
description: "Abstractions for integrating human feedback and interaction into Agent workflows."
slug: "/human-in-the-loop-api"
---
## dataclasses
### ConfirmationUIResult
Result of the confirmation UI interaction.
**Parameters:**
- **action** (<code>str</code>) – The action taken by the user such as "confirm", "reject", or "modify".
This action type is not enforced to allow for custom actions to be implemented.
- **feedback** (<code>str | None</code>) – Optional feedback message from the user. For example, if the user rejects the tool execution,
they might provide a reason for the rejection.
- **new_tool_params** (<code>dict\[str, Any\] | None</code>) – Optional set of new parameters for the tool. For example, if the user chooses to modify the tool parameters,
they can provide a new set of parameters here.
### ToolExecutionDecision
Decision made regarding tool execution.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **execute** (<code>bool</code>) – A boolean indicating whether to execute the tool with the provided parameters.
- **tool_call_id** (<code>str | None</code>) – Optional unique identifier for the tool call. This can be used to track and correlate the decision with a
specific tool invocation.
- **feedback** (<code>str | None</code>) – Optional feedback message.
For example, if the tool execution is rejected, this can contain the reason. Or if the tool parameters were
modified, this can contain the modification details.
- **final_tool_params** (<code>dict\[str, Any\] | None</code>) – Optional final parameters for the tool if execution is confirmed or modified.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Convert the ToolExecutionDecision to a dictionary representation.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary containing the tool execution decision details.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> ToolExecutionDecision
```
Populate the ToolExecutionDecision from a dictionary representation.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – A dictionary containing the tool execution decision details.
**Returns:**
- <code>ToolExecutionDecision</code> – An instance of ToolExecutionDecision.
## policies
### AlwaysAskPolicy
Bases: <code>ConfirmationPolicy</code>
Always ask for confirmation.
#### should_ask
```python
should_ask(
tool_name: str, tool_description: str, tool_params: dict[str, Any]
) -> bool
```
Always ask for confirmation before executing the tool.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
**Returns:**
- <code>bool</code> – Always returns True, indicating confirmation is needed.
### NeverAskPolicy
Bases: <code>ConfirmationPolicy</code>
Never ask for confirmation.
#### should_ask
```python
should_ask(
tool_name: str, tool_description: str, tool_params: dict[str, Any]
) -> bool
```
Never ask for confirmation, always proceed with tool execution.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
**Returns:**
- <code>bool</code> – Always returns False, indicating no confirmation is needed.
### AskOncePolicy
Bases: <code>ConfirmationPolicy</code>
Ask only once per tool with specific parameters.
#### __init__
```python
__init__() -> None
```
Creates an instance of AskOncePolicy.
#### should_ask
```python
should_ask(
tool_name: str, tool_description: str, tool_params: dict[str, Any]
) -> bool
```
Ask for confirmation only once per tool with specific parameters.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
**Returns:**
- <code>bool</code> – True if confirmation is needed, False if already asked with the same parameters.
#### update_after_confirmation
```python
update_after_confirmation(
tool_name: str,
tool_description: str,
tool_params: dict[str, Any],
confirmation_result: ConfirmationUIResult,
) -> None
```
Store the tool and parameters if the action was "confirm" to avoid asking again.
This method updates the internal state to remember that the user has already confirmed the execution of the
tool with the given parameters.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool that was executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters that were passed to the tool.
- **confirmation_result** (<code>ConfirmationUIResult</code>) – The result from the confirmation UI.
## strategies
### BlockingConfirmationStrategy
Confirmation strategy that blocks execution to gather user feedback.
#### __init__
```python
__init__(
*,
confirmation_policy: ConfirmationPolicy,
confirmation_ui: ConfirmationUI,
reject_template: str = REJECTION_FEEDBACK_TEMPLATE,
modify_template: str = MODIFICATION_FEEDBACK_TEMPLATE,
user_feedback_template: str = USER_FEEDBACK_TEMPLATE
) -> None
```
Initialize the BlockingConfirmationStrategy with a confirmation policy and UI.
**Parameters:**
- **confirmation_policy** (<code>ConfirmationPolicy</code>) – The confirmation policy to determine when to ask for user confirmation.
- **confirmation_ui** (<code>ConfirmationUI</code>) – The user interface to interact with the user for confirmation.
- **reject_template** (<code>str</code>) – Template for rejection feedback messages. It should include a `{tool_name}` placeholder.
- **modify_template** (<code>str</code>) – Template for modification feedback messages. It should include `{tool_name}` and `{final_tool_params}`
placeholders.
- **user_feedback_template** (<code>str</code>) – Template for user feedback messages. It should include a `{feedback}` placeholder.
#### run
```python
run(
*,
tool_name: str,
tool_description: str,
tool_params: dict[str, Any],
tool_call_id: str | None = None,
confirmation_strategy_context: dict[str, Any] | None = None
) -> ToolExecutionDecision
```
Run the human-in-the-loop strategy for a given tool and its parameters.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
- **tool_call_id** (<code>str | None</code>) – Optional unique identifier for the tool call. This can be used to track and correlate the decision with a
specific tool invocation.
- **confirmation_strategy_context** (<code>dict\[str, Any\] | None</code>) – Optional dictionary for passing request-scoped resources. Useful in web/server environments
to provide per-request objects (e.g., WebSocket connections, async queues, Redis pub/sub clients)
that strategies can use for non-blocking user interaction.
**Returns:**
- <code>ToolExecutionDecision</code> – A ToolExecutionDecision indicating whether to execute the tool with the given parameters, or a
feedback message if rejected.
#### run_async
```python
run_async(
*,
tool_name: str,
tool_description: str,
tool_params: dict[str, Any],
tool_call_id: str | None = None,
confirmation_strategy_context: dict[str, Any] | None = None
) -> ToolExecutionDecision
```
Async version of run. Calls the sync run() method by default.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
- **tool_call_id** (<code>str | None</code>) – Optional unique identifier for the tool call.
- **confirmation_strategy_context** (<code>dict\[str, Any\] | None</code>) – Optional dictionary for passing request-scoped resources.
**Returns:**
- <code>ToolExecutionDecision</code> – A ToolExecutionDecision indicating whether to execute the tool with the given parameters.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the BlockingConfirmationStrategy to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> BlockingConfirmationStrategy
```
Deserializes the BlockingConfirmationStrategy from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>BlockingConfirmationStrategy</code> – Deserialized BlockingConfirmationStrategy.
## user_interfaces
### RichConsoleUI
Bases: <code>ConfirmationUI</code>
Rich console interface for user interaction.
#### __init__
```python
__init__(console: Console | None = None) -> None
```
Creates an instance of RichConsoleUI.
#### get_user_confirmation
```python
get_user_confirmation(
tool_name: str, tool_description: str, tool_params: dict[str, Any]
) -> ConfirmationUIResult
```
Get user confirmation for tool execution via rich console prompts.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
**Returns:**
- <code>ConfirmationUIResult</code> – ConfirmationUIResult based on user input.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the RichConsoleConfirmationUI to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
### SimpleConsoleUI
Bases: <code>ConfirmationUI</code>
Simple console interface using standard input/output.
#### get_user_confirmation
```python
get_user_confirmation(
tool_name: str, tool_description: str, tool_params: dict[str, Any]
) -> ConfirmationUIResult
```
Get user confirmation for tool execution via simple console prompts.
**Parameters:**
- **tool_name** (<code>str</code>) – The name of the tool to be executed.
- **tool_description** (<code>str</code>) – The description of the tool.
- **tool_params** (<code>dict\[str, Any\]</code>) – The parameters to be passed to the tool.
docs-website/reference_versioned_docs/version-2.30/haystack-api/image_converters_api.md
+353
View File
@@ -0,0 +1,353 @@
---
title: "Image Converters"
id: image-converters-api
description: "Various converters to transform image data from one format to another."
slug: "/image-converters-api"
---
## document_to_image
### DocumentToImageContent
Converts documents sourced from PDF and image files into ImageContents.
This component processes a list of documents and extracts visual content from supported file formats, converting
them into ImageContents that can be used for multimodal AI tasks. It handles both direct image files and PDF
documents by extracting specific pages as images.
Documents are expected to have metadata containing:
- The `file_path_meta_field` key with a valid file path that exists when combined with `root_path`
- A supported image format (MIME type must be one of the supported image types)
- For PDF files, a `page_number` key specifying which page to extract
### Usage example
<!-- test-ignore -->
```python
from haystack import Document
from haystack.components.converters.image.document_to_image import DocumentToImageContent
converter = DocumentToImageContent(
file_path_meta_field="file_path",
root_path="/data/files",
detail="high",
size=(800, 600)
)
documents = [
Document(content="Optional description of image.jpg", meta={"file_path": "image.jpg"}),
Document(content="Text content of page 1 of doc.pdf", meta={"file_path": "doc.pdf", "page_number": 1})
]
result = converter.run(documents)
image_contents = result["image_contents"]
# [ImageContent(
# base64_image='/9j/4A...', mime_type='image/jpeg', detail='high', meta={'file_path': 'image.jpg'}
# ),
# ImageContent(
# base64_image='/9j/4A...', mime_type='image/jpeg', detail='high',
# meta={'page_number': 1, 'file_path': 'doc.pdf'}
# )]
```
#### __init__
```python
__init__(
*,
file_path_meta_field: str = "file_path",
root_path: str | None = None,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None
) -> None
```
Initialize the DocumentToImageContent component.
**Parameters:**
- **file_path_meta_field** (<code>str</code>) – The metadata field in the Document that contains the file path to the image or PDF.
- **root_path** (<code>str | None</code>) – The root directory path where document files are located. If provided, file paths in
document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
This will be passed to the created ImageContent objects.
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within the specified dimensions (width, height) while
maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
when working with models that have resolution constraints or when transmitting images to remote services.
#### run
```python
run(documents: list[Document]) -> dict[str, list[ImageContent | None]]
```
Convert documents with image or PDF sources into ImageContent objects.
This method processes the input documents, extracting images from supported file formats and converting them
into ImageContent objects.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of documents to process. Each document should have metadata containing at minimum
a 'file_path_meta_field' key. PDF documents additionally require a 'page_number' key to specify which
page to convert.
**Returns:**
- <code>dict\[str, list\[ImageContent | None\]\]</code> – Dictionary containing one key:
- "image_contents": ImageContents created from the processed documents. These contain base64-encoded image
data and metadata. The order corresponds to order of input documents.
**Raises:**
- <code>ValueError</code> – If any document is missing the required metadata keys, has an invalid file path, or has an unsupported
MIME type. The error message will specify which document and what information is missing or incorrect.
## file_to_document
### ImageFileToDocument
Converts image file references into empty Document objects with associated metadata.
This component is useful in pipelines where image file paths need to be wrapped in `Document` objects to be
processed by downstream components such as the `SentenceTransformersImageDocumentEmbedder`.
It does **not** extract any content from the image files, instead it creates `Document` objects with `None` as
their content and attaches metadata such as file path and any user-provided values.
### Usage example
```python
from haystack.components.converters.image import ImageFileToDocument
converter = ImageFileToDocument()
sources = ["image.jpg", "another_image.png"]
result = converter.run(sources=sources)
documents = result["documents"]
print(documents)
# [Document(id=..., meta: {'file_path': 'image.jpg'}),
# Document(id=..., meta: {'file_path': 'another_image.png'})]
```
#### __init__
```python
__init__(*, store_full_path: bool = False) -> None
```
Initialize the ImageFileToDocument component.
**Parameters:**
- **store_full_path** (<code>bool</code>) – If True, the full path of the file is stored in the metadata of the document.
If False, only the file name is stored.
#### run
```python
run(
*,
sources: list[str | Path | ByteStream],
meta: dict[str, Any] | list[dict[str, Any]] | None = None
) -> dict[str, list[Document]]
```
Convert image files into empty Document objects with metadata.
This method accepts image file references (as file paths or ByteStreams) and creates `Document` objects
without content. These documents are enriched with metadata derived from the input source and optional
user-provided metadata.
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – List of file paths or ByteStream objects to convert.
- **meta** (<code>dict\[str, Any\] | list\[dict\[str, Any\]\] | None</code>) – Optional metadata to attach to the documents.
This value can be a list of dictionaries or a single dictionary.
If it's a single dictionary, its content is added to the metadata of all produced documents.
If it's a list, its length must match the number of sources, as they are zipped together.
For ByteStream objects, their `meta` is added to the output documents.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary containing:
- `documents`: A list of `Document` objects with empty content and associated metadata.
## file_to_image
### ImageFileToImageContent
Converts image files to ImageContent objects.
### Usage example
```python
from haystack.components.converters.image import ImageFileToImageContent
converter = ImageFileToImageContent()
sources = ["image.jpg", "another_image.png"]
image_contents = converter.run(sources=sources)["image_contents"]
print(image_contents)
# [ImageContent(base64_image='...',
# mime_type='image/jpeg',
# detail=None,
# meta={'file_path': 'image.jpg'}),
# ...]
```
#### __init__
```python
__init__(
*,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None
) -> None
```
Create the ImageFileToImageContent component.
**Parameters:**
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
This will be passed to the created ImageContent objects.
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within the specified dimensions (width, height) while
maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
when working with models that have resolution constraints or when transmitting images to remote services.
#### run
```python
run(
sources: list[str | Path | ByteStream],
meta: dict[str, Any] | list[dict[str, Any]] | None = None,
*,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None
) -> dict[str, list[ImageContent]]
```
Converts files to ImageContent objects.
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – List of file paths or ByteStream objects to convert.
- **meta** (<code>dict\[str, Any\] | list\[dict\[str, Any\]\] | None</code>) – Optional metadata to attach to the ImageContent objects.
This value can be a list of dictionaries or a single dictionary.
If it's a single dictionary, its content is added to the metadata of all produced ImageContent objects.
If it's a list, its length must match the number of sources as they're zipped together.
For ByteStream objects, their `meta` is added to the output ImageContent objects.
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
This will be passed to the created ImageContent objects.
If not provided, the detail level will be the one set in the constructor.
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within the specified dimensions (width, height) while
maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
when working with models that have resolution constraints or when transmitting images to remote services.
If not provided, the size value will be the one set in the constructor.
**Returns:**
- <code>dict\[str, list\[ImageContent\]\]</code> – A dictionary with the following keys:
- `image_contents`: A list of ImageContent objects.
## pdf_to_image
### PDFToImageContent
Converts PDF files to ImageContent objects.
### Usage example
```python
from haystack.components.converters.image import PDFToImageContent
converter = PDFToImageContent()
sources = ["file.pdf", "another_file.pdf"]
image_contents = converter.run(sources=sources)["image_contents"]
print(image_contents)
# [ImageContent(base64_image='...',
# mime_type='application/pdf',
# detail=None,
# meta={'file_path': 'file.pdf', 'page_number': 1}),
# ...]
```
#### __init__
```python
__init__(
*,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None,
page_range: list[str | int] | None = None
) -> None
```
Create the PDFToImageContent component.
**Parameters:**
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
This will be passed to the created ImageContent objects.
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within the specified dimensions (width, height) while
maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
when working with models that have resolution constraints or when transmitting images to remote services.
- **page_range** (<code>list\[str | int\] | None</code>) – List of page numbers and/or page ranges to convert to images. Page numbers start at 1.
If None, all pages in the PDF will be converted. Pages outside the valid range (1 to number of pages)
will be skipped with a warning. For example, page_range=[1, 3] will convert only the first and third
pages of the document. It also accepts printable range strings, e.g.: ['1-3', '5', '8', '10-12']
will convert pages 1, 2, 3, 5, 8, 10, 11, 12.
#### run
```python
run(
sources: list[str | Path | ByteStream],
meta: dict[str, Any] | list[dict[str, Any]] | None = None,
*,
detail: Literal["auto", "high", "low"] | None = None,
size: tuple[int, int] | None = None,
page_range: list[str | int] | None = None
) -> dict[str, list[ImageContent]]
```
Converts files to ImageContent objects.
**Parameters:**
- **sources** (<code>list\[str | Path | ByteStream\]</code>) – List of file paths or ByteStream objects to convert.
- **meta** (<code>dict\[str, Any\] | list\[dict\[str, Any\]\] | None</code>) – Optional metadata to attach to the ImageContent objects.
This value can be a list of dictionaries or a single dictionary.
If it's a single dictionary, its content is added to the metadata of all produced ImageContent objects.
If it's a list, its length must match the number of sources as they're zipped together.
For ByteStream objects, their `meta` is added to the output ImageContent objects.
- **detail** (<code>Literal['auto', 'high', 'low'] | None</code>) – Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
This will be passed to the created ImageContent objects.
If not provided, the detail level will be the one set in the constructor.
- **size** (<code>tuple\[int, int\] | None</code>) – If provided, resizes the image to fit within the specified dimensions (width, height) while
maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
when working with models that have resolution constraints or when transmitting images to remote services.
If not provided, the size value will be the one set in the constructor.
- **page_range** (<code>list\[str | int\] | None</code>) – List of page numbers and/or page ranges to convert to images. Page numbers start at 1.
If None, all pages in the PDF will be converted. Pages outside the valid range (1 to number of pages)
will be skipped with a warning. For example, page_range=[1, 3] will convert only the first and third
pages of the document. It also accepts printable range strings, e.g.: ['1-3', '5', '8', '10-12']
will convert pages 1, 2, 3, 5, 8, 10, 11, 12.
If not provided, the page_range value will be the one set in the constructor.
**Returns:**
- <code>dict\[str, list\[ImageContent\]\]</code> – A dictionary with the following keys:
- `image_contents`: A list of ImageContent objects.
docs-website/reference_versioned_docs/version-2.30/haystack-api/joiners_api.md
+566
View File
@@ -0,0 +1,566 @@
---
title: "Joiners"
id: joiners-api
description: "Components that join list of different objects"
slug: "/joiners-api"
---
## answer_joiner
### JoinMode
Bases: <code>Enum</code>
Enum for AnswerJoiner join modes.
#### from_str
```python
from_str(string: str) -> JoinMode
```
Convert a string to a JoinMode enum.
### AnswerJoiner
Merges multiple lists of `Answer` objects into a single list.
Use this component to combine answers from different Generators into a single list.
Currently, the component supports only one join mode: `CONCATENATE`.
This mode concatenates multiple lists of answers into a single list.
### Usage example
In this example, AnswerJoiner merges answers from two different Generators:
```python
from haystack.components.builders import AnswerBuilder
from haystack.components.joiners import AnswerJoiner
from haystack.core.pipeline import Pipeline
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.dataclasses import ChatMessage
query = "What's Natural Language Processing?"
messages = [ChatMessage.from_system("You are a helpful, respectful and honest assistant. Be super concise."),
ChatMessage.from_user(query)]
pipe = Pipeline()
pipe.add_component("llm_1", OpenAIChatGenerator())
pipe.add_component("llm_2", OpenAIChatGenerator())
pipe.add_component("aba", AnswerBuilder())
pipe.add_component("abb", AnswerBuilder())
pipe.add_component("joiner", AnswerJoiner())
pipe.connect("llm_1.replies", "aba")
pipe.connect("llm_2.replies", "abb")
pipe.connect("aba.answers", "joiner")
pipe.connect("abb.answers", "joiner")
results = pipe.run(data={"llm_1": {"messages": messages},
"llm_2": {"messages": messages},
"aba": {"query": query},
"abb": {"query": query}})
```
#### __init__
```python
__init__(
join_mode: str | JoinMode = JoinMode.CONCATENATE,
top_k: int | None = None,
sort_by_score: bool = False,
) -> None
```
Creates an AnswerJoiner component.
**Parameters:**
- **join_mode** (<code>str | JoinMode</code>) – Specifies the join mode to use. Available modes:
- `concatenate`: Concatenates multiple lists of Answers into a single list.
- **top_k** (<code>int | None</code>) – The maximum number of Answers to return.
- **sort_by_score** (<code>bool</code>) – If `True`, sorts the documents by score in descending order.
If a document has no score, it is handled as if its score is -infinity.
#### run
```python
run(
answers: Variadic[list[AnswerType]], top_k: int | None = None
) -> dict[str, Any]
```
Joins multiple lists of Answers into a single list depending on the `join_mode` parameter.
**Parameters:**
- **answers** (<code>Variadic\[list\[AnswerType\]\]</code>) – Nested list of Answers to be merged.
- **top_k** (<code>int | None</code>) – The maximum number of Answers to return. Overrides the instance's `top_k` if provided.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `answers`: Merged list of Answers
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> AnswerJoiner
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>AnswerJoiner</code> – The deserialized component.
## branch
### BranchJoiner
A component that merges multiple input branches of a pipeline into a single output stream.
`BranchJoiner` receives multiple inputs of the same data type and forwards the first received value
to its output. This is useful for scenarios where multiple branches need to converge before proceeding.
### Common Use Cases:
- **Loop Handling:** `BranchJoiner` helps close loops in pipelines. For example, if a pipeline component validates
or modifies incoming data and produces an error-handling branch, `BranchJoiner` can merge both branches and send
(or resend in the case of a loop) the data to the component that evaluates errors. See "Usage example" below.
- **Decision-Based Merging:** `BranchJoiner` reconciles branches coming from Router components (such as
`ConditionalRouter`, `TextLanguageRouter`). Suppose a `TextLanguageRouter` directs user queries to different
Retrievers based on the detected language. Each Retriever processes its assigned query and passes the results
to `BranchJoiner`, which consolidates them into a single output before passing them to the next component, such
as a `PromptBuilder`.
### Example Usage:
```python
import json
from haystack import Pipeline
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.joiners import BranchJoiner
from haystack.components.validators import JsonSchemaValidator
from haystack.dataclasses import ChatMessage
# Define a schema for validation
person_schema = {
"type": "object",
"properties": {
"first_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
"last_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
"nationality": {"type": "string", "enum": ["Italian", "Portuguese", "American"]},
},
"required": ["first_name", "last_name", "nationality"]
}
# Initialize a pipeline
pipe = Pipeline()
# Add components to the pipeline
pipe.add_component("joiner", BranchJoiner(list[ChatMessage]))
pipe.add_component("generator", OpenAIChatGenerator(model="gpt-4.1-mini"))
pipe.add_component("validator", JsonSchemaValidator(json_schema=person_schema))
# And connect them
pipe.connect("joiner", "generator")
pipe.connect("generator.replies", "validator.messages")
pipe.connect("validator.validation_error", "joiner")
result = pipe.run(
data={
"generator": {"generation_kwargs": {"response_format": {"type": "json_object"}}},
"joiner": {"value": [ChatMessage.from_user("Create json from Peter Parker")]}}
)
print(json.loads(result["validator"]["validated"][0].text))
# >> {'first_name': 'Peter', 'last_name': 'Parker', 'nationality': 'American', 'name': 'Spider-Man', 'occupation':
# >> 'Superhero', 'age': 23, 'location': 'New York City'}
```
Note that `BranchJoiner` can manage only one data type at a time. In this case, `BranchJoiner` is created for
passing `list[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
connected components and also the type of data that `BranchJoiner` will send through its output.
In the code example, `BranchJoiner` receives a looped back `list[ChatMessage]` from the `JsonSchemaValidator` and
sends it down to the `OpenAIChatGenerator` for re-generation. We can have multiple loopback connections in the
pipeline. In this instance, the downstream component is only one (the `OpenAIChatGenerator`), but the pipeline could
have more than one downstream component.
#### __init__
```python
__init__(type_: type) -> None
```
Creates a `BranchJoiner` component.
**Parameters:**
- **type\_** (<code>type</code>) – The expected data type of inputs and outputs.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component into a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> BranchJoiner
```
Deserializes a `BranchJoiner` instance from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary containing serialized component data.
**Returns:**
- <code>BranchJoiner</code> – A deserialized `BranchJoiner` instance.
#### run
```python
run(**kwargs: Any) -> dict[str, Any]
```
Executes the `BranchJoiner`, selecting the first available input value and passing it downstream.
**Parameters:**
- \*\***kwargs** (<code>Any</code>) – The input data. Must be of the type declared by `type_` during initialization.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with a single key `value`, containing the first input received.
## document_joiner
### JoinMode
Bases: <code>Enum</code>
Enum for join mode.
#### from_str
```python
from_str(string: str) -> JoinMode
```
Convert a string to a JoinMode enum.
### DocumentJoiner
Joins multiple lists of documents into a single list.
It supports different join modes:
- concatenate: Keeps the highest-scored document in case of duplicates.
- merge: Calculates a weighted sum of scores for duplicates and merges them.
- reciprocal_rank_fusion: Merges and assigns scores based on reciprocal rank fusion.
- distribution_based_rank_fusion: Merges and assigns scores based on scores distribution in each Retriever.
### Usage example:
```python
from haystack import Pipeline, Document
from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder
from haystack.components.joiners import DocumentJoiner
from haystack.components.retrievers import InMemoryBM25Retriever
from haystack.components.retrievers import InMemoryEmbeddingRetriever
from haystack.document_stores.in_memory import InMemoryDocumentStore
document_store = InMemoryDocumentStore()
docs = [Document(content="Paris"), Document(content="Berlin"), Document(content="London")]
embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
docs_embeddings = embedder.run(docs)
document_store.write_documents(docs_embeddings['documents'])
p = Pipeline()
p.add_component(instance=InMemoryBM25Retriever(document_store=document_store), name="bm25_retriever")
p.add_component(
instance=SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"),
name="text_embedder",
)
p.add_component(instance=InMemoryEmbeddingRetriever(document_store=document_store), name="embedding_retriever")
p.add_component(instance=DocumentJoiner(), name="joiner")
p.connect("bm25_retriever", "joiner")
p.connect("embedding_retriever", "joiner")
p.connect("text_embedder", "embedding_retriever")
query = "What is the capital of France?"
p.run(data={"query": query, "text": query, "top_k": 1})
```
#### __init__
```python
__init__(
join_mode: str | JoinMode = JoinMode.CONCATENATE,
weights: list[float] | None = None,
top_k: int | None = None,
sort_by_score: bool = True,
) -> None
```
Creates a DocumentJoiner component.
**Parameters:**
- **join_mode** (<code>str | JoinMode</code>) – Specifies the join mode to use. Available modes:
- `concatenate`: Keeps the highest-scored document in case of duplicates.
- `merge`: Calculates a weighted sum of scores for duplicates and merges them.
- `reciprocal_rank_fusion`: Merges and assigns scores based on reciprocal rank fusion.
- `distribution_based_rank_fusion`: Merges and assigns scores based on scores
distribution in each Retriever.
- **weights** (<code>list\[float\] | None</code>) – Assign importance to each list of documents to influence how they're joined.
This parameter is ignored for
`concatenate` or `distribution_based_rank_fusion` join modes.
Weight for each list of documents must match the number of inputs.
- **top_k** (<code>int | None</code>) – The maximum number of documents to return.
- **sort_by_score** (<code>bool</code>) – If `True`, sorts the documents by score in descending order.
If a document has no score, it is handled as if its score is -infinity.
#### run
```python
run(
documents: Variadic[list[Document]], top_k: int | None = None
) -> dict[str, Any]
```
Joins multiple lists of Documents into a single list depending on the `join_mode` parameter.
**Parameters:**
- **documents** (<code>Variadic\[list\[Document\]\]</code>) – List of list of documents to be merged.
- **top_k** (<code>int | None</code>) – The maximum number of documents to return. Overrides the instance's `top_k` if provided.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following keys:
- `documents`: Merged list of Documents
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> DocumentJoiner
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize from.
**Returns:**
- <code>DocumentJoiner</code> – The deserialized component.
## list_joiner
### ListJoiner
A component that joins multiple lists into a single flat list.
The ListJoiner receives multiple lists of the same type and concatenates them into a single flat list.
The output order respects the pipeline's execution sequence, with earlier inputs being added first.
Usage example:
```python
from haystack.components.builders import ChatPromptBuilder
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.dataclasses import ChatMessage
from haystack import Pipeline
from haystack.components.joiners import ListJoiner
user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")]
feedback_prompt = """
You are given a question and an answer.
Your task is to provide a score and a brief feedback on the answer.
Question: {{query}}
Answer: {{response}}
"""
feedback_message = [ChatMessage.from_system(feedback_prompt)]
prompt_builder = ChatPromptBuilder(template=user_message)
feedback_prompt_builder = ChatPromptBuilder(template=feedback_message)
llm = OpenAIChatGenerator()
feedback_llm = OpenAIChatGenerator()
pipe = Pipeline()
pipe.add_component("prompt_builder", prompt_builder)
pipe.add_component("llm", llm)
pipe.add_component("feedback_prompt_builder", feedback_prompt_builder)
pipe.add_component("feedback_llm", feedback_llm)
pipe.add_component("list_joiner", ListJoiner(list[ChatMessage]))
pipe.connect("prompt_builder.prompt", "llm.messages")
pipe.connect("prompt_builder.prompt", "list_joiner")
pipe.connect("llm.replies", "list_joiner")
pipe.connect("llm.replies", "feedback_prompt_builder.response")
pipe.connect("feedback_prompt_builder.prompt", "feedback_llm.messages")
pipe.connect("feedback_llm.replies", "list_joiner")
query = "What is nuclear physics?"
ans = pipe.run(data={"prompt_builder": {"template_variables":{"query": query}},
"feedback_prompt_builder": {"template_variables":{"query": query}}})
print(ans["list_joiner"]["values"])
```
#### __init__
```python
__init__(list_type_: type | None = None) -> None
```
Creates a ListJoiner component.
**Parameters:**
- **list_type\_** (<code>type | None</code>) – The expected type of the lists this component will join (e.g., list[ChatMessage]).
If specified, all input lists must conform to this type. If None, the component defaults to handling
lists of any type including mixed types.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> ListJoiner
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>ListJoiner</code> – Deserialized component.
#### run
```python
run(values: Variadic[list[Any]]) -> dict[str, list[Any]]
```
Joins multiple lists into a single flat list.
**Parameters:**
- **values** (<code>Variadic\[list\[Any\]\]</code>) – The list to be joined.
**Returns:**
- <code>dict\[str, list\[Any\]\]</code> – Dictionary with 'values' key containing the joined list.
## string_joiner
### StringJoiner
Component to join strings from different components to a list of strings.
### Usage example
```python
from haystack.components.joiners import StringJoiner
from haystack.components.builders import PromptBuilder
from haystack.core.pipeline import Pipeline
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.dataclasses import ChatMessage
string_1 = "What's Natural Language Processing?"
string_2 = "What is life?"
pipeline = Pipeline()
pipeline.add_component("prompt_builder_1", PromptBuilder("Builder 1: {{query}}"))
pipeline.add_component("prompt_builder_2", PromptBuilder("Builder 2: {{query}}"))
pipeline.add_component("string_joiner", StringJoiner())
pipeline.connect("prompt_builder_1.prompt", "string_joiner.strings")
pipeline.connect("prompt_builder_2.prompt", "string_joiner.strings")
print(pipeline.run(data={"prompt_builder_1": {"query": string_1}, "prompt_builder_2": {"query": string_2}}))
# >> {"string_joiner": {"strings": ["Builder 1: What's Natural Language Processing?", "Builder 2: What is life?"]}}
```
#### run
```python
run(strings: Variadic[str]) -> dict[str, list[str]]
```
Joins strings into a list of strings
**Parameters:**
- **strings** (<code>Variadic\[str\]</code>) – strings from different components
**Returns:**
- <code>dict\[str, list\[str\]\]</code> – A dictionary with the following keys:
- `strings`: Merged list of strings
docs-website/reference_versioned_docs/version-2.30/haystack-api/pipeline_api.md
+497
View File
@@ -0,0 +1,497 @@
---
title: "Pipeline"
id: pipeline-api
description: "Arranges components and integrations in flow."
slug: "/pipeline-api"
---
## async_pipeline
### AsyncPipeline
Bases: <code>PipelineBase</code>
Asynchronous version of the Pipeline orchestration engine.
Manages components in a pipeline allowing for concurrent processing when the pipeline's execution graph permits.
This enables efficient processing of components by minimizing idle time and maximizing resource utilization.
#### run_async_generator
```python
run_async_generator(
data: dict[str, Any],
include_outputs_from: set[str] | None = None,
concurrency_limit: int = 4,
) -> AsyncIterator[dict[str, Any]]
```
Executes the pipeline step by step asynchronously, yielding partial outputs when any component finishes.
Usage:
```python
from haystack import Document
from haystack.components.builders import ChatPromptBuilder
from haystack.dataclasses import ChatMessage
from haystack.utils import Secret
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.builders.prompt_builder import PromptBuilder
from haystack import AsyncPipeline
import asyncio
# Write documents to InMemoryDocumentStore
document_store = InMemoryDocumentStore()
document_store.write_documents([
Document(content="My name is Jean and I live in Paris."),
Document(content="My name is Mark and I live in Berlin."),
Document(content="My name is Giorgio and I live in Rome.")
])
prompt_template = [
ChatMessage.from_user(
'''
Given these documents, answer the question.
Documents:
{% for doc in documents %}
{{ doc.content }}
{% endfor %}
Question: {{question}}
Answer:
''')
]
# Create and connect pipeline components
retriever = InMemoryBM25Retriever(document_store=document_store)
prompt_builder = ChatPromptBuilder(template=prompt_template)
llm = OpenAIChatGenerator()
rag_pipeline = AsyncPipeline()
rag_pipeline.add_component("retriever", retriever)
rag_pipeline.add_component("prompt_builder", prompt_builder)
rag_pipeline.add_component("llm", llm)
rag_pipeline.connect("retriever", "prompt_builder.documents")
rag_pipeline.connect("prompt_builder", "llm")
# Prepare input data
question = "Who lives in Paris?"
data = {
"retriever": {"query": question},
"prompt_builder": {"question": question},
}
# Process results as they become available
async def process_results():
async for partial_output in rag_pipeline.run_async_generator(
data=data,
include_outputs_from={"retriever", "llm"}
):
# Each partial_output contains the results from a completed component
if "retriever" in partial_output:
print("Retrieved documents:", len(partial_output["retriever"]["documents"]))
if "llm" in partial_output:
print("Generated answer:", partial_output["llm"]["replies"][0])
asyncio.run(process_results())
```
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Initial input data to the pipeline.
- **concurrency_limit** (<code>int</code>) – The maximum number of components that are allowed to run concurrently.
- **include_outputs_from** (<code>set\[str\] | None</code>) – Set of component names whose individual outputs are to be
included in the pipeline's output. For components that are
invoked multiple times (in a loop), only the last-produced
output is included.
**Returns:**
- <code>AsyncIterator\[dict\[str, Any\]\]</code> – An async iterator containing partial (and final) outputs.
**Raises:**
- <code>ValueError</code> – If invalid inputs are provided to the pipeline.
- <code>PipelineMaxComponentRuns</code> – If a component exceeds the maximum number of allowed executions within the pipeline.
- <code>PipelineRuntimeError</code> – If the Pipeline contains cycles with unsupported connections that would cause
it to get stuck and fail running.
Or if a Component fails or returns output in an unsupported type.
#### run_async
```python
run_async(
data: dict[str, Any],
include_outputs_from: set[str] | None = None,
concurrency_limit: int = 4,
) -> dict[str, Any]
```
Provides an asynchronous interface to run the pipeline with provided input data.
This method allows the pipeline to be integrated into an asynchronous workflow, enabling non-blocking
execution of pipeline components.
Usage:
```python
import asyncio
from haystack import Document
from haystack.components.builders import ChatPromptBuilder
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.core.pipeline import AsyncPipeline
from haystack.dataclasses import ChatMessage
from haystack.document_stores.in_memory import InMemoryDocumentStore
# Write documents to InMemoryDocumentStore
document_store = InMemoryDocumentStore()
document_store.write_documents([
Document(content="My name is Jean and I live in Paris."),
Document(content="My name is Mark and I live in Berlin."),
Document(content="My name is Giorgio and I live in Rome.")
])
prompt_template = [
ChatMessage.from_user(
'''
Given these documents, answer the question.
Documents:
{% for doc in documents %}
{{ doc.content }}
{% endfor %}
Question: {{question}}
Answer:
''')
]
retriever = InMemoryBM25Retriever(document_store=document_store)
prompt_builder = ChatPromptBuilder(template=prompt_template)
llm = OpenAIChatGenerator()
rag_pipeline = AsyncPipeline()
rag_pipeline.add_component("retriever", retriever)
rag_pipeline.add_component("prompt_builder", prompt_builder)
rag_pipeline.add_component("llm", llm)
rag_pipeline.connect("retriever", "prompt_builder.documents")
rag_pipeline.connect("prompt_builder", "llm")
# Ask a question
question = "Who lives in Paris?"
async def run_inner(data, include_outputs_from):
return await rag_pipeline.run_async(data=data, include_outputs_from=include_outputs_from)
data = {
"retriever": {"query": question},
"prompt_builder": {"question": question},
}
results = asyncio.run(run_inner(data, include_outputs_from={"retriever", "llm"}))
print(results["llm"]["replies"])
# [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text='Jean lives in Paris.')],
# _name=None, _meta={'model': 'gpt-5-mini', 'index': 0, 'finish_reason': 'stop', 'usage':
# {'completion_tokens': 6, 'prompt_tokens': 69, 'total_tokens': 75,
# 'completion_tokens_details': CompletionTokensDetails(accepted_prediction_tokens=0,
# audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), 'prompt_tokens_details':
# PromptTokensDetails(audio_tokens=0, cached_tokens=0)}})]
```
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – A dictionary of inputs for the pipeline's components. Each key is a component name
and its value is a dictionary of that component's input parameters:
```
data = {
"comp1": {"input1": 1, "input2": 2},
}
```
For convenience, this format is also supported when input names are unique:
```
data = {
"input1": 1, "input2": 2,
}
```
- **include_outputs_from** (<code>set\[str\] | None</code>) – Set of component names whose individual outputs are to be
included in the pipeline's output. For components that are
invoked multiple times (in a loop), only the last-produced
output is included.
- **concurrency_limit** (<code>int</code>) – The maximum number of components that should be allowed to run concurrently.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary where each entry corresponds to a component name
and its output. If `include_outputs_from` is `None`, this dictionary
will only contain the outputs of leaf components, i.e., components
without outgoing connections.
**Raises:**
- <code>ValueError</code> – If invalid inputs are provided to the pipeline.
- <code>PipelineRuntimeError</code> – If the Pipeline contains cycles with unsupported connections that would cause
it to get stuck and fail running.
Or if a Component fails or returns output in an unsupported type.
- <code>PipelineMaxComponentRuns</code> – If a Component reaches the maximum number of times it can be run in this Pipeline.
#### run
```python
run(
data: dict[str, Any],
include_outputs_from: set[str] | None = None,
concurrency_limit: int = 4,
) -> dict[str, Any]
```
Provides a synchronous interface to run the pipeline with given input data.
Internally, the pipeline components are executed asynchronously, but the method itself
will block until the entire pipeline execution is complete.
In case you need asynchronous methods, consider using `run_async` or `run_async_generator`.
Usage:
```python
from haystack import Document
from haystack.components.builders import ChatPromptBuilder
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.core.pipeline import AsyncPipeline
from haystack.dataclasses import ChatMessage
from haystack.document_stores.in_memory import InMemoryDocumentStore
# Write documents to InMemoryDocumentStore
document_store = InMemoryDocumentStore()
document_store.write_documents([
Document(content="My name is Jean and I live in Paris."),
Document(content="My name is Mark and I live in Berlin."),
Document(content="My name is Giorgio and I live in Rome.")
])
prompt_template = [
ChatMessage.from_user(
'''
Given these documents, answer the question.
Documents:
{% for doc in documents %}
{{ doc.content }}
{% endfor %}
Question: {{question}}
Answer:
''')
]
retriever = InMemoryBM25Retriever(document_store=document_store)
prompt_builder = ChatPromptBuilder(template=prompt_template)
llm = OpenAIChatGenerator()
rag_pipeline = AsyncPipeline()
rag_pipeline.add_component("retriever", retriever)
rag_pipeline.add_component("prompt_builder", prompt_builder)
rag_pipeline.add_component("llm", llm)
rag_pipeline.connect("retriever", "prompt_builder.documents")
rag_pipeline.connect("prompt_builder", "llm")
# Ask a question
question = "Who lives in Paris?"
data = {
"retriever": {"query": question},
"prompt_builder": {"question": question},
}
results = rag_pipeline.run(data)
print(results["llm"]["replies"])
# [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text='Jean lives in Paris.')],
# _name=None, _meta={'model': 'gpt-5-mini', 'index': 0, 'finish_reason': 'stop', 'usage':
# {'completion_tokens': 6, 'prompt_tokens': 69, 'total_tokens': 75, 'completion_tokens_details':
# CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0,
# rejected_prediction_tokens=0), 'prompt_tokens_details': PromptTokensDetails(audio_tokens=0,
# cached_tokens=0)}})]
```
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – A dictionary of inputs for the pipeline's components. Each key is a component name
and its value is a dictionary of that component's input parameters:
```
data = {
"comp1": {"input1": 1, "input2": 2},
}
```
For convenience, this format is also supported when input names are unique:
```
data = {
"input1": 1, "input2": 2,
}
```
- **include_outputs_from** (<code>set\[str\] | None</code>) – Set of component names whose individual outputs are to be
included in the pipeline's output. For components that are
invoked multiple times (in a loop), only the last-produced
output is included.
- **concurrency_limit** (<code>int</code>) – The maximum number of components that should be allowed to run concurrently.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary where each entry corresponds to a component name
and its output. If `include_outputs_from` is `None`, this dictionary
will only contain the outputs of leaf components, i.e., components
without outgoing connections.
**Raises:**
- <code>ValueError</code> – If invalid inputs are provided to the pipeline.
- <code>PipelineRuntimeError</code> – If the Pipeline contains cycles with unsupported connections that would cause
it to get stuck and fail running.
Or if a Component fails or returns output in an unsupported type.
- <code>PipelineMaxComponentRuns</code> – If a Component reaches the maximum number of times it can be run in this Pipeline.
- <code>RuntimeError</code> – If called from within an async context. Use `run_async` instead.
## pipeline
### Pipeline
Bases: <code>PipelineBase</code>
Synchronous version of the orchestration engine.
Orchestrates component execution according to the execution graph, one after the other.
#### run
```python
run(
data: dict[str, Any],
include_outputs_from: set[str] | None = None,
*,
break_point: Breakpoint | AgentBreakpoint | None = None,
pipeline_snapshot: PipelineSnapshot | None = None,
snapshot_callback: SnapshotCallback | None = None
) -> dict[str, Any]
```
Runs the Pipeline with given input data.
Usage:
```python
from haystack import Pipeline, Document
from haystack.components.builders.answer_builder import AnswerBuilder
from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder
from haystack.components.generators.chat import OpenAIChatGenerator
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.dataclasses import ChatMessage
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.utils import Secret
# Write documents to InMemoryDocumentStore
document_store = InMemoryDocumentStore()
document_store.write_documents([
Document(content="My name is Jean and I live in Paris."),
Document(content="My name is Mark and I live in Berlin."),
Document(content="My name is Giorgio and I live in Rome.")
])
retriever = InMemoryBM25Retriever(document_store=document_store)
prompt_template = """
Given these documents, answer the question.
Documents:
{% for doc in documents %}
{{ doc.content }}
{% endfor %}
Question: {{question}}
Answer:
"""
template = [ChatMessage.from_user(prompt_template)]
prompt_builder = ChatPromptBuilder(
template=template,
required_variables=["question", "documents"],
variables=["question", "documents"]
)
llm = OpenAIChatGenerator()
rag_pipeline = Pipeline()
rag_pipeline.add_component("retriever", retriever)
rag_pipeline.add_component("prompt_builder", prompt_builder)
rag_pipeline.add_component("llm", llm)
rag_pipeline.connect("retriever", "prompt_builder.documents")
rag_pipeline.connect("prompt_builder", "llm")
question = "Who lives in Paris?"
results = rag_pipeline.run(
{
"retriever": {"query": question},
"prompt_builder": {"question": question},
}
)
print(results["llm"]["replies"][0].text)
# Jean lives in Paris
```
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – A dictionary of inputs for the pipeline's components. Each key is a component name
and its value is a dictionary of that component's input parameters:
```
data = {
"comp1": {"input1": 1, "input2": 2},
}
```
For convenience, this format is also supported when input names are unique:
```
data = {
"input1": 1, "input2": 2,
}
```
- **include_outputs_from** (<code>set\[str\] | None</code>) – Set of component names whose individual outputs are to be
included in the pipeline's output. For components that are
invoked multiple times (in a loop), only the last-produced
output is included.
- **break_point** (<code>Breakpoint | AgentBreakpoint | None</code>) – A set of breakpoints that can be used to debug the pipeline execution.
- **pipeline_snapshot** (<code>PipelineSnapshot | None</code>) – A dictionary containing a snapshot of a previously saved pipeline execution.
- **snapshot_callback** (<code>SnapshotCallback | None</code>) – Optional callback function that is invoked when a pipeline snapshot is created.
The callback receives a `PipelineSnapshot` object and can return an optional string
(e.g., a file path or identifier).
If provided, the callback is used instead of the default file-saving behavior,
allowing custom handling of snapshots (e.g., saving to a database, sending to a remote service).
If not provided, the default behavior saves snapshots to a JSON file.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary where each entry corresponds to a component name
and its output. If `include_outputs_from` is `None`, this dictionary
will only contain the outputs of leaf components, i.e., components
without outgoing connections.
**Raises:**
- <code>ValueError</code> – If invalid inputs are provided to the pipeline.
- <code>PipelineRuntimeError</code> – If the Pipeline contains cycles with unsupported connections that would cause
it to get stuck and fail running.
Or if a Component fails or returns output in an unsupported type.
- <code>PipelineMaxComponentRuns</code> – If a Component reaches the maximum number of times it can be run in this Pipeline.
- <code>PipelineBreakpointException</code> – When a pipeline_breakpoint is triggered. Contains the component name, state, and partial results.
docs-website/reference_versioned_docs/version-2.30/haystack-api/preprocessors_api.md
+992
View File
@@ -0,0 +1,992 @@
---
title: "PreProcessors"
id: preprocessors-api
description: "Preprocess your Documents and texts. Clean, split, and more."
slug: "/preprocessors-api"
---
## csv_document_cleaner
### CSVDocumentCleaner
A component for cleaning CSV documents by removing empty rows and columns.
This component processes CSV content stored in Documents, allowing
for the optional ignoring of a specified number of rows and columns before performing
the cleaning operation. Additionally, it provides options to keep document IDs and
control whether empty rows and columns should be removed.
#### __init__
```python
__init__(
*,
ignore_rows: int = 0,
ignore_columns: int = 0,
remove_empty_rows: bool = True,
remove_empty_columns: bool = True,
keep_id: bool = False
) -> None
```
Initializes the CSVDocumentCleaner component.
**Parameters:**
- **ignore_rows** (<code>int</code>) – Number of rows to ignore from the top of the CSV table before processing.
- **ignore_columns** (<code>int</code>) – Number of columns to ignore from the left of the CSV table before processing.
- **remove_empty_rows** (<code>bool</code>) – Whether to remove rows that are entirely empty.
- **remove_empty_columns** (<code>bool</code>) – Whether to remove columns that are entirely empty.
- **keep_id** (<code>bool</code>) – Whether to retain the original document ID in the output document.
Rows and columns ignored using these parameters are preserved in the final output, meaning
they are not considered when removing empty rows and columns.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Cleans CSV documents by removing empty rows and columns while preserving specified ignored rows and columns.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of Documents containing CSV-formatted content.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with a list of cleaned Documents under the key "documents".
Processing steps:
1. Reads each document's content as a CSV table.
1. Retains the specified number of `ignore_rows` from the top and `ignore_columns` from the left.
1. Drops any rows and columns that are entirely empty (if enabled by `remove_empty_rows` and
`remove_empty_columns`).
1. Reattaches the ignored rows and columns to maintain their original positions.
1. Returns the cleaned CSV content as a new `Document` object, with an option to retain the original
document ID.
## csv_document_splitter
### CSVDocumentSplitter
A component for splitting CSV documents into sub-tables based on split arguments.
The splitter supports two modes of operation:
- identify consecutive empty rows or columns that exceed a given threshold
and uses them as delimiters to segment the document into smaller tables.
- split each row into a separate sub-table, represented as a Document.
#### __init__
```python
__init__(
row_split_threshold: int | None = 2,
column_split_threshold: int | None = 2,
read_csv_kwargs: dict[str, Any] | None = None,
split_mode: SplitMode = "threshold",
) -> None
```
Initializes the CSVDocumentSplitter component.
**Parameters:**
- **row_split_threshold** (<code>int | None</code>) – The minimum number of consecutive empty rows required to trigger a split.
- **column_split_threshold** (<code>int | None</code>) – The minimum number of consecutive empty columns required to trigger a split.
- **read_csv_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments to pass to `pandas.read_csv`.
By default, the component with options:
- `header=None`
- `skip_blank_lines=False` to preserve blank lines
- `dtype=object` to prevent type inference (e.g., converting numbers to floats).
See https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html for more information.
- **split_mode** (<code>SplitMode</code>) – If `threshold`, the component will split the document based on the number of
consecutive empty rows or columns that exceed the `row_split_threshold` or `column_split_threshold`.
If `row-wise`, the component will split each row into a separate sub-table.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Processes and splits a list of CSV documents into multiple sub-tables.
**Splitting Process:**
1. Applies a row-based split if `row_split_threshold` is provided.
1. Applies a column-based split if `column_split_threshold` is provided.
1. If both thresholds are specified, performs a recursive split by rows first, then columns, ensuring
further fragmentation of any sub-tables that still contain empty sections.
1. Sorts the resulting sub-tables based on their original positions within the document.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – A list of Documents containing CSV-formatted content.
Each document is assumed to contain one or more tables separated by empty rows or columns.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with a key `"documents"`, mapping to a list of new `Document` objects,
each representing an extracted sub-table from the original CSV.
The metadata of each document includes:
\- A field `source_id` to track the original document.
\- A field `row_idx_start` to indicate the starting row index of the sub-table in the original table.
\- A field `col_idx_start` to indicate the starting column index of the sub-table in the original table.
\- A field `split_id` to indicate the order of the split in the original document.
\- All other metadata copied from the original document.
- If a document cannot be processed, it is returned unchanged.
- The `meta` field from the original document is preserved in the split documents.
## document_cleaner
### DocumentCleaner
Cleans the text in the documents.
It removes extra whitespaces,
empty lines, specified substrings, regexes,
page headers and footers (in this order).
### Usage example:
```python
from haystack import Document
from haystack.components.preprocessors import DocumentCleaner
doc = Document(content="This is a document to clean\n\n\nsubstring to remove")
cleaner = DocumentCleaner(remove_substrings = ["substring to remove"])
result = cleaner.run(documents=[doc])
assert result["documents"][0].content == "This is a document to clean "
```
#### __init__
```python
__init__(
remove_empty_lines: bool = True,
remove_extra_whitespaces: bool = True,
remove_repeated_substrings: bool = False,
keep_id: bool = False,
remove_substrings: list[str] | None = None,
remove_regex: str | None = None,
unicode_normalization: Literal["NFC", "NFKC", "NFD", "NFKD"] | None = None,
ascii_only: bool = False,
strip_whitespaces: bool = False,
replace_regexes: dict[str, str] | None = None,
) -> None
```
Initialize DocumentCleaner.
**Parameters:**
- **remove_empty_lines** (<code>bool</code>) – If `True`, removes empty lines.
- **remove_extra_whitespaces** (<code>bool</code>) – If `True`, removes extra whitespaces.
- **remove_repeated_substrings** (<code>bool</code>) – If `True`, removes repeated substrings (headers and footers) from pages.
Pages must be separated by a form feed character "\\f",
which is supported by `TextFileToDocument` and `AzureOCRDocumentConverter`.
- **remove_substrings** (<code>list\[str\] | None</code>) – List of substrings to remove from the text.
- **remove_regex** (<code>str | None</code>) – Regex to match and replace substrings by "".
- **keep_id** (<code>bool</code>) – If `True`, keeps the IDs of the original documents.
- **unicode_normalization** (<code>Literal['NFC', 'NFKC', 'NFD', 'NFKD'] | None</code>) – Unicode normalization form to apply to the text.
Note: This will run before any other steps.
- **ascii_only** (<code>bool</code>) – Whether to convert the text to ASCII only.
Will remove accents from characters and replace them with ASCII characters.
Other non-ASCII characters will be removed.
Note: This will run before any pattern matching or removal.
- **strip_whitespaces** (<code>bool</code>) – If `True`, removes leading and trailing whitespace from the document content
using Python's `str.strip()`. Unlike `remove_extra_whitespaces`, this only affects the beginning
and end of the text, preserving internal whitespace (useful for markdown formatting).
- **replace_regexes** (<code>dict\[str, str\] | None</code>) – A dictionary mapping regex patterns to their replacement strings.
For example, `{r'\n\n+': '\n'}` replaces multiple consecutive newlines with a single newline.
This is applied after `remove_regex` and allows custom replacements instead of just removal.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Cleans up the documents.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of Documents to clean.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: List of cleaned Documents.
**Raises:**
- <code>TypeError</code> – if documents is not a list of Documents.
## document_preprocessor
### DocumentPreprocessor
A SuperComponent that first splits and then cleans documents.
This component consists of a DocumentSplitter followed by a DocumentCleaner in a single pipeline.
It takes a list of documents as input and returns a processed list of documents.
Usage example:
```python
from haystack import Document
from haystack.components.preprocessors import DocumentPreprocessor
doc = Document(content="I love pizza!")
preprocessor = DocumentPreprocessor()
result = preprocessor.run(documents=[doc])
print(result["documents"])
```
#### __init__
```python
__init__(
*,
split_by: Literal[
"function", "page", "passage", "period", "word", "line", "sentence"
] = "word",
split_length: int = 250,
split_overlap: int = 0,
split_threshold: int = 0,
splitting_function: Callable[[str], list[str]] | None = None,
respect_sentence_boundary: bool = False,
language: Language = "en",
use_split_rules: bool = True,
extend_abbreviations: bool = True,
remove_empty_lines: bool = True,
remove_extra_whitespaces: bool = True,
remove_repeated_substrings: bool = False,
keep_id: bool = False,
remove_substrings: list[str] | None = None,
remove_regex: str | None = None,
unicode_normalization: Literal["NFC", "NFKC", "NFD", "NFKD"] | None = None,
ascii_only: bool = False
) -> None
```
Initialize a DocumentPreProcessor that first splits and then cleans documents.
**Splitter Parameters**:
**Parameters:**
- **split_by** (<code>Literal['function', 'page', 'passage', 'period', 'word', 'line', 'sentence']</code>) – The unit of splitting: "function", "page", "passage", "period", "word", "line", or "sentence".
- **split_length** (<code>int</code>) – The maximum number of units (words, lines, pages, and so on) in each split.
- **split_overlap** (<code>int</code>) – The number of overlapping units between consecutive splits.
- **split_threshold** (<code>int</code>) – The minimum number of units per split. If a split is smaller than this, it's merged
with the previous split.
- **splitting_function** (<code>Callable\\[[str\], list\[str\]\] | None</code>) – A custom function for splitting if `split_by="function"`.
- **respect_sentence_boundary** (<code>bool</code>) – If `True`, splits by words but tries not to break inside a sentence.
- **language** (<code>Language</code>) – Language used by the sentence tokenizer if `split_by="sentence"` or
`respect_sentence_boundary=True`.
- **use_split_rules** (<code>bool</code>) – Whether to apply additional splitting heuristics for the sentence splitter.
- **extend_abbreviations** (<code>bool</code>) – Whether to extend the sentence splitter with curated abbreviations for certain
languages.
**Cleaner Parameters**:
- **remove_empty_lines** (<code>bool</code>) – If `True`, removes empty lines.
- **remove_extra_whitespaces** (<code>bool</code>) – If `True`, removes extra whitespaces.
- **remove_repeated_substrings** (<code>bool</code>) – If `True`, removes repeated substrings like headers/footers across pages.
- **keep_id** (<code>bool</code>) – If `True`, keeps the original document IDs.
- **remove_substrings** (<code>list\[str\] | None</code>) – A list of strings to remove from the document content.
- **remove_regex** (<code>str | None</code>) – A regex pattern whose matches will be removed from the document content.
- **unicode_normalization** (<code>Literal['NFC', 'NFKC', 'NFD', 'NFKD'] | None</code>) – Unicode normalization form to apply to the text, for example `"NFC"`.
- **ascii_only** (<code>bool</code>) – If `True`, converts text to ASCII only.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serialize SuperComponent to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> DocumentPreprocessor
```
Deserializes the SuperComponent from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>DocumentPreprocessor</code> – Deserialized SuperComponent.
## document_splitter
### DocumentSplitter
Splits long documents into smaller chunks.
This is a common preprocessing step during indexing. It helps Embedders create meaningful semantic representations
and prevents exceeding language model context limits.
The DocumentSplitter is compatible with the following DocumentStores:
- [Astra](https://docs.haystack.deepset.ai/docs/astradocumentstore)
- [Chroma](https://docs.haystack.deepset.ai/docs/chromadocumentstore) limited support, overlapping information is
not stored
- [Elasticsearch](https://docs.haystack.deepset.ai/docs/elasticsearch-document-store)
- [OpenSearch](https://docs.haystack.deepset.ai/docs/opensearch-document-store)
- [Pgvector](https://docs.haystack.deepset.ai/docs/pgvectordocumentstore)
- [Pinecone](https://docs.haystack.deepset.ai/docs/pinecone-document-store) limited support, overlapping
information is not stored
- [Qdrant](https://docs.haystack.deepset.ai/docs/qdrant-document-store)
- [Weaviate](https://docs.haystack.deepset.ai/docs/weaviatedocumentstore)
### Usage example
```python
from haystack import Document
from haystack.components.preprocessors import DocumentSplitter
doc = Document(content="Moonlight shimmered softly, wolves howled nearby, night enveloped everything.")
splitter = DocumentSplitter(split_by="word", split_length=3, split_overlap=0)
result = splitter.run(documents=[doc])
```
#### __init__
```python
__init__(
split_by: Literal[
"function", "page", "passage", "period", "word", "line", "sentence"
] = "word",
split_length: int = 200,
split_overlap: int = 0,
split_threshold: int = 0,
splitting_function: Callable[[str], list[str]] | None = None,
respect_sentence_boundary: bool = False,
language: Language = "en",
use_split_rules: bool = True,
extend_abbreviations: bool = True,
*,
skip_empty_documents: bool = True
) -> None
```
Initialize DocumentSplitter.
**Parameters:**
- **split_by** (<code>Literal['function', 'page', 'passage', 'period', 'word', 'line', 'sentence']</code>) – The unit for splitting your documents. Choose from:
- `word` for splitting by spaces (" ")
- `period` for splitting by periods (".")
- `page` for splitting by form feed ("\\f")
- `passage` for splitting by double line breaks ("\\n\\n")
- `line` for splitting each line ("\\n")
- `sentence` for splitting by NLTK sentence tokenizer
- **split_length** (<code>int</code>) – The maximum number of units in each split.
- **split_overlap** (<code>int</code>) – The number of overlapping units for each split.
- **split_threshold** (<code>int</code>) – The minimum number of units per split. If a split has fewer units
than the threshold, it's attached to the previous split.
- **splitting_function** (<code>Callable\\[[str\], list\[str\]\] | None</code>) – Necessary when `split_by` is set to "function".
This is a function which must accept a single `str` as input and return a `list` of `str` as output,
representing the chunks after splitting.
- **respect_sentence_boundary** (<code>bool</code>) – Choose whether to respect sentence boundaries when splitting by "word".
If True, uses NLTK to detect sentence boundaries, ensuring splits occur only between sentences.
- **language** (<code>Language</code>) – Choose the language for the NLTK tokenizer. The default is English ("en").
- **use_split_rules** (<code>bool</code>) – Choose whether to use additional split rules when splitting by `sentence`.
- **extend_abbreviations** (<code>bool</code>) – Choose whether to extend NLTK's PunktTokenizer abbreviations with a list
of curated abbreviations, if available. This is currently supported for English ("en") and German ("de").
- **skip_empty_documents** (<code>bool</code>) – Choose whether to skip documents with empty content. Default is True.
Set to False when downstream components in the Pipeline (like LLMDocumentContentExtractor) can extract text
from non-textual documents.
#### warm_up
```python
warm_up() -> None
```
Warm up the DocumentSplitter by loading the sentence tokenizer.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Split documents into smaller parts.
Splits documents by the unit expressed in `split_by`, with a length of `split_length`
and an overlap of `split_overlap`.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – The documents to split.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: List of documents with the split texts. Each document includes:
- A metadata field `source_id` to track the original document.
- A metadata field `page_number` to track the original page number.
- All other metadata copied from the original document.
**Raises:**
- <code>TypeError</code> – if the input is not a list of Documents.
- <code>ValueError</code> – if the content of a document is None.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> DocumentSplitter
```
Deserializes the component from a dictionary.
## embedding_based_document_splitter
### EmbeddingBasedDocumentSplitter
Splits documents based on embedding similarity using cosine distances between sequential sentence groups.
This component first splits text into sentences, optionally groups them, calculates embeddings for each group,
and then uses cosine distance between sequential embeddings to determine split points. Any distance above
the specified percentile is treated as a break point. The component also tracks page numbers based on form feed
characters (` `) in the original document.
This component is inspired by [5 Levels of Text Splitting](https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb) by Greg Kamradt.
### Usage example
```python
from haystack import Document
from haystack.components.embedders import SentenceTransformersDocumentEmbedder
from haystack.components.preprocessors import EmbeddingBasedDocumentSplitter
# Create a document with content that has a clear topic shift
doc = Document(
content="This is a first sentence. This is a second sentence. This is a third sentence. "
"Completely different topic. The same completely different topic."
)
# Initialize the embedder to calculate semantic similarities
embedder = SentenceTransformersDocumentEmbedder()
# Configure the splitter with parameters that control splitting behavior
splitter = EmbeddingBasedDocumentSplitter(
document_embedder=embedder,
sentences_per_group=2, # Group 2 sentences before calculating embeddings
percentile=0.95, # Split when cosine distance exceeds 95th percentile
min_length=50, # Merge splits shorter than 50 characters
max_length=1000 # Further split chunks longer than 1000 characters
)
result = splitter.run(documents=[doc])
# The result contains a list of Document objects, each representing a semantic chunk
# Each split document includes metadata: source_id, split_id, and page_number
print(f"Original document split into {len(result['documents'])} chunks")
for i, split_doc in enumerate(result['documents']):
print(f"Chunk {i}: {split_doc.content[:50]}...")
```
#### __init__
```python
__init__(
*,
document_embedder: DocumentEmbedder,
sentences_per_group: int = 3,
percentile: float = 0.95,
min_length: int = 50,
max_length: int = 1000,
language: Language = "en",
use_split_rules: bool = True,
extend_abbreviations: bool = True
) -> None
```
Initialize EmbeddingBasedDocumentSplitter.
**Parameters:**
- **document_embedder** (<code>DocumentEmbedder</code>) – The DocumentEmbedder to use for calculating embeddings.
- **sentences_per_group** (<code>int</code>) – Number of sentences to group together before embedding.
- **percentile** (<code>float</code>) – Percentile threshold for cosine distance. Distances above this percentile
are treated as break points.
- **min_length** (<code>int</code>) – Minimum length of splits in characters. Splits below this length will be merged.
- **max_length** (<code>int</code>) – Maximum length of splits in characters. Splits above this length will be recursively split.
- **language** (<code>Language</code>) – Language for sentence tokenization.
- **use_split_rules** (<code>bool</code>) – Whether to use additional split rules for sentence tokenization. Applies additional
split rules from SentenceSplitter to the sentence spans.
- **extend_abbreviations** (<code>bool</code>) – If True, the abbreviations used by NLTK's PunktTokenizer are extended by a list
of curated abbreviations. Currently supported languages are: en, de.
If False, the default abbreviations are used.
#### warm_up
```python
warm_up() -> None
```
Warm up the component by initializing the sentence splitter.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Split documents based on embedding similarity.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – The documents to split.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: List of documents with the split texts. Each document includes:
- A metadata field `source_id` to track the original document.
- A metadata field `split_id` to track the split number.
- A metadata field `page_number` to track the original page number.
- All other metadata copied from the original document.
**Raises:**
- <code>RuntimeError</code> – If the component wasn't warmed up.
- <code>TypeError</code> – If the input is not a list of Documents.
- <code>ValueError</code> – If the document content is None or empty.
#### run_async
```python
run_async(documents: list[Document]) -> dict[str, list[Document]]
```
Asynchronously split documents based on embedding similarity.
This is the asynchronous version of the `run` method with the same parameters and return values.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – The documents to split.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: List of documents with the split texts. Each document includes:
- A metadata field `source_id` to track the original document.
- A metadata field `split_id` to track the split number.
- A metadata field `page_number` to track the original page number.
- All other metadata copied from the original document.
**Raises:**
- <code>RuntimeError</code> – If the component wasn't warmed up.
- <code>TypeError</code> – If the input is not a list of Documents.
- <code>ValueError</code> – If the document content is None or empty.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Serialized dictionary representation of the component.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> EmbeddingBasedDocumentSplitter
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize and create the component.
**Returns:**
- <code>EmbeddingBasedDocumentSplitter</code> – The deserialized component.
## hierarchical_document_splitter
### HierarchicalDocumentSplitter
Splits a documents into different block sizes building a hierarchical tree structure of blocks of different sizes.
The root node of the tree is the original document, the leaf nodes are the smallest blocks. The blocks in between
are connected such that the smaller blocks are children of the parent-larger blocks.
## Usage example
```python
from haystack import Document
from haystack.components.preprocessors import HierarchicalDocumentSplitter
doc = Document(content="This is a simple test document")
splitter = HierarchicalDocumentSplitter(block_sizes={3, 2}, split_overlap=0, split_by="word")
splitter.run([doc])
# >> {'documents': [Document(id=3f7..., content: 'This is a simple test document', meta: {'block_size': 0, 'parent_id': None, 'children_ids': ['5ff..', '8dc..'], 'level': 0}),
# >> Document(id=5ff.., content: 'This is a ', meta: {'block_size': 3, 'parent_id': '3f7..', 'children_ids': ['f19..', '52c..'], 'level': 1, 'source_id': '3f7..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
# >> Document(id=8dc.., content: 'simple test document', meta: {'block_size': 3, 'parent_id': '3f7..', 'children_ids': ['39d..', 'e23..'], 'level': 1, 'source_id': '3f7..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 10}),
# >> Document(id=f19.., content: 'This is ', meta: {'block_size': 2, 'parent_id': '5ff..', 'children_ids': [], 'level': 2, 'source_id': '5ff..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
# >> Document(id=52c.., content: 'a ', meta: {'block_size': 2, 'parent_id': '5ff..', 'children_ids': [], 'level': 2, 'source_id': '5ff..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 8}),
# >> Document(id=39d.., content: 'simple test ', meta: {'block_size': 2, 'parent_id': '8dc..', 'children_ids': [], 'level': 2, 'source_id': '8dc..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
# >> Document(id=e23.., content: 'document', meta: {'block_size': 2, 'parent_id': '8dc..', 'children_ids': [], 'level': 2, 'source_id': '8dc..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 12})]}
```
#### __init__
```python
__init__(
block_sizes: set[int],
split_overlap: int = 0,
split_by: Literal["word", "sentence", "page", "passage"] = "word",
) -> None
```
Initialize HierarchicalDocumentSplitter.
**Parameters:**
- **block_sizes** (<code>set\[int\]</code>) – Set of block sizes to split the document into. The blocks are split in descending order.
- **split_overlap** (<code>int</code>) – The number of overlapping units for each split.
- **split_by** (<code>Literal['word', 'sentence', 'page', 'passage']</code>) – The unit for splitting your documents.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Builds a hierarchical document structure for each document in a list of documents.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of Documents to split into hierarchical blocks.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – List of HierarchicalDocument
#### build_hierarchy_from_doc
```python
build_hierarchy_from_doc(document: Document) -> list[Document]
```
Build a hierarchical tree document structure from a single document.
Given a document, this function splits the document into hierarchical blocks of different sizes represented
as HierarchicalDocument objects.
**Parameters:**
- **document** (<code>Document</code>) – Document to split into hierarchical blocks.
**Returns:**
- <code>list\[Document\]</code> – List of HierarchicalDocument
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Returns a dictionary representation of the component.
**Returns:**
- <code>dict\[str, Any\]</code> – Serialized dictionary representation of the component.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> HierarchicalDocumentSplitter
```
Deserialize this component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – The dictionary to deserialize and create the component.
**Returns:**
- <code>HierarchicalDocumentSplitter</code> – The deserialized component.
## markdown_header_splitter
### MarkdownHeaderSplitter
Split documents at ATX-style Markdown headers (#), with optional secondary splitting.
This component processes text documents by:
- Splitting them into chunks at Markdown headers (e.g., '#', '##', etc.), preserving header hierarchy as metadata.
- Optionally applying a secondary split (by word, passage, period, or line) to each chunk
(using haystack's DocumentSplitter).
- Preserving and propagating metadata such as parent headers, page numbers, and split IDs.
#### __init__
```python
__init__(
*,
page_break_character: str = "\x0c",
keep_headers: bool = True,
header_split_levels: list[int] | None = None,
secondary_split: Literal["word", "passage", "period", "line"] | None = None,
split_length: int = 200,
split_overlap: int = 0,
split_threshold: int = 0,
skip_empty_documents: bool = True
) -> None
```
Initialize the MarkdownHeaderSplitter.
**Parameters:**
- **page_break_character** (<code>str</code>) – Character used to identify page breaks. Defaults to form feed (" ").
- **keep_headers** (<code>bool</code>) – If True, headers are kept in the content. If False, headers are moved to metadata.
Defaults to True.
- **header_split_levels** (<code>list\[int\] | None</code>) – List of header levels (1–6) to split on. For example, `[1, 2]` splits only
on `#` and `##` headers, merging content under deeper headers into the preceding chunk. Defaults to
all levels `[1, 2, 3, 4, 5, 6]`.
- **secondary_split** (<code>Literal['word', 'passage', 'period', 'line'] | None</code>) – Optional secondary split condition after header splitting.
Options are None, "word", "passage", "period", "line". Defaults to None.
- **split_length** (<code>int</code>) – The maximum number of units in each split when using secondary splitting. Defaults to 200.
- **split_overlap** (<code>int</code>) – The number of overlapping units for each split when using secondary splitting.
Defaults to 0.
- **split_threshold** (<code>int</code>) – The minimum number of units per split when using secondary splitting. Defaults to 0.
- **skip_empty_documents** (<code>bool</code>) – Choose whether to skip documents with empty content. Default is True.
Set to False when downstream components in the Pipeline (like LLMDocumentContentExtractor) can extract text
from non-textual documents.
#### warm_up
```python
warm_up() -> None
```
Warm up the MarkdownHeaderSplitter.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Run the markdown header splitter with optional secondary splitting.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of documents to split
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary with the following key:
- `documents`: List of documents with the split texts. Each document includes:
- A metadata field `source_id` to track the original document.
- A metadata field `page_number` to track the original page number.
- A metadata field `split_id` to identify the split chunk index within its parent document.
- All other metadata copied from the original document.
**Raises:**
- <code>ValueError</code> – If a document has `None` content.
- <code>TypeError</code> – If a document's content is not a string.
## recursive_splitter
### RecursiveDocumentSplitter
Recursively chunk text into smaller chunks.
This component is used to split text into smaller chunks, it does so by recursively applying a list of separators
to the text.
The separators are applied in the order they are provided, typically this is a list of separators that are
applied in a specific order, being the last separator the most specific one.
Each separator is applied to the text, it then checks each of the resulting chunks, it keeps the chunks that
are within the split_length, for the ones that are larger than the split_length, it applies the next separator in the
list to the remaining text.
This is done until all chunks are smaller than the split_length parameter.
Example:
```python
from haystack import Document
from haystack.components.preprocessors import RecursiveDocumentSplitter
chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\n\n", "\n", ".", " "])
text = ('''Artificial intelligence (AI) - Introduction
AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
AI technology is widely used throughout industry, government, and science. Some high-profile applications include advanced web search engines; recommendation systems; interacting via human speech; autonomous vehicles; generative and creative tools; and superhuman play and analysis in strategy games.''')
doc = Document(content=text)
doc_chunks = chunker.run([doc])
print(doc_chunks["documents"])
# [
# Document(id=..., content: 'Artificial intelligence (AI) - Introduction\n\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []})
# Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []})
# Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []})
# Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []})
# ]
```
#### __init__
```python
__init__(
*,
split_length: int = 200,
split_overlap: int = 0,
split_unit: Literal["word", "char", "token"] = "word",
separators: list[str] | None = None,
sentence_splitter_params: dict[str, Any] | None = None
) -> None
```
Initializes a RecursiveDocumentSplitter.
**Parameters:**
- **split_length** (<code>int</code>) – The maximum length of each chunk by default in words, but can be in characters or tokens.
See the `split_units` parameter.
- **split_overlap** (<code>int</code>) – The number of characters to overlap between consecutive chunks.
- **split_unit** (<code>Literal['word', 'char', 'token']</code>) – The unit of the split_length parameter. It can be either "word", "char", or "token".
If "token" is selected, the text will be split into tokens using the tiktoken tokenizer (o200k_base).
- **separators** (<code>list\[str\] | None</code>) – An optional list of separator strings to use for splitting the text. The string
separators will be treated as regular expressions unless the separator is "sentence", in that case the
text will be split into sentences using a custom sentence tokenizer based on NLTK.
See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter.
If no separators are provided, the default separators ["\\n\\n", "sentence", "\\n", " "] are used.
- **sentence_splitter_params** (<code>dict\[str, Any\] | None</code>) – Optional parameters to pass to the sentence tokenizer.
See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter for more information.
**Raises:**
- <code>ValueError</code> – If the overlap is greater than or equal to the chunk size or if the overlap is negative, or
if any separator is not a string.
#### warm_up
```python
warm_up() -> None
```
Warm up the sentence tokenizer and tiktoken tokenizer if needed.
#### run
```python
run(documents: list[Document]) -> dict[str, list[Document]]
```
Split a list of documents into documents with smaller chunks of text.
**Parameters:**
- **documents** (<code>list\[Document\]</code>) – List of Documents to split.
**Returns:**
- <code>dict\[str, list\[Document\]\]</code> – A dictionary containing a key "documents" with a List of Documents with smaller chunks of text corresponding
to the input documents.
## text_cleaner
### TextCleaner
Cleans text strings.
It can remove substrings matching a list of regular expressions, convert text to lowercase,
remove punctuation, and remove numbers.
Use it to clean up text data before evaluation.
### Usage example
```python
from haystack.components.preprocessors import TextCleaner
text_to_clean = "1Moonlight shimmered softly, 300 Wolves howled nearby, Night enveloped everything."
cleaner = TextCleaner(convert_to_lowercase=True, remove_punctuation=False, remove_numbers=True)
result = cleaner.run(texts=[text_to_clean])
```
#### __init__
```python
__init__(
remove_regexps: list[str] | None = None,
convert_to_lowercase: bool = False,
remove_punctuation: bool = False,
remove_numbers: bool = False,
) -> None
```
Initializes the TextCleaner component.
**Parameters:**
- **remove_regexps** (<code>list\[str\] | None</code>) – A list of regex patterns to remove matching substrings from the text.
- **convert_to_lowercase** (<code>bool</code>) – If `True`, converts all characters to lowercase.
- **remove_punctuation** (<code>bool</code>) – If `True`, removes punctuation from the text.
- **remove_numbers** (<code>bool</code>) – If `True`, removes numerical digits from the text.
#### run
```python
run(texts: list[str]) -> dict[str, Any]
```
Cleans up the given list of strings.
**Parameters:**
- **texts** (<code>list\[str\]</code>) – List of strings to clean.
**Returns:**
- <code>dict\[str, Any\]</code> – A dictionary with the following key:
- `texts`: the cleaned list of strings.
docs-website/reference_versioned_docs/version-2.30/haystack-api/query_api.md
+129
View File
@@ -0,0 +1,129 @@
---
title: "Query"
id: query-api
description: "Components for query processing and expansion."
slug: "/query-api"
---
## query_expander
### QueryExpander
A component that returns a list of semantically similar queries to improve retrieval recall in RAG systems.
The component uses a chat generator to expand queries. The chat generator is expected to return a JSON response
with the following structure:
```json
{"queries": ["expanded query 1", "expanded query 2", "expanded query 3"]}
```
### Usage example
```python
from haystack.components.generators.chat.openai import OpenAIChatGenerator
from haystack.components.query import QueryExpander
expander = QueryExpander(
chat_generator=OpenAIChatGenerator(model="gpt-4.1-mini"),
n_expansions=3
)
result = expander.run(query="green energy sources")
print(result["queries"])
# Output: ['alternative query 1', 'alternative query 2', 'alternative query 3', 'green energy sources']
# Note: Up to 3 additional queries + 1 original query (if include_original_query=True)
# To control total number of queries:
expander = QueryExpander(n_expansions=2, include_original_query=True) # Up to 3 total
# or
expander = QueryExpander(n_expansions=3, include_original_query=False) # Exactly 3 total
```
#### __init__
```python
__init__(
*,
chat_generator: ChatGenerator | None = None,
prompt_template: str | None = None,
n_expansions: int = 4,
include_original_query: bool = True
) -> None
```
Initialize the QueryExpander component.
**Parameters:**
- **chat_generator** (<code>ChatGenerator | None</code>) – The chat generator component to use for query expansion.
If None, a default OpenAIChatGenerator with gpt-4.1-mini model is used.
- **prompt_template** (<code>str | None</code>) – Custom [PromptBuilder](https://docs.haystack.deepset.ai/docs/promptbuilder)
template for query expansion. The template should instruct the LLM to return a JSON response with the
structure: `{"queries": ["query1", "query2", "query3"]}`. The template should include 'query' and
'n_expansions' variables.
- **n_expansions** (<code>int</code>) – Number of alternative queries to generate (default: 4).
- **include_original_query** (<code>bool</code>) – Whether to include the original query in the output.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> QueryExpander
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary with serialized data.
**Returns:**
- <code>QueryExpander</code> – Deserialized component.
#### run
```python
run(query: str, n_expansions: int | None = None) -> dict[str, list[str]]
```
Expand the input query into multiple semantically similar queries.
The language of the original query is preserved in the expanded queries.
**Parameters:**
- **query** (<code>str</code>) – The original query to expand.
- **n_expansions** (<code>int | None</code>) – Number of additional queries to generate (not including the original).
If None, uses the value from initialization. Can be 0 to generate no additional queries.
**Returns:**
- <code>dict\[str, list\[str\]\]</code> – Dictionary with "queries" key containing the list of expanded queries.
If include_original_query=True, the original query will be included in addition
to the n_expansions alternative queries.
**Raises:**
- <code>ValueError</code> – If n_expansions is not positive (less than or equal to 0).
#### warm_up
```python
warm_up() -> None
```
Warm up the LLM provider component.
docs-website/reference_versioned_docs/version-2.30/haystack-api/rankers_api.md
+1158
View File
File diff suppressed because it is too large Load Diff
docs-website/reference_versioned_docs/version-2.30/haystack-api/readers_api.md
+193
View File
@@ -0,0 +1,193 @@
---
title: "Readers"
id: readers-api
description: "Takes a query and a set of Documents as input and returns ExtractedAnswers by selecting a text span within the Documents."
slug: "/readers-api"
---
## extractive
### ExtractiveReader
Locates and extracts answers to a given query from Documents.
The ExtractiveReader component performs extractive question answering.
It assigns a score to every possible answer span independently of other answer spans.
This fixes a common issue of other implementations which make comparisons across documents harder by normalizing
each document's answers independently.
Example usage:
```python
from haystack import Document
from haystack.components.readers import ExtractiveReader
docs = [
Document(content="Python is a popular programming language"),
Document(content="python ist eine beliebte Programmiersprache"),
]
reader = ExtractiveReader()
question = "What is a popular programming language?"
result = reader.run(query=question, documents=docs)
assert "Python" in result["answers"][0].data
```
#### __init__
```python
__init__(
model: Path | str = "deepset/roberta-base-squad2-distilled",
device: ComponentDevice | None = None,
token: Secret | None = Secret.from_env_var(
["HF_API_TOKEN", "HF_TOKEN"], strict=False
),
top_k: int = 20,
score_threshold: float | None = None,
max_seq_length: int = 384,
stride: int = 128,
max_batch_size: int | None = None,
answers_per_seq: int | None = None,
no_answer: bool = True,
calibration_factor: float = 0.1,
overlap_threshold: float | None = 0.01,
model_kwargs: dict[str, Any] | None = None,
) -> None
```
Creates an instance of ExtractiveReader.
**Parameters:**
- **model** (<code>Path | str</code>) – A Hugging Face transformers question answering model.
Can either be a path to a folder containing the model files or an identifier for the Hugging Face hub.
- **device** (<code>ComponentDevice | None</code>) – The device on which the model is loaded. If `None`, the default device is automatically selected.
- **token** (<code>Secret | None</code>) – The API token used to download private models from Hugging Face.
- **top_k** (<code>int</code>) – Number of answers to return per query. It is required even if score_threshold is set.
An additional answer with no text is returned if no_answer is set to True (default).
- **score_threshold** (<code>float | None</code>) – Returns only answers with the probability score above this threshold.
- **max_seq_length** (<code>int</code>) – Maximum number of tokens. If a sequence exceeds it, the sequence is split.
- **stride** (<code>int</code>) – Number of tokens that overlap when sequence is split because it exceeds max_seq_length.
- **max_batch_size** (<code>int | None</code>) – Maximum number of samples that are fed through the model at the same time.
- **answers_per_seq** (<code>int | None</code>) – Number of answer candidates to consider per sequence.
This is relevant when a Document was split into multiple sequences because of max_seq_length.
- **no_answer** (<code>bool</code>) – Whether to return an additional `no answer` with an empty text and a score representing the
probability that the other top_k answers are incorrect.
- **calibration_factor** (<code>float</code>) – Factor used for calibrating probabilities.
- **overlap_threshold** (<code>float | None</code>) – If set this will remove duplicate answers if they have an overlap larger than the
supplied threshold. For example, for the answers "in the river in Maine" and "the river" we would remove
one of these answers since the second answer has a 100% (1.0) overlap with the first answer.
However, for the answers "the river in" and "in Maine" there is only a max overlap percentage of 25% so
both of these answers could be kept if this variable is set to 0.24 or lower.
If None is provided then all answers are kept.
- **model_kwargs** (<code>dict\[str, Any\] | None</code>) – Additional keyword arguments passed to `AutoModelForQuestionAnswering.from_pretrained`
when loading the model specified in `model`. For details on what kwargs you can pass,
see the model's documentation.
#### to_dict
```python
to_dict() -> dict[str, Any]
```
Serializes the component to a dictionary.
**Returns:**
- <code>dict\[str, Any\]</code> – Dictionary with serialized data.
#### from_dict
```python
from_dict(data: dict[str, Any]) -> ExtractiveReader
```
Deserializes the component from a dictionary.
**Parameters:**
- **data** (<code>dict\[str, Any\]</code>) – Dictionary to deserialize from.
**Returns:**
- <code>ExtractiveReader</code> – Deserialized component.
#### warm_up
```python
warm_up() -> None
```
Initializes the component.
#### deduplicate_by_overlap
```python
deduplicate_by_overlap(
answers: list[ExtractedAnswer], overlap_threshold: float | None
) -> list[ExtractedAnswer]
```
De-duplicates overlapping Extractive Answers.
De-duplicates overlapping Extractive Answers from the same document based on how much the spans of the
answers overlap.
**Parameters:**
- **answers** (<code>list\[ExtractedAnswer\]</code>) – List of answers to be deduplicated.
- **overlap_threshold** (<code>float | None</code>) – If set this will remove duplicate answers if they have an overlap larger than the
supplied threshold. For example, for the answers "in the river in Maine" and "the river" we would remove
one of these answers since the second answer has a 100% (1.0) overlap with the first answer.
However, for the answers "the river in" and "in Maine" there is only a max overlap percentage of 25% so
both of these answers could be kept if this variable is set to 0.24 or lower.
If None is provided then all answers are kept.
**Returns:**
- <code>list\[ExtractedAnswer\]</code> – List of deduplicated answers.
#### run
```python
run(
query: str,
documents: list[Document],
top_k: int | None = None,
score_threshold: float | None = None,
max_seq_length: int | None = None,
stride: int | None = None,
max_batch_size: int | None = None,
answers_per_seq: int | None = None,
no_answer: bool | None = None,
overlap_threshold: float | None = None,
) -> dict[str, Any]
```
Locates and extracts answers from the given Documents using the given query.
**Parameters:**
- **query** (<code>str</code>) – Query string.
- **documents** (<code>list\[Document\]</code>) – List of Documents in which you want to search for an answer to the query.
- **top_k** (<code>int | None</code>) – The maximum number of answers to return.
An additional answer is returned if no_answer is set to True (default).
- **score_threshold** (<code>float | None</code>) – Returns only answers with the score above this threshold.
- **max_seq_length** (<code>int | None</code>) – Maximum number of tokens. If a sequence exceeds it, the sequence is split.
- **stride** (<code>int | None</code>) – Number of tokens that overlap when sequence is split because it exceeds max_seq_length.
- **max_batch_size** (<code>int | None</code>) – Maximum number of samples that are fed through the model at the same time.
- **answers_per_seq** (<code>int | None</code>) – Number of answer candidates to consider per sequence.
This is relevant when a Document was split into multiple sequences because of max_seq_length.
- **no_answer** (<code>bool | None</code>) – Whether to return no answer scores.
- **overlap_threshold** (<code>float | None</code>) – If set this will remove duplicate answers if they have an overlap larger than the
supplied threshold. For example, for the answers "in the river in Maine" and "the river" we would remove
one of these answers since the second answer has a 100% (1.0) overlap with the first answer.
However, for the answers "the river in" and "in Maine" there is only a max overlap percentage of 25% so
both of these answers could be kept if this variable is set to 0.24 or lower.
If None is provided then all answers are kept.
**Returns:**
- <code>dict\[str, Any\]</code> – List of answers sorted by (desc.) answer score.

Some files were not shown because too many files have changed in this diff Show More