Merge branch 'main' into fix/tool-span-error-tag

2026-10-12 02:12:50 +00:00 · 2026-10-09 10:19:35 +02:00
parent 6d5fc939ec ce356edf23
commit f16dea83f3
1785 changed files with 12555 additions and 2546 deletions
@@ -6,7 +6,7 @@
 # Pinned by digest for supply-chain integrity. Bump periodically (the OSS-Fuzz
 # base-builder is updated frequently with toolchain fixes); resolve a fresh digest with:
 #   docker buildx imagetools inspect gcr.io/oss-fuzz-base/base-builder-python:latest --format '{{.Manifest.Digest}}'
-FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:1de9e4ac301ddea42b8846480d5e4903580ed5071357c9173a4d26a4af65a6fa
+FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:7c97d379ceb773b0bd345f0ced72b037d26e9bb6c4a0c9a176525e65975c5694

 COPY . $SRC/haystack
 WORKDIR $SRC/haystack
@@ -57,7 +57,11 @@ if __name__ == "__main__":
    ### Docusaurus updates

    # copy docs to versioned_docs/version-target_unstable
-    shutil.copytree("docs-website/docs", f"docs-website/versioned_docs/version-{target_unstable}")
+    shutil.copytree(
+        src="docs-website/docs",
+        dst=f"docs-website/versioned_docs/version-{target_unstable}",
+        ignore=shutil.ignore_patterns("AGENTS.md"),
+    )

    # copy reference to reference_versioned_docs/version-target_unstable
    shutil.copytree("docs-website/reference", f"docs-website/reference_versioned_docs/version-{target_unstable}")
@@ -62,7 +62,7 @@ jobs:
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

-      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: "22"

@@ -79,7 +79,7 @@ jobs:

      - name: Set up Node.js
        if: steps.changed.outputs.needs_check == 'true'
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: "22"

@@ -24,7 +24,7 @@ jobs:
          persist-credentials: false

      - name: Install Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: "22"

@@ -63,7 +63,7 @@ jobs:
          fi

      - name: Upload result
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: docstring-label-result
          path: result/
@@ -22,7 +22,7 @@ jobs:
      actions: read
    steps:
      - name: Download result
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: docstring-label-result
          path: result
@@ -0,0 +1,159 @@
+name: Priority label sync
+
+# Mirror the priority label of the issues a PR links onto the PR. If the linked issues
+# carry different priorities, the highest one wins. Once no linked issue has a priority,
+# or the PR no longer links an issue, the priority labels this workflow added are removed
+# again; priority labels set by hand stay.
+#
+# We use "schedule" in addition to the "pull_request_target" and "issues" triggers to catch
+# links made via the "Development" sidebar, which emit no event.
+#
+# Uses pull_request_target so it also has write permissions on fork PRs; this
+# is safe because the workflow never checks out or executes PR code.
+
+on:
+  pull_request_target:
+    types: [opened, reopened, edited]
+  issues:
+    types: [labeled, unlabeled]
+  schedule:
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# A group keeps only one pending run, so events for other labels get a group of their own
+# to not cancel a pending priority run.
+concurrency:
+  group: >-
+    priority-label-sync-${{
+      (github.event_name == 'issues' && !contains(fromJSON('["P0", "P1", "P2", "P3"]'), github.event.label.name)) && github.run_id
+      || github.event.pull_request.number || github.event.issue.number || 'sweep'
+    }}
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    runs-on: ubuntu-slim
+    if: ${{ github.event_name != 'issues' || contains(fromJSON('["P0", "P1", "P2", "P3"]'), github.event.label.name) }}
+    permissions:
+      issues: read
+      pull-requests: write
+    steps:
+      - name: Sync priority labels from linked issues
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            // Ordered from highest to lowest priority.
+            const PRIORITIES = ["P0", "P1", "P2", "P3"];
+            // Actor of label events created with GITHUB_TOKEN.
+            const WORKFLOW_ACTOR = "github-actions";
+
+            const PR_FIELDS = `
+              number
+              labels(first: 100) { nodes { name } }
+              timelineItems(itemTypes: [LABELED_EVENT], last: 100) {
+                nodes { ... on LabeledEvent { label { name } actor { login } } }
+              }
+              closingIssuesReferences(first: 25) {
+                nodes {
+                  repository { nameWithOwner }
+                  labels(first: 100) { nodes { name } }
+                }
+              }`;
+
+            async function syncPr(pr) {
+              const issues = pr.closingIssuesReferences.nodes.filter(
+                (issue) => issue.repository.nameWithOwner === `${owner}/${repo}`,
+              );
+              const issuePriorities = new Set(
+                issues.flatMap((issue) => issue.labels.nodes.map((l) => l.name)),
+              );
+              const wanted = PRIORITIES.find((p) => issuePriorities.has(p));
+              const current = pr.labels.nodes.map((l) => l.name).filter((n) => PRIORITIES.includes(n));
+
+              // A label counts as added by this workflow if its most recent "labeled" event is ours.
+              // Labels whose event is out of the fetched range count as set by hand.
+              const addedBy = new Map();
+              for (const event of pr.timelineItems.nodes) addedBy.set(event.label.name, event.actor?.login);
+              const addedByWorkflow = (name) => addedBy.get(name) === WORKFLOW_ACTOR;
+
+              const toRemove = wanted ? current.filter((n) => n !== wanted) : current.filter(addedByWorkflow);
+              for (const name of toRemove) {
+                try {
+                  await github.rest.issues.removeLabel({ owner, repo, issue_number: pr.number, name });
+                } catch (error) {
+                  // Already removed, e.g. by a concurrent run.
+                  if (error.status !== 404) throw error;
+                }
+                core.info(`Removed ${name} from PR #${pr.number}`);
+              }
+              if (wanted && !current.includes(wanted)) {
+                await github.rest.issues.addLabels({ owner, repo, issue_number: pr.number, labels: [wanted] });
+                core.info(`Added ${wanted} to PR #${pr.number}`);
+              }
+            }
+
+            // Keep going when one PR fails so the others still get synced.
+            async function syncPrs(prs) {
+              const failed = [];
+              for (const pr of prs) {
+                try {
+                  await syncPr(pr);
+                } catch (error) {
+                  core.error(`Failed to sync PR #${pr.number}: ${error.message}`);
+                  failed.push(pr.number);
+                }
+              }
+              return failed;
+            }
+
+            let failed = [];
+            if (context.eventName === "pull_request_target") {
+              const result = await github.graphql(
+                `query($owner: String!, $repo: String!, $number: Int!) {
+                  repository(owner: $owner, name: $repo) { pullRequest(number: $number) { ${PR_FIELDS} } }
+                }`,
+                { owner, repo, number: context.payload.pull_request.number },
+              );
+              failed = await syncPrs([result.repository.pullRequest]);
+            } else if (context.eventName === "issues") {
+              const result = await github.graphql(
+                `query($owner: String!, $repo: String!, $number: Int!) {
+                  repository(owner: $owner, name: $repo) {
+                    issue(number: $number) {
+                      closedByPullRequestsReferences(first: 25, includeClosedPrs: false) {
+                        nodes { repository { nameWithOwner } ${PR_FIELDS} }
+                      }
+                    }
+                  }
+                }`,
+                { owner, repo, number: context.payload.issue.number },
+              );
+              const prs = result.repository.issue.closedByPullRequestsReferences.nodes.filter(
+                (pr) => pr.repository.nameWithOwner === `${owner}/${repo}`,
+              );
+              if (!prs.length) core.info(`Issue #${context.payload.issue.number} has no open linked PRs.`);
+              failed = await syncPrs(prs);
+            } else {
+              let cursor = null;
+              do {
+                const result = await github.graphql(
+                  `query($owner: String!, $repo: String!, $cursor: String) {
+                    repository(owner: $owner, name: $repo) {
+                      pullRequests(states: OPEN, first: 50, after: $cursor) {
+                        pageInfo { hasNextPage endCursor }
+                        nodes { ${PR_FIELDS} }
+                      }
+                    }
+                  }`,
+                  { owner, repo, cursor },
+                );
+                const page = result.repository.pullRequests;
+                failed.push(...(await syncPrs(page.nodes)));
+                cursor = page.pageInfo.hasNextPage ? page.pageInfo.endCursor : null;
+              } while (cursor);
+            }
+            if (failed.length) core.setFailed(`Failed to sync PRs: ${failed.map((n) => `#${n}`).join(", ")}`);
@@ -54,7 +54,7 @@ jobs:
          publish_results: true

      - name: Upload artifact
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: SARIF file
          path: results.sarif
@@ -170,7 +170,7 @@ jobs:

      - name: Upload coverage comment to be posted
        if: matrix.os == 'ubuntu-latest' && github.event_name == 'pull_request' && steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: python-coverage-comment-action
          path: python-coverage-comment-action.txt
@@ -72,12 +72,20 @@ jobs:
            SUMMARY="Initial generation of the platform components table."
          fi

+          PAGES_CHANGED=$(git diff --name-only HEAD -- \
+            'docs-website/docs/pipeline-components/**' \
+            'docs-website/docs/document-stores/**' \
+            'docs-website/docs/memory-stores/**' | wc -l | tr -d ' ')
+          PAGES_SUMMARY="Component doc pages relabeled (\`platform_availability\`): **${PAGES_CHANGED}**."
+
          {
            echo "body<<EOF"
            echo "## Platform Components Table Update"
            echo ""
            echo "${SUMMARY}"
            echo ""
+            echo "${PAGES_SUMMARY}"
+            echo ""
            echo "_Auto-generated by the [update-platform-components](https://github.com/deepset-ai/haystack/actions/workflows/update-platform-components.yml) workflow._"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
@@ -92,6 +100,9 @@ jobs:
          title: "chore: update platform components table"
          add-paths: |
            ${{ env.OUTPUT_FILE }}
+            docs-website/docs/pipeline-components/**
+            docs-website/docs/document-stores/**
+            docs-website/docs/memory-stores/**
          body: ${{ steps.pr_body.outputs.body }}
          labels: |
            documentation
@@ -94,7 +94,8 @@ Also see directory-specific guidelines:
 - Keep inline comments and private-helper docs to what is non-obvious — remove restatements, keep durable caveats and rationale
 - Keep docstrings current with signatures and behavior, in the existing Haystack style: each public `:param` by meaning, default, and constraints; `:returns:` contracts; exceptions in the existing `:raises ValueError:` style; aliases like `ToolsType` reflected — stale docs mislead users and assistants
 - Keep doc examples minimal, runnable, and local: default constructors with required env vars like `OPENAI_API_KEY` noted nearby, only the imports the snippet uses, no restated defaults (name a model only for model-specific behavior), expected output as comments, generally `# >> ...`
- When behavior, fields, or names change, update every surface in the same PR: `haystack/components/` docstrings and examples, `docs-website/docs/` plus the current `versioned_docs/version-*/` page (e.g. `concepts/data-classes.mdx`), and `experimental` wording, `pydoc` IDs, and generated markdown filenames when promoting features
+- When behavior, fields, or names change, update every relevant surface in the same PR: `haystack/components/` docstrings and examples, documentation pages, and `experimental` wording, `pydoc` IDs, and generated markdown filenames when promoting features
+- Choose which documentation versions to update: update `docs-website/docs/` by default; also update the latest stable `versioned_docs/version-X.Y/` pages for corrections to stable docs or already-released core integration features.

 ## Code Style

@@ -1,5 +1,7 @@
 import { VercelRequest, VercelResponse } from "@vercel/node";

+const DEEPSET_API_BASE = "https://api.cloud.deepset.ai/api/v1";
+
 export default async function handler(req: VercelRequest, res: VercelResponse) {
  if (req.method !== "POST") {
    res.setHeader("Allow", "POST");
@@ -12,10 +14,10 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
    return res.status(400).json({ error: "Query is required" });
  }

-  const { SEARCH_API_WORKSPACE, SEARCH_API_PIPELINE, SEARCH_API_TOKEN } =
+  const { SEARCH_API_WORKSPACE, SEARCH_API_DEPLOYMENT, SEARCH_API_TOKEN } =
    process.env;

-  if (!SEARCH_API_WORKSPACE || !SEARCH_API_PIPELINE || !SEARCH_API_TOKEN) {
+  if (!SEARCH_API_WORKSPACE || !SEARCH_API_DEPLOYMENT || !SEARCH_API_TOKEN) {
    console.error(
      "Search API environment variables are not configured on the server."
    );
@@ -23,9 +25,30 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
  }

  try {
+    const headers = {
+      "Content-Type": "application/json",
+      "X-Client-Source": "haystack-docs",
+      Authorization: `Bearer ${SEARCH_API_TOKEN}`,
+    };
+    const deploymentUrl = `${DEEPSET_API_BASE}/workspaces/${SEARCH_API_WORKSPACE}/deployments/${SEARCH_API_DEPLOYMENT}`;
+
+    // Deployment chat requires a search session
+    const sessionResponse = await fetch(`${deploymentUrl}/search_sessions`, {
+      method: "POST",
+      headers,
+    });
+    if (!sessionResponse.ok) {
+      console.error("Haystack API error:", await sessionResponse.text());
+      return res
+        .status(sessionResponse.status)
+        .json({ error: `API error: ${sessionResponse.statusText}` });
+    }
+    const { search_session_id } = await sessionResponse.json();
+
    // Build the request body with optional filters
    const requestBody: any = {
      queries: [query],
+      search_session_id,
    };

    // Add filters if provided (for future backend filtering support)
@@ -43,18 +66,11 @@ export default async function handler(req: VercelRequest, res: VercelResponse) {
      };
    }

-    const apiResponse = await fetch(
-      `https://api.cloud.deepset.ai/api/v1/workspaces/${SEARCH_API_WORKSPACE}/pipelines/${SEARCH_API_PIPELINE}/search`,
-      {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "X-Client-Source": "haystack-docs",
-          Authorization: `Bearer ${SEARCH_API_TOKEN}`,
-        },
-        body: JSON.stringify(requestBody),
-      }
-    );
+    const apiResponse = await fetch(`${deploymentUrl}/chat`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(requestBody),
+    });

    if (!apiResponse.ok) {
      const errorData = await apiResponse.text();
@@ -0,0 +1,50 @@
+---
+title: "Choosing Between Pipelines and Agents"
+id: choosing-between-pipelines-and-agents
+slug: "/choosing-between-pipelines-and-agents"
+description: "Pipelines and Agents solve the same problem - connecting components to answer a query - with a different tradeoff between predictability and freedom. Learn which one fits your use case, and how to combine them."
+---
+
+# Choosing Between Pipelines and Agents
+
+Pipelines and Agents solve the same problem - connecting components to answer a query - with a different tradeoff between predictability and freedom.
+
+If you came here to build a pipeline, you might actually be looking for an Agent, or the other way around. This page explains the difference so you can pick the right starting point.
+
+## The core tradeoff
+
+A [`Pipeline`](../pipelines.mdx) is a graph you wire yourself. You decide which components run, in what order, and how data flows between them. At runtime, the pipeline follows that graph exactly - the same input takes the same path every time, which makes pipelines predictable, inspectable, and reproducible to evaluate.
+
+An [`Agent`](../../pipeline-components/agents-1/agent.mdx) hands control flow to the LLM instead. Given a system prompt, a set of tools, and a goal, the agent decides at each step whether to call a tool, call another one, or stop and answer. That buys adaptability - the agent can handle requests you didn't enumerate in advance - at the cost of determinism: the number of steps, the tools used, the latency, and the cost of a single run are not fixed ahead of time.
+
+## When a pipeline is the right fit
+
+Reach for a `Pipeline` when:
+
+- The steps to answer a query are known in advance and don't depend on the query itself (for example, a RAG pipeline that always retrieves, then generates).
+- You need predictable latency and cost - a fixed pipeline shape means bounded LLM calls.
+- You need to audit or reproduce a specific run, or evaluate quality across a stable set of steps (see [Evaluation](../../optimization/evaluation.mdx)).
+
+## When an agent is the right fit
+
+Reach for an `Agent` when:
+
+- The number and order of steps depend on the query, or on what an earlier step returns - for example, a support request that may or may not need a database lookup, a web search, both, or neither.
+- The right tool to call can only be decided from intermediate results, not from the input alone.
+- The task is open-ended enough that hand-wiring every path would mean re-implementing the LLM's own reasoning.
+
+## They are not mutually exclusive
+
+An `Agent` is itself a Haystack component, so you can place one inside a `Pipeline` alongside deterministic steps - for example, a fixed preprocessing pipeline that hands off to an agent only for the parts of the task that need judgment.
+
+The reverse also works: wrap a whole pipeline as a tool with [`PipelineTool`](../../tools/pipelinetool.mdx) so an agent can call it as one predictable step in its own reasoning, without knowing what happens inside.
+
+If you need adaptability without giving up all predictability, [pipeline loops](../pipelines/pipeline-loops.mdx) are a middle ground: a fixed, bounded loop (for example, "retry generation until validation passes") without handing full control flow to the LLM.
+
+| You have... | Reach for... |
+| --- | --- |
+| A fixed sequence of steps, known upfront | `Pipeline` |
+| A bounded retry or refinement loop | `Pipeline` with a loop |
+| Steps that depend on the query or on intermediate results | `Agent` |
+| A pipeline that should be callable as one step by an agent | `Pipeline` wrapped in `PipelineTool` |
+| An agent that should run as one step inside a larger, fixed flow | `Agent` used as a pipeline component |
@@ -51,3 +51,9 @@ If you want to reuse pipelines, you can save them to disk in YAML format or shar

 Here is a short Haystack pipeline, illustrated:
 <ClickableImage src="/img/00f5fe8-Pipeline_Illustrations_2.png" alt="RAG architecture overview showing query flow through retrieval and generation stages, with document stores providing context for the language model" />
+
+### Agents
+
+A pipeline runs the same graph every time - predictable, but fixed. An [Agent](agents.mdx) hands step-by-step control to the LLM instead: it decides which tool to call next based on what the previous step returned, which suits open-ended tasks where the right sequence of steps isn't known upfront.
+
+An `Agent` is itself a component, so you can use one inside a pipeline, or wrap a whole pipeline as a tool for an agent to call. See [Choosing Between Pipelines and Agents](agents/choosing-between-pipelines-and-agents.mdx) for how to decide, and [Multi-Agent Systems](agents/multi-agent-systems.mdx) for composing several agents together.
@@ -22,6 +22,12 @@ Here are the currently available guides on Haystack pipeline deployment:
 - [Deploying with Kubernetes](deployment/kubernetes.mdx)
 - [Deploying with OpenShift](deployment/openshift.mdx)

+### Other Integrations
+
+| Integration | Description |
+| --- | --- |
+| [Ray](https://haystack.deepset.ai/integrations/ray) | Run and scale your pipelines in a distributed manner. |
+
 ### Hayhooks

 Haystack can be easily integrated into any HTTP application, but if you don’t have one, you can use Hayhooks, a ready-made application that serves Haystack pipelines as REST endpoints. We’ll be using Hayhooks throughout this guide to streamline the code examples. Refer to the Hayhooks [overview](hayhooks.mdx) for a quick start, or the [official Hayhooks documentation](https://deepset-ai.github.io/hayhooks/) for comprehensive guides and reference.
@@ -34,5 +40,5 @@ The **Haystack Enterprise Platform** gives you production infrastructure without
 - **Zero-downtime deploys** — auto-scaling replicas absorb demand spikes without dropping traffic
 - **Built-in testing and analytics** — a Playground for testing pipelines, plus dashboards for monitoring them in production

-[Explore the Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform)
+[Explore the Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform)
 :::
@@ -7,7 +7,7 @@ description: "Learn how to deploy your Haystack pipelines on Haystack Enterprise

 # Haystack Enterprise Platform

-Learn how to deploy your Haystack pipelines on [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform), without managing your own infrastructure.
+Learn how to deploy your Haystack pipelines on [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform), without managing your own infrastructure.

 ## Bringing a Pipeline onto the Platform

@@ -1,18 +0,0 @@
---
-title: "External Integrations"
-id: external-integrations-development
-slug: "/external-integrations-development"
-description: "External integrations that enable tracing, monitoring, and deploying your pipelines."
---
-
-# External Integrations
-
-External integrations that enable tracing, monitoring, and deploying your pipelines.
-
-| Name | Description |
-| --- | --- |
-| [Arize Phoenix](https://haystack.deepset.ai/integrations/arize-phoenix) | Trace your pipelines with Arize Phoenix.                 |
-| [Arize AI](https://haystack.deepset.ai/integrations/arize)              | Trace and monitor your pipelines with Arize AI.          |
-| [Burr](https://haystack.deepset.ai/integrations/burr)                   | Build Burr agents using Haystack.                        |
-| [Context AI](https://haystack.deepset.ai/integrations/context-ai)       | Log conversations for analytics by Context.ai.           |
-| [Ray](https://haystack.deepset.ai/integrations/ray)                     | Run and scale your pipelines with in distributed manner. |
@@ -120,3 +120,9 @@ We recommend JSON logging when deploying Haystack to production. Haystack will a
 ### Disabling Structured Logging

 To disable structured logging despite an existing installation of structlog, set the environment variable `HAYSTACK_LOGGING_IGNORE_STRUCTLOG` to `true` when running Haystack.
+
+## Other Integrations
+
+| Integration | Description |
+| --- | --- |
+| [Context AI](https://haystack.deepset.ai/integrations/context-ai) | Log conversations for analytics by Context.ai. |
@@ -13,7 +13,7 @@ Instrumented applications typically send traces to a trace collector or a tracin

 :::info[Tracing on Haystack Enterprise Platform]

-Deploy your pipeline on [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform) and get traces out of the box, no tracer setup needed. See [Haystack Enterprise Platform](tracing/haystack-enterprise-platform.mdx) for details.
+Deploy your pipeline on [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform) and get traces out of the box, no tracer setup needed. See [Haystack Enterprise Platform](tracing/haystack-enterprise-platform.mdx) for details.

 :::

@@ -21,7 +21,7 @@ Deploy your pipeline on [Haystack Enterprise Platform](https://www.deepset.ai/pr

 | Tracer | Description |
 | --- | --- |
-| [Haystack Enterprise Platform](tracing/haystack-enterprise-platform.mdx) | Get pipeline traces automatically when you deploy on [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform), no tracer setup needed. |
+| [Haystack Enterprise Platform](tracing/haystack-enterprise-platform.mdx) | Get pipeline traces automatically when you deploy on [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform), no tracer setup needed. |
 | [OpenTelemetry](tracing/opentelemetry.mdx) | Send traces to any [OpenTelemetry](https://opentelemetry.io/)-compatible backend using the `OpenTelemetryTracer` or the `OpenTelemetryConnector` component. Includes a Jaeger setup for local development. |
 | [MLflow](tracing/mlflow.mdx) | Capture traces with [MLflow](https://mlflow.org/)'s native Haystack tracing support. |
 | [Datadog](tracing/datadog.mdx) | Trace your pipelines with [Datadog](https://www.datadoghq.com/) using the `DatadogTracer` or the `DatadogConnector` component. |
@@ -29,6 +29,8 @@ Deploy your pipeline on [Haystack Enterprise Platform](https://www.deepset.ai/pr
 | [Weights & Biases Weave](tracing/weave.mdx) | Trace and visualize pipeline execution in [Weights & Biases](https://wandb.ai/site/) using the `WeaveTracer` or the `WeaveConnector` component. |
 | [Rhesis](tracing/rhesis.mdx) | Trace your pipelines and `Agent` runs in [Rhesis](https://rhesis.ai) using the `RhesisConnector` component, and correlate traces with test runs and conversation turns. |
 | [LoggingTracer](tracing/logging-tracer.mdx) | Inspect the data flowing through your pipeline in real time through logs, with no backend setup. |
+| [Arize Phoenix](https://haystack.deepset.ai/integrations/arize-phoenix) | Trace your pipelines with Arize Phoenix. |
+| [Arize AI](https://haystack.deepset.ai/integrations/arize) | Trace and monitor your pipelines with Arize AI. |
 | [Custom Tracer](tracing/custom-tracer.mdx) | Connect any tracing backend by implementing the `Tracer` interface. |

 ## Enabling and Disabling Tracing
@@ -7,7 +7,7 @@ description: "Learn how pipeline tracing works out of the box on Haystack Enterp

 # Haystack Enterprise Platform

-Learn how pipeline tracing works out of the box on [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform).
+Learn how pipeline tracing works out of the box on [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform).

 <div className="key-value-table">

@@ -2,6 +2,7 @@
 title: "InMemoryDocumentStore"
 id: inmemorydocumentstore
 slug: "/inmemorydocumentstore"
+hep_available: false
 ---

 # InMemoryDocumentStore
@@ -17,10 +17,20 @@ Haystack is an open-source AI orchestration framework that you can use to build

 Haystack is designed in a modular way, allowing you to combine the best technology from OpenAI, Google, Anthropic, and open-source projects like Hugging Face's Transformers.

-The core foundation of Haystack consists of components and pipelines, along with Document Stores, Agents, Tools, and many integrations. Read more about Haystack concepts in the [Haystack Concepts Overview](concepts/concepts-overview.mdx).
+The core foundation of Haystack consists of components and [pipelines](concepts/pipelines.mdx), along with Document Stores, [Agents](concepts/agents.mdx), Tools, and many integrations. A pipeline is a graph you wire yourself, so it runs the same way every time; an agent hands step-by-step control to the LLM, trading that predictability for the ability to handle requests you didn't enumerate in advance. See [Choosing Between Pipelines and Agents](concepts/agents/choosing-between-pipelines-and-agents.mdx) to decide which one your use case needs - and note that an agent can be a step inside a pipeline, and a pipeline can be a tool an agent calls, so it's rarely all-or-nothing. Read more about Haystack concepts in the [Haystack Concepts Overview](concepts/concepts-overview.mdx).

 Supported by an engaged community of developers, Haystack has grown into a comprehensive and user-friendly framework for LLM-based development.

+## Start here
+
+New to Haystack? Follow this path:
+
+1. [Installation](overview/installation.mdx) - install the `haystack-ai` package.
+2. [Get Started](overview/get-started.mdx) - build your first RAG pipeline and tool-calling agent.
+3. [Core Concepts](concepts/concepts-overview.mdx) - components, pipelines, agents, and the other building blocks.
+4. [Deployment](development/deployment.mdx) - serve your pipeline or agent as an HTTP endpoint.
+5. [Production Usage](production-usage.mdx) - evaluate, trace, and log it once it's running.
+
 :::tip[Looking to scale with confidence?]

 The **Haystack Enterprise Platform** gives you production infrastructure without the operational overhead:
@@ -29,5 +39,5 @@ The **Haystack Enterprise Platform** gives you production infrastructure without
 - **Zero-downtime deploys** — auto-scaling replicas absorb demand spikes without dropping traffic
 - **Built-in testing and analytics** — a Playground for testing pipelines, plus dashboards for monitoring them in production

-[Explore the Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform)
+[Explore the Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform)
 :::
@@ -14,7 +14,7 @@ Have a look at this page to learn how to quickly get up and running with Haystac

 :::info[Ready to scale with confidence?]

-[Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform) hosts the same Haystack components for you, with a visual Pipeline Builder, a [REST API](https://docs.cloud.deepset.ai/docs/create-a-pipeline-with-rest-api) for publishing pipeline YAML straight from your codebase, and support for [custom Python components](https://docs.cloud.deepset.ai/docs/upload-a-custom-component-to-deepset-cloud). Get zero-downtime deploys, managed scaling, and built-in tracing without running your own infrastructure. Follow the [Quick Start Guide](https://docs.cloud.deepset.ai/docs/quick-start-guide) to try it.
+[Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform) hosts the same Haystack components for you, with a visual Pipeline Builder, a [REST API](https://docs.cloud.deepset.ai/docs/create-a-pipeline-with-rest-api) for publishing pipeline YAML straight from your codebase, and support for [custom Python components](https://docs.cloud.deepset.ai/docs/upload-a-custom-component-to-deepset-cloud). Get zero-downtime deploys, managed scaling, and built-in tracing without running your own infrastructure. Follow the [Quick Start Guide](https://docs.cloud.deepset.ai/docs/quick-start-guide) to try it.

 :::

@@ -47,7 +47,7 @@ ImportError: "Haystack failed to import the optional dependency 'pypdf'. Run 'pi

 :::info[Ready to scale with confidence?]

-Rather than self-hosting, [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform) runs your pipelines for you: build them visually in the Pipeline Builder, publish pipeline YAML through the [REST API](https://docs.cloud.deepset.ai/docs/create-a-pipeline-with-rest-api), or bring your own [custom Python components](https://docs.cloud.deepset.ai/docs/upload-a-custom-component-to-deepset-cloud). You get zero-downtime deploys, managed scaling, and built-in tracing out of the box. See the [Quick Start Guide](https://docs.cloud.deepset.ai/docs/quick-start-guide) to get a pipeline running in minutes.
+Rather than self-hosting, [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform) runs your pipelines for you: build them visually in the Pipeline Builder, publish pipeline YAML through the [REST API](https://docs.cloud.deepset.ai/docs/create-a-pipeline-with-rest-api), or bring your own [custom Python components](https://docs.cloud.deepset.ai/docs/upload-a-custom-component-to-deepset-cloud). You get zero-downtime deploys, managed scaling, and built-in tracing out of the box. See the [Quick Start Guide](https://docs.cloud.deepset.ai/docs/quick-start-guide) to get a pipeline running in minutes.

 :::

@@ -7,7 +7,7 @@ description: "A complete list of Haystack components available on the Haystack E

 # Haystack Enterprise Components

-The [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform) currently supports **257 components** and **82 integrations**. The following table lists them grouped by integration partner.
+The [Haystack Enterprise Platform](https://www.deepset.ai/haystack-platform) currently supports **260 components** and **82 integrations**. The following table lists them grouped by integration partner.

 ## Core Components

@@ -60,6 +60,9 @@ The [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/
 | [MetadataRouter](https://docs.haystack.deepset.ai/docs/metadatarouter) | Router | ✅ Available |
 | [MetaFieldGroupingRanker](https://docs.haystack.deepset.ai/docs/metafieldgroupingranker) | Ranker | ✅ Available |
 | [MetaFieldRanker](https://docs.haystack.deepset.ai/docs/metafieldranker) | Ranker | ✅ Available |
+| [MockChatGenerator](https://docs.haystack.deepset.ai/docs/mockchatgenerator) | Generator | ✅ Available |
+| [MockDocumentEmbedder](https://docs.haystack.deepset.ai/docs/mockdocumentembedder) | Embedder | ✅ Available |
+| [MockTextEmbedder](https://docs.haystack.deepset.ai/docs/mocktextembedder) | Embedder | ✅ Available |
 | [MSGToDocument](https://docs.haystack.deepset.ai/docs/msgtodocument) | Converter | ✅ Available |
 | [MultiFileConverter](https://docs.haystack.deepset.ai/docs/multifileconverter) | Converter | ✅ Available |
 | [MultiQueryEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/multiqueryembeddingretriever) | Retriever | ✅ Available |
@@ -13,6 +13,8 @@ Haystack relies on anonymous usage statistics to continuously improve. That's wh

 Telemetry in Haystack comprises anonymous usage statistics of base components, such as `DocumentStore`, `Retriever`, `Reader`, or any other pipeline component. We receive an event every time these components are initialized. This way, we know which components are most relevant to our community. For the same reason, an event is also sent when one of the tutorials is executed.

+When a pipeline contains an `Agent`, the event also includes the Agent's chat generator type and model, and the type and name of each of its tools. Tool descriptions are never collected.
+
 Each event contains an anonymous, randomly generated user ID (`uuid`)  and a collection of properties about your execution environment. They **never** contain properties that can be used to identify you, such as:

 - IP addresses
@@ -3,6 +3,7 @@ title: "Agent"
 id: agent
 slug: "/agent"
 description: "The `Agent` component is a tool-using agent that interacts with chat-based LLMs and tools to solve complex queries iteratively. It can execute external tools, manage state across multiple LLM calls, and stop execution based on configurable `exit_conditions`."
+hep_available: true
 ---

 # Agent
@@ -3,6 +3,7 @@ title: "FunASRTranscriber"
 id: funasrtranscriber
 slug: "/funasrtranscriber"
 description: "Transcribe audio files to Documents using FunASR — a local, open-source speech recognition toolkit supporting 50+ languages."
+hep_available: false
 ---

 # FunASRTranscriber
@@ -3,6 +3,7 @@ title: "LocalWhisperTranscriber"
 id: localwhispertranscriber
 slug: "/localwhispertranscriber"
 description: "Use `LocalWhisperTranscriber` to transcribe audio files using OpenAI's Whisper model using your local installation of Whisper."
+hep_available: true
 ---

 # LocalWhisperTranscriber
@@ -3,6 +3,7 @@ title: "RemoteWhisperTranscriber"
 id: remotewhispertranscriber
 slug: "/remotewhispertranscriber"
 description: "Use `RemoteWhisperTranscriber` to transcribe audio files using OpenAI's Whisper model."
+hep_available: true
 ---

 # RemoteWhisperTranscriber
@@ -3,6 +3,7 @@ title: "AnswerBuilder"
 id: answerbuilder
 slug: "/answerbuilder"
 description: "Use this component in pipelines that contain a Generator to parse its replies."
+hep_available: true
 ---

 # AnswerBuilder
@@ -3,6 +3,7 @@ title: "ChatPromptBuilder"
 id: chatpromptbuilder
 slug: "/chatpromptbuilder"
 description: "This component constructs prompts dynamically by processing chat messages."
+hep_available: true
 ---

 # ChatPromptBuilder
@@ -3,6 +3,7 @@ title: "PromptBuilder"
 id: promptbuilder
 slug: "/promptbuilder"
 description: "Use this component in pipelines before a Generator to render a prompt template and fill in variable values."
+hep_available: true
 ---

 # PromptBuilder
@@ -3,6 +3,7 @@ title: "CacheChecker"
 id: cachechecker
 slug: "/cachechecker"
 description: "This component checks for the presence of documents in a Document Store based on a specified cache field."
+hep_available: true
 ---

 # CacheChecker
@@ -13,3 +13,4 @@ Use Classifiers to classify your documents by specific traits and update the met
 | --- | --- |
 | [DocumentLanguageClassifier](classifiers/documentlanguageclassifier.mdx)                       | Classify documents by language.                       |
 | [TransformersZeroShotDocumentClassifier](classifiers/transformerszeroshotdocumentclassifier.mdx) | Classify the documents based on the provided labels. |
+| [TypeSafeDocumentClassifier](classifiers/typesafedocumentclassifier.mdx) | Answer typed questions about documents with a TypeSafe System One model and store the answers in their metadata. |
@@ -3,6 +3,7 @@ title: "DocumentLanguageClassifier"
 id: documentlanguageclassifier
 slug: "/documentlanguageclassifier"
 description: "Use this component to classify documents by language and add language information to metadata."
+hep_available: true
 ---

 # DocumentLanguageClassifier
@@ -3,6 +3,7 @@ title: "TransformersZeroShotDocumentClassifier"
 id: transformerszeroshotdocumentclassifier
 slug: "/transformerszeroshotdocumentclassifier"
 description: "Classifies the documents based on the provided labels and adds them to their metadata."
+hep_available: true
 ---

 # TransformersZeroShotDocumentClassifier
@@ -0,0 +1,203 @@
+---
+title: "TypeSafeDocumentClassifier"
+id: typesafedocumentclassifier
+slug: "/typesafedocumentclassifier"
+description: "Answers typed questions about each document with a TypeSafe System One model, such as Jev, and stores the answers in its metadata."
+hep_available: false
+---
+
+# TypeSafeDocumentClassifier
+
+Answers typed questions about each document with a TypeSafe System One model, such as Jev, and stores the answers in its metadata.
+
+<div className="key-value-table">
+
+|  |  |
+| --- | --- |
+| **Most common position in a pipeline** | Before a [MetadataRouter](../routers/metadatarouter.mdx) |
+| **Mandatory init variables** | `questions`: The questions to answer for every document, keyed by question ID <br /> <br />`api_key`: The TypeSafe API key. Can be set with the `TYPESAFE_API_KEY` env var. |
+| **Mandatory run variables** | `documents`: A list of documents to classify |
+| **Output variables** | `documents`: The classified documents, with the answers in their metadata <br /> <br />`failed_documents`: The documents that couldn't be classified |
+| **API reference** | [TypeSafe](/reference/integrations-typesafe) |
+| **GitHub link** | https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/typesafe |
+| **Package name** | `typesafe-haystack` |
+
+</div>
+
+## Overview
+
+`TypeSafeDocumentClassifier` sends each document to a [TypeSafe](https://typesafe.ai) System One model and stores the answers in the document's metadata. System One models such as Jev don't generate text. They answer every question about a document in one request and return calibrated probabilities.
+
+Each question has one of three types:
+
+- `choice`: picks one label from `criteria`, a dict of label to description (or `None`).
+- `score`: rates the text on the ordered levels in `criteria`, a list of level descriptions.
+- `noul`: returns the probability that the yes/no question in `instructions` is true.
+
+You can write questions as dicts or use the `Choice`, `Score`, and `Noul` objects from the `typesafe-sdk` package. See the [TypeSafe documentation](https://docs.typesafe.ai/primitives) for the full question and answer format.
+
+The answers are stored under `meta["typesafe"]`, keyed by question ID. You can change the field name with `metadata_field`. The component classifies the document's `content` by default. To classify a metadata field instead, set `classification_field`.
+
+Documents without text to classify are returned in `failed_documents` with the reason in `meta["classification_error"]`. So are documents whose request failed, unless you set `raise_on_failure=True`, in which case the first failed request raises its error.
+
+Each document is sent as its own request. `max_workers` sets how many requests run at once, in both `run` and `run_async`.
+
+### Authentication
+
+The component reads the API key from the `TYPESAFE_API_KEY` environment variable by default. You can also pass it at initialization with `api_key`:
+
+```python
+from haystack.utils import Secret
+from haystack_integrations.components.classifiers.typesafe import (
+    TypeSafeDocumentClassifier,
+)
+
+classifier = TypeSafeDocumentClassifier(
+    questions={
+        "refund": {
+            "type": "noul",
+            "instructions": "Is the customer asking for a refund?",
+        }
+    },
+    api_key=Secret.from_token("<your-api-key>"),
+)
+```
+
+### Running models locally with Ollaya
+
+The component works with any server that implements the TypeSafe API, such as [Ollaya](https://ollaya.dev), which runs open decision models on your own machine. Set `api_base_url` to the server and `model` to one of its models. Ollaya accepts any non-empty API key unless it is configured with one.
+
+```shell
+docker run -d --name ollaya -p 11435:11435 -v ollaya:/home/ollaya/.ollaya ghcr.io/ollaya-dev/ollaya
+docker exec ollaya ollaya pull laya:en
+```
+
+```python
+from haystack.utils import Secret
+from haystack_integrations.components.classifiers.typesafe import (
+    TypeSafeDocumentClassifier,
+)
+
+classifier = TypeSafeDocumentClassifier(
+    questions={
+        "refund": {
+            "type": "noul",
+            "instructions": "Is the customer asking for a refund?",
+        }
+    },
+    model="laya:en",
+    api_key=Secret.from_token("local"),
+    api_base_url="http://localhost:11435",
+)
+```
+
+## Usage
+
+Install the `typesafe-haystack` package to use the `TypeSafeDocumentClassifier`:
+
+```shell
+pip install typesafe-haystack
+```
+
+### On its own
+
+```python
+from haystack import Document
+from haystack_integrations.components.classifiers.typesafe import (
+    TypeSafeDocumentClassifier,
+)
+
+classifier = TypeSafeDocumentClassifier(
+    questions={
+        "department": {
+            "type": "choice",
+            "instructions": "Which department should handle this ticket?",
+            "criteria": {"billing": None, "technical": None, "sales": None},
+        },
+        "urgency": {
+            "type": "score",
+            "instructions": "How urgent is this ticket?",
+            "criteria": ["can wait", "this week", "today"],
+        },
+        "refund": {
+            "type": "noul",
+            "instructions": "Is the customer asking for a refund?",
+        },
+    },
+)
+
+result = classifier.run(
+    documents=[Document(content="I was charged twice, please refund me today.")]
+)
+print(result["documents"][0].meta["typesafe"])
+# {'department': {'type': 'choice', 'choice': 'billing', 'confidence': 0.9516,
+#                 'probabilities': {'billing': 0.9677, 'technical': 0.0221, 'sales': 0.0102}},
+#  'urgency': {'type': 'score', 'score': 1.948, 'confidence': 0.9389,
+#              'legend': {'0': 'can wait', '1': 'this week', '2': 'today'},
+#              'probabilities': {'0': 0.0113, '1': 0.0295, '2': 0.9593}},
+#  'refund': {'type': 'noul', 'noul': 0.9498}}
+```
+
+### In a pipeline
+
+The following pipeline classifies support tickets by department and sends each one to a different output of a `MetadataRouter`, which matches on the nested answer field:
+
+```python
+from haystack import Document, Pipeline
+from haystack.components.routers import MetadataRouter
+from haystack_integrations.components.classifiers.typesafe import (
+    TypeSafeDocumentClassifier,
+)
+
+labels = ["billing", "technical"]
+
+pipeline = Pipeline()
+pipeline.add_component(
+    "classifier",
+    TypeSafeDocumentClassifier(
+        questions={
+            "department": {
+                "type": "choice",
+                "instructions": "Which department should handle this ticket?",
+                "criteria": dict.fromkeys(labels),
+            }
+        },
+    ),
+)
+pipeline.add_component(
+    "router",
+    MetadataRouter(
+        rules={
+            label: {
+                "field": "meta.typesafe.department.choice",
+                "operator": "==",
+                "value": label,
+            }
+            for label in labels
+        }
+    ),
+)
+pipeline.connect("classifier.documents", "router.documents")
+
+result = pipeline.run(
+    {
+        "classifier": {
+            "documents": [
+                Document(content="I was charged twice for my subscription."),
+                Document(
+                    content="The app crashes every time I open the settings page."
+                ),
+            ]
+        }
+    }
+)
+print(
+    {
+        output: [document.content for document in documents]
+        for output, documents in result["router"].items()
+    }
+)
+# {'billing': ['I was charged twice for my subscription.'],
+#  'technical': ['The app crashes every time I open the settings page.'],
+#  'unmatched': []}
+```
@@ -3,6 +3,7 @@ title: "DatadogConnector"
 id: datadogconnector
 slug: "/datadogconnector"
 description: "Learn how to work with Datadog in Haystack."
+hep_available: true
 ---

 # DatadogConnector
@@ -3,6 +3,7 @@ title: "GitHubFileEditor"
 id: githubfileeditor
 slug: "/githubfileeditor"
 description: "This is a component for editing files in GitHub repositories through the GitHub API."
+hep_available: true
 ---

 # GitHubFileEditor
@@ -3,6 +3,7 @@ title: "GitHubIssueCommenter"
 id: githubissuecommenter
 slug: "/githubissuecommenter"
 description: "This component posts comments to GitHub issues using the GitHub API."
+hep_available: true
 ---

 # GitHubIssueCommenter
@@ -3,6 +3,7 @@ title: "GitHubIssueViewer"
 id: githubissueviewer
 slug: "/githubissueviewer"
 description: "This component fetches and parses GitHub issues into Haystack documents."
+hep_available: true
 ---

 # GitHubIssueViewer
@@ -3,6 +3,7 @@ title: "GitHubPRCreator"
 id: githubprcreator
 slug: "/githubprcreator"
 description: "This component creates pull requests from a fork back to the original repository through the GitHub API."
+hep_available: true
 ---

 # GitHubPRCreator
@@ -3,6 +3,7 @@ title: "GitHubRepoForker"
 id: githubrepoforker
 slug: "/githubrepoforker"
 description: "This component forks a GitHub repository from an issue URL through the GitHub API."
+hep_available: true
 ---

 # GitHubRepoForker
@@ -3,6 +3,7 @@ title: "GitHubRepoViewer"
 id: githubrepoviewer
 slug: "/githubrepoviewer"
 description: "This component navigates and fetches content from GitHub repositories through the GitHub API."
+hep_available: true
 ---

 # GitHubRepoViewer
@@ -3,6 +3,7 @@ title: "JinaReaderConnector"
 id: jinareaderconnector
 slug: "/jinareaderconnector"
 description: "Use Jina AI’s Reader API with Haystack."
+hep_available: true
 ---

 # JinaReaderConnector
@@ -3,6 +3,7 @@ title: "LangfuseConnector"
 id: langfuseconnector
 slug: "/langfuseconnector"
 description: "Learn how to work with Langfuse in Haystack."
+hep_available: true
 ---

 # LangfuseConnector
@@ -3,6 +3,7 @@ title: "OAuthTokenResolver"
 id: oauthtokenresolver
 slug: "/oauthtokenresolver"
 description: "Resolves an OAuth access token at pipeline runtime and emits it for downstream components such as the SharePoint and Google Drive retrievers and fetchers."
+hep_available: true
 ---

 # OAuthTokenResolver
@@ -3,6 +3,7 @@ title: "OpenAPIConnector"
 id: openapiconnector
 slug: "/openapiconnector"
 description: "`OpenAPIConnector` is a component that acts as an interface between the Haystack ecosystem and OpenAPI services."
+hep_available: true
 ---

 # OpenAPIConnector
@@ -3,6 +3,7 @@ title: "OpenAPIServiceConnector"
 id: openapiserviceconnector
 slug: "/openapiserviceconnector"
 description: "`OpenAPIServiceConnector` is a component that acts as an interface between the Haystack ecosystem and OpenAPI services."
+hep_available: true
 ---

 # OpenAPIServiceConnector
@@ -3,6 +3,7 @@ title: "OpenTelemetryConnector"
 id: opentelemetryconnector
 slug: "/opentelemetryconnector"
 description: "Learn how to work with OpenTelemetry in Haystack."
+hep_available: true
 ---

 # OpenTelemetryConnector
@@ -3,6 +3,7 @@ title: "WeaveConnector"
 id: weaveconnector
 slug: "/weaveconnector"
 description: "Learn how to use Weights & Biases Weave framework for tracing and monitoring your pipeline components."
+hep_available: true
 ---

 # WeaveConnector
@@ -3,6 +3,7 @@ title: "AmazonTextractConverter"
 id: amazontextractconverter
 slug: "/amazontextractconverter"
 description: "`AmazonTextractConverter` converts images and single-page PDFs to documents using AWS Textract. It supports plain text OCR, structured analysis of tables, forms, signatures, and layout, as well as natural-language queries over the document."
+hep_available: true
 ---

 # AmazonTextractConverter
@@ -3,6 +3,7 @@ title: "AzureDocumentIntelligenceConverter"
 id: azuredocumentintelligenceconverter
 slug: "/azuredocumentintelligenceconverter"
 description: "`AzureDocumentIntelligenceConverter` converts files to Documents using Azure's Document Intelligence service with GitHub Flavored Markdown output for better LLM/RAG integration. It supports PDF, JPEG, PNG, BMP, TIFF, DOCX, XLSX, PPTX, and HTML."
+hep_available: true
 ---

 # AzureDocumentIntelligenceConverter
@@ -3,6 +3,7 @@ title: "AzureOCRDocumentConverter"
 id: azureocrdocumentconverter
 slug: "/azureocrdocumentconverter"
 description: "`AzureOCRDocumentConverter` converts files to documents using Azure's Document Intelligence service. It supports the following file formats: PDF (both searchable and image-only), JPEG, PNG, BMP, TIFF, DOCX, XLSX, PPTX, and HTML."
+hep_available: true
 ---

 # AzureOCRDocumentConverter
@@ -3,6 +3,7 @@ title: "CSVToDocument"
 id: csvtodocument
 slug: "/csvtodocument"
 description: "Converts CSV files to documents."
+hep_available: true
 ---

 # CSVToDocument
@@ -3,6 +3,7 @@ title: "DoclingConverter"
 id: doclingconverter
 slug: "/doclingconverter"
 description: "`DoclingConverter` converts PDF, DOCX, HTML, and other document formats to Haystack Documents using Docling, with support for layout-aware chunking, Markdown, and JSON export."
+hep_available: true
 ---

 # DoclingConverter
@@ -3,6 +3,7 @@ title: "DoclingServeConverter"
 id: doclingserveconverter
 slug: "/doclingserveconverter"
 description: "`DoclingServeConverter` converts PDF, DOCX, HTML, and other document formats to Haystack Documents by calling a remote DoclingServe HTTP server, with no local ML dependencies."
+hep_available: true
 ---

 # DoclingServeConverter
@@ -3,6 +3,7 @@ title: "DocumentToImageContent"
 id: documenttoimagecontent
 slug: "/documenttoimagecontent"
 description: "`DocumentToImageContent` extracts visual data from image or PDF file-based documents and converts them into `ImageContent` objects. These are ready for multimodal AI pipelines, including tasks like image question-answering and captioning."
+hep_available: true
 ---

 # DocumentToImageContent
@@ -3,6 +3,7 @@ title: "DOCXToDocument"
 id: docxtodocument
 slug: "/docxtodocument"
 description: "Convert DOCX files to documents."
+hep_available: true
 ---

 # DOCXToDocument
@@ -3,6 +3,7 @@ title: "FileToFileContent"
 id: filetofilecontent
 slug: "/filetofilecontent"
 description: "`FileToFileContent` reads local files and converts them into `FileContent` objects"
+hep_available: true
 ---

 # FileToFileContent
@@ -3,6 +3,7 @@ title: "GotenbergFileConverter"
 id: gotenbergfileconverter
 slug: "/gotenbergfileconverter"
 description: "A component that converts Office, OpenDocument, HTML, and Markdown files, and web pages to PDF using a Gotenberg server."
+hep_available: false
 ---

 # GotenbergFileConverter
@@ -3,6 +3,7 @@ title: "HTMLToDocument"
 id: htmltodocument
 slug: "/htmltodocument"
 description: "A component that converts HTML files to documents."
+hep_available: true
 ---

 # HTMLToDocument
@@ -28,6 +29,16 @@ The `HTMLToDocument` component converts HTML files into documents. It can be use

 When you initialize the component, you can optionally set  `extraction_kwargs`,  a dictionary containing keyword arguments to customize the extraction process. These are passed to the underlying Trafilatura `extract` function. For the full list of available arguments, see the [Trafilatura documentation](https://trafilatura.readthedocs.io/en/latest/corefunctions.html#extract).

+### Character encoding
+
+By default, `HTMLToDocument` lets Trafilatura detect the HTML encoding and extract the text (`encoding=None`). If the result contains incorrect characters, you can install `trafilatura[all]`, which includes an additional encoding detector:
+
+```shell
+pip install "trafilatura[all]"
+```
+
+If you know the encoding, set it on the converter to bypass automatic detection, for example `HTMLToDocument(encoding="latin-1")`.
+
 ## Usage

 ### On its own
@@ -3,6 +3,7 @@ title: "ImageFileToDocument"
 id: imagefiletodocument
 slug: "/imagefiletodocument"
 description: "Converts image file references into empty `Document` objects with associated metadata."
+hep_available: true
 ---

 # ImageFileToDocument
@@ -3,6 +3,7 @@ title: "ImageFileToImageContent"
 id: imagefiletoimagecontent
 slug: "/imagefiletoimagecontent"
 description: "`ImageFileToImageContent` reads local image files and converts them into `ImageContent` objects. These are ready for multimodal AI pipelines, including tasks like image captioning, visual QA, or prompt-based generation."
+hep_available: true
 ---

 # ImageFileToImageContent
@@ -3,6 +3,7 @@ title: "JSONConverter"
 id: jsonconverter
 slug: "/jsonconverter"
 description: "Converts JSON files to text documents."
+hep_available: true
 ---

 # JSONConverter
@@ -3,6 +3,7 @@ title: "KreuzbergConverter"
 id: kreuzbergconverter
 slug: "/kreuzbergconverter"
 description: "`KreuzbergConverter` converts files to Haystack Documents using Kreuzberg, a document intelligence framework with a Rust core that extracts text from 91+ file formats entirely locally with no external API calls."
+hep_available: true
 ---

 # KreuzbergConverter
@@ -3,6 +3,7 @@ title: "LibreOfficeFileConverter"
 id: libreofficefileconverter
 slug: "/libreofficefileconverter"
 description: "A component that converts office files (documents, spreadsheets, presentations) between formats using LibreOffice's command line interface."
+hep_available: false
 ---

 # LibreOfficeFileConverter
@@ -3,6 +3,7 @@ title: "MarkdownToDocument"
 id: markdowntodocument
 slug: "/markdowntodocument"
 description: "A component that converts Markdown files to documents."
+hep_available: true
 ---

 # MarkdownToDocument
@@ -3,6 +3,7 @@ title: "MarkItDownConverter"
 id: markitdownconverter
 slug: "/markitdownconverter"
 description: "A component that converts files (PDF, Word, PowerPoint, Excel, HTML, images, and more) to Documents using Microsoft's MarkItDown library."
+hep_available: true
 ---

 # MarkItDownConverter
@@ -3,6 +3,7 @@ title: "MistralOCRDocumentConverter"
 id: mistralocrdocumentconverter
 slug: "/mistralocrdocumentconverter"
 description: "`MistralOCRDocumentConverter` extracts text from documents using Mistral's OCR API, with optional structured annotations for both individual image regions and full documents. It supports various input formats including local files, URLs, and Mistral file IDs."
+hep_available: true
 ---

 # MistralOCRDocumentConverter
@@ -3,6 +3,7 @@ title: "MSGToDocument"
 id: msgtodocument
 slug: "/msgtodocument"
 description: "Converts Microsoft Outlook .msg files to documents."
+hep_available: true
 ---

 # MSGToDocument
@@ -3,6 +3,7 @@ title: "MultiFileConverter"
 id: multifileconverter
 slug: "/multifileconverter"
 description: "Converts CSV, DOCX, HTML, JSON, MD, PPTX, PDF, TXT, and XSLX files to documents."
+hep_available: true
 ---

 # MultiFileConverter
@@ -3,6 +3,7 @@ title: "OpenAPIServiceToFunctions"
 id: openapiservicetofunctions
 slug: "/openapiservicetofunctions"
 description: "`OpenAPIServiceToFunctions` is a component that transforms OpenAPI service specifications into a format compatible with LLM tool calling."
+hep_available: true
 ---

 # OpenAPIServiceToFunctions
@@ -3,6 +3,7 @@ title: "OpenDataLoaderConverter"
 id: opendataloaderconverter
 slug: "/opendataloaderconverter"
 description: "`OpenDataLoaderConverter` converts PDF files to Haystack Documents using OpenDataLoader PDF, a local PDF parser that extracts layout-aware Markdown, text, HTML, or JSON."
+hep_available: false
 ---

 # OpenDataLoaderConverter
@@ -3,6 +3,7 @@ title: "OutputAdapter"
 id: outputadapter
 slug: "/outputadapter"
 description: "This component helps the output of one component fit smoothly into the input of another. It uses Jinja expressions to define how this adaptation occurs."
+hep_available: true
 ---

 # OutputAdapter
@@ -3,6 +3,7 @@ title: "PaddleOCRVLDocumentConverter"
 id: paddleocrvldocumentconverter
 slug: "/paddleocrvldocumentconverter"
 description: "`PaddleOCRVLDocumentConverter` extracts text from documents using PaddleOCR's large model document parsing API."
+hep_available: false
 ---

 # PaddleOCRVLDocumentConverter
@@ -3,6 +3,7 @@ title: "PDFMinerToDocument"
 id: pdfminertodocument
 slug: "/pdfminertodocument"
 description: "A component that converts complex PDF files to documents using pdfminer arguments."
+hep_available: true
 ---

 # PDFMinerToDocument
@@ -3,6 +3,7 @@ title: "PDFToImageContent"
 id: pdftoimagecontent
 slug: "/pdftoimagecontent"
 description: "`PDFToImageContent` reads local PDF files and converts them into `ImageContent` objects. These are ready for multimodal AI pipelines, including tasks like image captioning, visual QA, or prompt-based generation."
+hep_available: true
 ---

 # PDFToImageContent
@@ -3,6 +3,7 @@ title: "PPTXToDocument"
 id: pptxtodocument
 slug: "/pptxtodocument"
 description: "Convert PPTX files to documents."
+hep_available: true
 ---

 # PPTXToDocument
@@ -3,6 +3,7 @@ title: "PyPDFToDocument"
 id: pypdftodocument
 slug: "/pypdftodocument"
 description: "A component that converts PDF files to Documents."
+hep_available: true
 ---

 # PyPDFToDocument
@@ -3,6 +3,7 @@ title: "TextFileToDocument"
 id: textfiletodocument
 slug: "/textfiletodocument"
 description: "Converts text files to documents."
+hep_available: true
 ---

 # TextFileToDocument
@@ -3,6 +3,7 @@ title: "TikaDocumentConverter"
 id: tikadocumentconverter
 slug: "/tikadocumentconverter"
 description: "An integration for converting files of different types (PDF, DOCX, HTML, and more) to documents."
+hep_available: true
 ---

 # TikaDocumentConverter
@@ -3,6 +3,7 @@ title: "TwelveLabsVideoConverter"
 id: twelvelabsvideoconverter
 slug: "/twelvelabsvideoconverter"
 description: "`TwelveLabsVideoConverter` converts videos to Haystack Documents using the TwelveLabs Pegasus video-language model. Pegasus analyzes each video on the fly — its visuals and its own audio (via ASR) — and returns text, so each video becomes a Document whose content is the analysis."
+hep_available: false
 ---

 # TwelveLabsVideoConverter
@@ -3,6 +3,7 @@ title: "UnstructuredFileConverter"
 id: unstructuredfileconverter
 slug: "/unstructuredfileconverter"
 description: "Use this component to convert text files and directories to a document."
+hep_available: true
 ---

 # UnstructuredFileConverter
@@ -3,6 +3,7 @@ title: "XLSXToDocument"
 id: xlsxtodocument
 slug: "/xlsxtodocument"
 description: "Converts Excel files into documents."
+hep_available: true
 ---

 # XLSXToDocument
@@ -3,6 +3,7 @@ title: "S3Downloader"
 id: s3downloader
 slug: "/s3downloader"
 description: "`S3Downloader` downloads files from AWS S3 buckets to the local filesystem and enriches documents with the local file path."
+hep_available: true
 ---

 # S3Downloader
@@ -3,6 +3,7 @@ title: "AmazonBedrockDocumentEmbedder"
 id: amazonbedrockdocumentembedder
 slug: "/amazonbedrockdocumentembedder"
 description: "This component computes embeddings for documents using models through Amazon Bedrock API."
+hep_available: true
 ---

 # AmazonBedrockDocumentEmbedder
@@ -3,6 +3,7 @@ title: "AmazonBedrockDocumentImageEmbedder"
 id: amazonbedrockdocumentimageembedder
 slug: "/amazonbedrockdocumentimageembedder"
 description: "`AmazonBedrockDocumentImageEmbedder` computes image embeddings for documents using models exposed through the Amazon Bedrock API. It  stores the obtained vectors in the embedding field of each document."
+hep_available: true
 ---

 # AmazonBedrockDocumentImageEmbedder
@@ -3,6 +3,7 @@ title: "AmazonBedrockTextEmbedder"
 id: amazonbedrocktextembedder
 slug: "/amazonbedrocktextembedder"
 description: "This component computes embeddings for text (such as a query) using models through Amazon Bedrock API."
+hep_available: true
 ---

 # AmazonBedrockTextEmbedder
@@ -3,6 +3,7 @@ title: "AzureOpenAIDocumentEmbedder"
 id: azureopenaidocumentembedder
 slug: "/azureopenaidocumentembedder"
 description: "This component computes the embeddings of a list of documents and stores the obtained vectors in the embedding field of each document. It uses Azure cognitive services for text and document embedding with models deployed on Azure."
+hep_available: true
 ---

 # AzureOpenAIDocumentEmbedder
@@ -3,6 +3,7 @@ title: "AzureOpenAITextEmbedder"
 id: azureopenaitextembedder
 slug: "/azureopenaitextembedder"
 description: "When you perform embedding retrieval, you use this component to transform your query into a vector. Then, the embedding Retriever looks for similar or relevant documents."
+hep_available: true
 ---

 # AzureOpenAITextEmbedder
@@ -3,6 +3,7 @@ title: "CohereDocumentEmbedder"
 id: coheredocumentembedder
 slug: "/coheredocumentembedder"
 description: "This component computes the embeddings of a list of documents and stores the obtained vectors in the embedding field of each document. It uses Cohere embedding models."
+hep_available: true
 ---

 # CohereDocumentEmbedder
@@ -3,6 +3,7 @@ title: "CohereDocumentImageEmbedder"
 id: coheredocumentimageembedder
 slug: "/coheredocumentimageembedder"
 description: "`CohereDocumentImageEmbedder` computes the image embeddings of a list of documents and stores the obtained vectors in the embedding field of each document. It uses Cohere embedding models with the ability to embed text and images into the same vector space."
+hep_available: true
 ---

 # CohereDocumentImageEmbedder
@@ -3,6 +3,7 @@ title: "CohereTextEmbedder"
 id: coheretextembedder
 slug: "/coheretextembedder"
 description: "This component transforms a string into a vector that captures its semantics using a Cohere embedding model.  When you perform embedding retrieval, you use this component to transform your query into a vector. Then, the embedding Retriever looks for similar or relevant documents."
+hep_available: true
 ---

 # CohereTextEmbedder
@@ -3,6 +3,7 @@ title: "EdenAIDocumentEmbedder"
 id: edenaidocumentembedder
 slug: "/edenaidocumentembedder"
 description: "This component computes the embeddings of a list of documents using Eden AI's OpenAI-compatible API."
+hep_available: true
 ---

 # EdenAIDocumentEmbedder
@@ -3,6 +3,7 @@ title: "EdenAITextEmbedder"
 id: edenaitextembedder
 slug: "/edenaitextembedder"
 description: "This component transforms a string into a vector using Eden AI's OpenAI-compatible API. Use it for embedding retrieval to transform your query into an embedding."
+hep_available: true
 ---

 # EdenAITextEmbedder
@@ -3,6 +3,7 @@ title: "FastembedDocumentEmbedder"
 id: fastembeddocumentembedder
 slug: "/fastembeddocumentembedder"
 description: "This component computes the embeddings of a list of documents using the models supported by FastEmbed."
+hep_available: true
 ---

 # FastembedDocumentEmbedder
@@ -3,6 +3,7 @@ title: "FastembedSparseDocumentEmbedder"
 id: fastembedsparsedocumentembedder
 slug: "/fastembedsparsedocumentembedder"
 description: "Use this component to enrich a list of documents with their sparse embeddings."
+hep_available: true
 ---

 # FastembedSparseDocumentEmbedder
--- a/Show More
+++ b/Show More