"""Auto-install pinned upstream binaries for optional tools. Today this only manages `ripgrep`. The SDK shells out to `rg` via `PATH`, so installing into `~/.deepagents/bin/` and prepending that directory to `os.environ["PATH"]` is sufficient — no SDK change required. The pinned `RIPGREP_VERSION` and `RIPGREP_ASSETS` table is the single source of truth for what gets downloaded and verified. When bumping the version, refresh both the version and the SHA-256 entries together. """ from __future__ import annotations import logging import os import sys from pathlib import Path from typing import TYPE_CHECKING, Literal from deepagents_code._env_vars import OFFLINE, RIPGREP_INSTALLER, is_env_truthy if TYPE_CHECKING: import tarfile import zipfile logger = logging.getLogger(__name__) RIPGREP_VERSION = "14.1.1" """Pinned upstream ripgrep release. Bump alongside `RIPGREP_ASSETS`.""" _RELEASE_URL_PREFIX = ( "https://github.com/BurntSushi/ripgrep/releases/download/" + RIPGREP_VERSION ) RIPGREP_ASSETS: dict[tuple[str, str], tuple[str, str]] = { ("darwin", "arm64"): ( f"ripgrep-{RIPGREP_VERSION}-aarch64-apple-darwin.tar.gz", "24ad76777745fbff131c8fbc466742b011f925bfa4fffa2ded6def23b5b937be", ), ("darwin", "x86_64"): ( f"ripgrep-{RIPGREP_VERSION}-x86_64-apple-darwin.tar.gz", "fc87e78f7cb3fea12d69072e7ef3b21509754717b746368fd40d88963630e2b3", ), ("linux", "arm64"): ( f"ripgrep-{RIPGREP_VERSION}-aarch64-unknown-linux-gnu.tar.gz", "c827481c4ff4ea10c9dc7a4022c8de5db34a5737cb74484d62eb94a95841ab2f", ), ("linux", "x86_64"): ( f"ripgrep-{RIPGREP_VERSION}-x86_64-unknown-linux-musl.tar.gz", "4cf9f2741e6c465ffdb7c26f38056a59e2a2544b51f7cc128ef28337eeae4d8e", ), # Windows on ARM runs x64 binaries via emulation; upstream does not # ship an arm64-windows build for ripgrep, so both Windows entries # point at the same x86_64 MSVC asset. ("win32", "arm64"): ( f"ripgrep-{RIPGREP_VERSION}-x86_64-pc-windows-msvc.zip", "d0f534024c42afd6cb4d38907c25cd2b249b79bbe6cc1dbee8e3e37c2b6e25a1", ), ("win32", "x86_64"): ( f"ripgrep-{RIPGREP_VERSION}-x86_64-pc-windows-msvc.zip", "d0f534024c42afd6cb4d38907c25cd2b249b79bbe6cc1dbee8e3e37c2b6e25a1", ), } """`(sys.platform, normalized arch) -> (asset filename, sha256 hex)`.""" BIN_DIR: Path = Path.home() / ".deepagents" / "bin" """Directory holding managed binaries. Prepended to `PATH` on startup.""" _DOWNLOAD_TIMEOUT_SECONDS = 120 _VERSION_CHECK_TIMEOUT_SECONDS = 5 _DOWNLOAD_CHUNK_BYTES = 1 << 16 _ARCH_ALIASES = { "aarch64": "arm64", "arm64": "arm64", "amd64": "x86_64", "x86_64": "x86_64", "x64": "x86_64", } class ChecksumMismatchError(Exception): """Raised when a downloaded archive fails SHA-256 verification. Distinct from generic install failure so callers can surface a loud, user-visible notice — a checksum mismatch is a supply-chain anomaly (CDN poisoning, MITM, tampered mirror) and must not be silently treated like "you're offline". """ UnavailableReason = Literal["unsupported", "artifact_not_found"] """Stable reason token for logging and telemetry.""" class ManagedToolUnavailableError(Exception): """Raised when no managed helper binary is available for this system. Distinct from transient network/download failures so callers can tell users whether retrying can help or whether they need a different install path. """ def __init__( self, *, tool: Literal["ripgrep"], reason: UnavailableReason, message: str ) -> None: """Initialize the unavailable-tool error. Args: tool: Managed helper name. reason: Stable reason token for logging and telemetry. message: User-facing remediation message. """ super().__init__(message) self.tool = tool self.reason = reason self.message = message def _normalized_arch() -> str | None: """Return a normalized arch key matching `RIPGREP_ASSETS`. Returns `None` for unsupported architectures (e.g. 32-bit, ppc, s390x). """ import platform raw = platform.machine().lower() return _ARCH_ALIASES.get(raw) def _unsupported_ripgrep_error( platform_name: str, arch: str | None ) -> ManagedToolUnavailableError: """Return a clear unsupported-platform error for managed ripgrep.""" target = platform_name if arch is None else f"{platform_name}/{arch}" return ManagedToolUnavailableError( tool="ripgrep", reason="unsupported", message=( f"Managed ripgrep is not available for this system ({target}). " "Install ripgrep manually, or set DEEPAGENTS_CODE_RIPGREP_INSTALLER=system." ), ) def _artifact_not_found_error( platform_name: str, arch: str ) -> ManagedToolUnavailableError: """Return a clear missing-artifact error for managed ripgrep.""" return ManagedToolUnavailableError( tool="ripgrep", reason="artifact_not_found", message=( f"Managed ripgrep artifact for {platform_name}/{arch} was not found " f"in pinned ripgrep {RIPGREP_VERSION}. Install ripgrep manually, or " "try a newer dcode version." ), ) def managed_rg_path() -> Path: """Return the managed ripgrep binary path (`.exe` on Windows).""" name = "rg.exe" if sys.platform == "win32" else "rg" return BIN_DIR / name def is_offline() -> bool: """Return whether managed-tool downloads are disabled via env var.""" return is_env_truthy(OFFLINE) RipgrepInstaller = Literal["managed", "system"] """The two recognized ripgrep installer modes.""" INSTALLER_MANAGED: RipgrepInstaller = "managed" """Default installer mode: fetch the pinned, checksummed upstream binary.""" INSTALLER_SYSTEM: RipgrepInstaller = "system" """Installer mode that defers ripgrep to the system package manager / `PATH`.""" def ripgrep_installer() -> RipgrepInstaller: """Return the configured ripgrep installer mode. Reads `RIPGREP_INSTALLER` and normalizes it to `INSTALLER_MANAGED` or `INSTALLER_SYSTEM`, falling back to `INSTALLER_MANAGED` for unset or unrecognized values. The `strip().lower()` normalization must stay in sync with the `case` block in `scripts/install.sh` so both layers agree on the parsed mode. """ raw = os.environ.get(RIPGREP_INSTALLER, "").strip().lower() if raw == INSTALLER_SYSTEM: return INSTALLER_SYSTEM if raw and raw != INSTALLER_MANAGED: logger.warning( "Unrecognized %s=%r; expected %r or %r. Defaulting to %r.", RIPGREP_INSTALLER, raw, INSTALLER_MANAGED, INSTALLER_SYSTEM, INSTALLER_MANAGED, ) return INSTALLER_MANAGED def prefers_system_ripgrep() -> bool: """Return whether the user opted into the `system` ripgrep installer. In `system` mode ripgrep is provisioned by the OS package manager or an existing `PATH` entry rather than the managed download. """ return ripgrep_installer() == INSTALLER_SYSTEM def prepend_managed_bin_to_path() -> None: """Idempotently prepend `BIN_DIR` to `os.environ["PATH"]`. Safe to call on every startup. Callers do not need to check whether the directory exists — adding a non-existent directory to `PATH` is harmless and matches behavior of common version managers. """ bin_str = str(BIN_DIR) current = os.environ.get("PATH", "") parts = current.split(os.pathsep) if current else [] if parts and parts[0] == bin_str: return parts = [bin_str, *(p for p in parts if p != bin_str)] os.environ["PATH"] = os.pathsep.join(parts) def _path_without_managed_bin() -> str | None: """Return `PATH` with `BIN_DIR` removed.""" current = os.environ.get("PATH") if not current: return None managed_dir = BIN_DIR.resolve() parts = [ part for part in current.split(os.pathsep) if not part or Path(part).resolve() != managed_dir ] return os.pathsep.join(parts) def _managed_binary_is_current(binary: Path) -> bool: """Return whether the on-disk managed `rg` matches `RIPGREP_VERSION`. Returns `False` on any concrete failure (`OSError`, non-zero exit, empty stdout, version mismatch) so a corrupted or wrong-arch binary written by a previously crashed install gets re-fetched. Only `TimeoutExpired` "falls open" — that case suggests a sandboxed subprocess rather than a broken binary. """ import subprocess # noqa: S404 # fixed-argv probe of a managed binary try: result = subprocess.run( # noqa: S603 # fixed argv, managed path [str(binary), "--version"], check=False, capture_output=True, text=True, timeout=_VERSION_CHECK_TIMEOUT_SECONDS, ) except subprocess.TimeoutExpired: logger.debug("rg --version probe timed out for %s; assuming current", binary) return True except OSError: logger.debug( "rg --version probe failed for %s; treating as stale", binary, exc_info=True, ) return False if result.returncode != 0: logger.debug( "rg --version exited %d for %s; treating as stale", result.returncode, binary, ) return False first_line = (result.stdout or "").splitlines()[:1] if not first_line: return False return RIPGREP_VERSION in first_line[0] def _download_to(url: str, dest: Path) -> None: """Stream `url` to `dest`, bounded by a wall-clock deadline. `urlopen(timeout=...)` only bounds per-operation socket waits, so a slow trickle of bytes from a flaky peer could otherwise stretch the transfer well beyond the configured timeout. The chunked read here enforces an end-to-end deadline, checked between chunk reads. A non-200 response is rejected before any bytes are written: a proxy interstitial or an unfollowed redirect returned with a non-200 status must not be streamed to disk and then surface downstream as a misleading SHA-256 failure (which reads as a supply-chain anomaly). `urlopen` already raises `HTTPError` for 4xx/5xx, so this guards the residual 2xx/3xx cases. Raises: TimeoutError: When total transfer time exceeds the deadline. urllib.error.URLError: When the response status is not 200. """ import time import urllib.error import urllib.request deadline = time.monotonic() + _DOWNLOAD_TIMEOUT_SECONDS with ( urllib.request.urlopen(url, timeout=_DOWNLOAD_TIMEOUT_SECONDS) as resp, # noqa: S310 # fixed https GitHub release URL dest.open("wb") as fh, ): status = getattr(resp, "status", None) if status is not None and status != 200: # noqa: PLR2004 # HTTP 200 OK msg = f"Unexpected HTTP {status} response fetching {url}" raise urllib.error.URLError(msg) while True: if time.monotonic() > deadline: msg = ( f"Download of {url} exceeded {_DOWNLOAD_TIMEOUT_SECONDS}s deadline" ) raise TimeoutError(msg) chunk = resp.read(_DOWNLOAD_CHUNK_BYTES) if not chunk: break fh.write(chunk) def _verify_sha256(path: Path, expected_hex: str) -> None: """Verify `path` matches `expected_hex`. Raises: ChecksumMismatchError: When the SHA-256 of `path` differs from `expected_hex`. """ import hashlib digest = hashlib.sha256() with path.open("rb") as fh: for chunk in iter(lambda: fh.read(1 << 20), b""): digest.update(chunk) actual = digest.hexdigest() if actual != expected_hex: msg = ( f"Checksum mismatch for {path.name}: expected {expected_hex}, got {actual}" ) raise ChecksumMismatchError(msg) def _validate_legacy_tar_member(member: tarfile.TarInfo, extract_root: Path) -> None: """Reject tar members that cannot be safely extracted without filters. Raises: tarfile.TarError: If a member would extract outside `extract_root` or uses a tar entry type this fallback does not support. """ import tarfile target = extract_root / member.name try: target.resolve().relative_to(extract_root.resolve()) except ValueError as exc: msg = f"Refusing to extract unsafe tar member {member.name!r}" raise tarfile.TarError(msg) from exc if not (member.isfile() or member.isdir()): msg = f"Refusing to extract unsupported tar member {member.name!r}" raise tarfile.TarError(msg) def _extract_tar_data(tf: tarfile.TarFile, extract_root: Path) -> None: """Extract a tar archive with `data` filtering when available. Python versions before the PEP 706 backport (3.11.0-3.11.3) lack the `filter` keyword on `extractall`, and this package supports those patch versions. The fallback validates the pinned release archive before using the legacy API. Raises: TypeError: Re-raised when `extractall` rejects a non-`filter` keyword (i.e. an unrelated `TypeError` we should not swallow). """ try: tf.extractall(extract_root, filter="data") except TypeError as exc: if "filter" not in str(exc): raise members = tf.getmembers() for member in members: _validate_legacy_tar_member(member, extract_root) tf.extractall(extract_root, members=members) # noqa: S202 # validated above def _extract_rg(archive: Path, extract_root: Path) -> Path: """Extract `archive` and locate the `rg` binary inside. Handles both `.tar.gz` and `.zip` archives. Release archives nest the binary under `ripgrep--/`, so we walk the tree to find it rather than hard-coding the prefix. Malformed archives or unsafe members propagate `tarfile.TarError` / `zipfile.BadZipFile`. Returns: Absolute path to the extracted `rg` (or `rg.exe`) binary. Raises: FileNotFoundError: When the archive does not contain an `rg` binary. """ import tarfile import zipfile if archive.suffix == ".zip": with zipfile.ZipFile(archive) as zf: _extract_zip_validated(zf, extract_root) else: with tarfile.open(archive, mode="r:*") as tf: _extract_tar_data(tf, extract_root) target_name = "rg.exe" if sys.platform == "win32" else "rg" for path in extract_root.rglob(target_name): if path.is_file(): return path msg = f"Could not find {target_name} inside {archive.name}" raise FileNotFoundError(msg) def _extract_zip_validated(zf: zipfile.ZipFile, extract_root: Path) -> None: """Extract a zip archive after validating each member's path. `ZipFile.extractall` does sanitize absolute paths and parent-relative components on modern Python, but defense-in-depth here keeps the SHA-256-verified archive from being the only line of defense against a zip-slip variant in a future upstream archive. Raises: zipfile.BadZipFile: If a member would extract outside `extract_root`. """ import zipfile extract_root.mkdir(parents=True, exist_ok=True) root = extract_root.resolve() for member in zf.infolist(): target = (extract_root / member.filename).resolve() try: target.relative_to(root) except ValueError as exc: msg = f"Refusing to extract unsafe zip member {member.filename!r}" raise zipfile.BadZipFile(msg) from exc zf.extractall(extract_root) # noqa: S202 # validated above def _install_ripgrep_sync(asset: str, sha256: str) -> Path: """Download, verify, extract, and install ripgrep atomically. Staging happens *inside* `BIN_DIR` so the final rename is on the same filesystem and therefore atomic on POSIX. Windows keeps replacing the user-facing `rg.exe` directly because symlink support varies by developer mode and policy. POSIX installs use a versioned real binary plus a relative `rg` symlink so moving or bind-mounting `~/.deepagents` does not bake in the original home directory path. `_verify_sha256` propagates `ChecksumMismatchError` to abort install before any move. Returns: Absolute path to the installed `rg` entrypoint. """ import os import tempfile BIN_DIR.mkdir(parents=True, exist_ok=True) url = f"{_RELEASE_URL_PREFIX}/{asset}" with tempfile.TemporaryDirectory(prefix=".deepagents-rg-", dir=BIN_DIR) as tmp_str: tmp = Path(tmp_str) archive = tmp / asset _download_to(url, archive) _verify_sha256(archive, sha256) extracted = _extract_rg(archive, tmp / "unpacked") if sys.platform != "win32": extracted.chmod(0o755) dest = managed_rg_path() if sys.platform == "win32": extracted.replace(dest) return dest real = BIN_DIR / f"rg-{RIPGREP_VERSION}" extracted.replace(real) link = tmp / "rg-link" link.symlink_to(os.path.relpath(real, start=BIN_DIR)) link.replace(dest) return dest async def ensure_ripgrep() -> Path | None: """Ensure a usable `rg` binary is available, installing if necessary. Resolution order: 1. If the `system` installer is selected, return a non-managed `rg` found on `PATH`, or `None` when only the managed binary is present. 2. If a managed `rg` exists *and* matches `RIPGREP_VERSION`, return it. 3. Otherwise, if a system `rg` is on `PATH` and no managed binary exists, return its resolved path. This is gated on the *absence* of a managed binary: once a managed `rg` exists, the pinned version always wins, so a stale managed binary is re-fetched rather than deferring to a system `rg` and the resolved version stays deterministic. 4. If offline, return `None` so callers fall back to the existing notification + slow path. 5. If no managed asset matches the platform/arch, return a non-managed `rg` on `PATH` when one exists; otherwise raise `ManagedToolUnavailableError` so callers can explain that retrying will not help. 6. Otherwise download → SHA-256 verify → extract → install → prepend `BIN_DIR` to `PATH` → return the installed path. On a checksum mismatch, raises `ChecksumMismatchError` so callers can surface a loud notice. On a 404, raises `ManagedToolUnavailableError`; other failures log and return `None`. A stale managed binary is never proactively deleted. The atomic replace in `_install_ripgrep_sync` overwrites it on success, and on failure the user is strictly better off keeping the older copy than being left with no `rg` at all. Returns: Path to a usable `rg` binary, or `None` when one could not be located or installed. """ import asyncio import platform import shutil import tarfile import urllib.error import zipfile managed = managed_rg_path() managed_exists = managed.exists() or managed.is_symlink() def non_managed_rg() -> Path | None: system_rg = shutil.which("rg", path=_path_without_managed_bin()) if system_rg is not None: return Path(system_rg) return None if prefers_system_ripgrep(): system_rg_path = non_managed_rg() if system_rg_path is not None: return system_rg_path logger.debug( "Skipping managed ripgrep download: %s=%s", RIPGREP_INSTALLER, INSTALLER_SYSTEM, ) return None if managed_exists and _managed_binary_is_current(managed): return managed if not managed_exists: system_rg = shutil.which("rg") if system_rg is not None: return Path(system_rg) if is_offline(): logger.debug("Skipping ripgrep install: %s is set", OFFLINE) return None if sys.platform == "android": logger.debug("Skipping ripgrep install: unsupported platform 'android'") system_rg_path = non_managed_rg() if system_rg_path is not None: return system_rg_path error = _unsupported_ripgrep_error(sys.platform, None) raise error arch = _normalized_arch() if arch is None: logger.debug( "Skipping ripgrep install: unsupported arch %r", platform.machine() ) system_rg_path = non_managed_rg() if system_rg_path is not None: return system_rg_path error = _unsupported_ripgrep_error(sys.platform, platform.machine()) raise error asset_entry = RIPGREP_ASSETS.get((sys.platform, arch)) if asset_entry is None: logger.debug( "Skipping ripgrep install: no asset for (%s, %s)", sys.platform, arch ) system_rg_path = non_managed_rg() if system_rg_path is not None: return system_rg_path error = _unsupported_ripgrep_error(sys.platform, arch) raise error asset, sha256 = asset_entry if managed_exists: logger.info( "Managed ripgrep at %s is stale; replacing with %s", managed, RIPGREP_VERSION, ) try: # `_install_ripgrep_sync` atomically replaces the destination on # success, so we deliberately leave any stale binary in place # until the verified replacement is ready. A failed download must # not strand the user with no `rg` at all. installed = await asyncio.to_thread(_install_ripgrep_sync, asset, sha256) except urllib.error.HTTPError as exc: if exc.code == 404: # noqa: PLR2004 # HTTP 404 Not Found logger.warning( "Managed ripgrep artifact was not found: %s/%s", sys.platform, arch ) error = _artifact_not_found_error(sys.platform, arch) raise error from exc logger.warning( "Could not download ripgrep from %s", _RELEASE_URL_PREFIX, exc_info=True ) return None except (urllib.error.URLError, TimeoutError): logger.warning( "Could not download ripgrep from %s", _RELEASE_URL_PREFIX, exc_info=True ) return None except (tarfile.TarError, zipfile.BadZipFile, FileNotFoundError) as exc: logger.exception( "ripgrep install failed: archive error (%s)", type(exc).__name__ ) return None except PermissionError: logger.exception( "ripgrep install failed: cannot write to %s — check permissions", BIN_DIR ) return None except OSError as exc: logger.exception( "ripgrep install failed: %s (errno=%s)", type(exc).__name__, exc.errno ) return None else: prepend_managed_bin_to_path() return installed