Merge pull request #174 from Gitlawb/fix/served-git-concurrency-cap

feat(node,git): cap concurrent served git ops with a 503 load-shed (#62)
This commit is contained in:
Kevin Codex authored and GitHub committed 2026-08-10 10:00:06 +08:00
commit 5981257be5
24 files changed
+18665 -630

No files matched your search

+127 -4
View File
@@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb
# ── Database pool & startup resilience ────────────────────────────────────
# Maximum connections in the PostgreSQL pool. A cap, not a floor —
# connections open lazily. Size against the DB server's max_connections,
# remembering admin tooling opens its own pool.
GITLAWB_DB_MAX_CONNECTIONS=20
# remembering admin tooling opens its own pool. Each concurrent write pins one
# connection for its whole duration (the connection-affine advisory lock), so the
# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8
# headroom — keep this comfortably above that (default 48 for pushes 32).
GITLAWB_DB_MAX_CONNECTIONS=48
# Seconds a request waits for a pool connection before failing with 503.
GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5
# Upper bound on each startup connect+migrate attempt, in seconds. Keep it
@@ -109,15 +112,135 @@ GITLAWB_MAX_PACK_BYTES=2147483648
# Max seconds a served git upload-pack / receive-pack (clone / push) may run
# before it is aborted with a 504. Bounds a hung git that would otherwise pin a
# worker and, on push, the repo write lock. Does NOT cover the info/refs
# advertisement or the withheld-blob path, which remain unbounded. Default 600.
# worker and, on push, the repo write lock. Also bounds both info/refs
# advertisements, the withheld-blob pack build, and the push-side candidate
# discovery (rev-list / cat-file), all reaped via process-group teardown (#174).
# On the path-scoped upload-pack path the withheld-blob classification walk and
# the pack serve share ONE deadline, so this value bounds their COMBINED duration
# rather than giving each stage a full budget: a walk that consumes it leaves the
# serve nothing and the clone gets a 504. Serving large path-scoped repos may
# need a higher value here than when each stage was budgeted separately.
# Must be 1..=3153600000 (100 years): the node derives deadlines from this value,
# and a larger one cannot be represented. Default 600.
GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600
# Max seconds the storage-ACQUISITION phase of a served git op may run before the
# request is shed with a 503, separate from the git-run timeout above. A
# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS
# only starts once git spawns, so without this a stalled backend (a hung Tigris
# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains
# the pool until every later request 503s. On expiry the permit is released
# (fail-closed). Kept separate because acquisition and git execution are distinct
# cost centers. Must be positive; set very large to effectively disable. Default 30.
GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30
# Max concurrent git READ ops (upload-pack + the upload-pack info/refs
# advertisement) served at once, a global pool separate from the push pool below.
# The anon receive-pack info/refs advertisement has its OWN pool (see below), not
# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from
# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one
# caller cannot monopolize the pool. Default 128.
GITLAWB_MAX_CONCURRENT_GIT_OPS=128
# Max concurrent git-receive-pack (push) POST operations, in a pool separate from
# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an
# authenticated push at admission. The anon receive-pack info/refs advertisement
# runs in a SEPARATE pool of the same size (disjoint from this one), so an
# advertisement flood cannot shed a push either. Over-cap sheds a 503 +
# Retry-After. Default 32.
GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32
# Max pushes PARKED at once waiting for the per-repo write lease of the SAME repo.
# A parked push holds its already-buffered pack body in memory, so this bounds that
# memory for a hot repo. Past the cap the newest push sheds a clean 503 +
# Retry-After ("repo is busy with another push") instead of joining the queue; a
# push to a DIFFERENT repo is never affected. The lease holder is not counted.
# Raising GITLAWB_MAX_CONCURRENT_GIT_PUSHES does not raise this; set it explicitly.
# Default 8.
GITLAWB_REPO_LEASE_MAX_WAITERS=8
# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all
# repos. The per-repo encrypt-task set caps the pin-task COUNT; this caps how many
# of those loops RUN CONCURRENTLY, and so how much MB-scale object-id list memory is
# held WHILE BEING PINNED. It does not bound retained memory in general: on the local
# IPFS path the caller builds its list before this pool is entered, so a loop parked
# waiting for a permit still holds its full list, and how many park is capped only
# per repo. Do not size memory from this knob alone. A loop DEFERS (waits) when the
# pool is full, never drops a pin. Default 8.
GITLAWB_MAX_CONCURRENT_PIN_TASKS=8
# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement)
# a single caller may hold, so one caller cannot monopolize the read pool. Keyed
# on the resolved SOURCE IP, never the DID: a signature does not move a caller off
# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below:
# left unset, a node behind an edge/NAT keys all callers on the edge IP and this
# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a
# high-fanout caller (CI behind one NAT) then needs the operator to raise this.
# Default 16.
GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16
# Two further per-source concurrency caps exist on the PUSH side but have NO
# environment variable of their own: both are DERIVED as
# GITLAWB_MAX_CONCURRENT_GIT_PUSHES / 8, with a floor of 1. They are not settable
# independently, so raising the push pool raises both proportionally, and the
# default push pool of 32 gives each a cap of 4.
# * The anon receive-pack info/refs advertisement cap: one source IP may hold at
# most this many slots in the dedicated advert pool, so saturating that pool
# takes ~8 distinct source IPs.
# * The authenticated receive-pack POST cap: one source IP may hold at most this
# many WRITE-pool slots. This one is load-bearing for push availability, since
# it is acquired before the global write permit: without it, one host minting
# disposable did:key identities could open enough slow pushes to monopolize the
# write pool and 503 every other source (owner enforcement defaults off, and
# the push rate limiter caps arrival rate, not in-flight concurrency).
# Keyed on the resolved source IP, never the DID, so a DID farm does not defeat
# them; keying granularity follows GITLAWB_TRUSTED_PROXY like the read cap above.
# ── Push rate limiting (git-receive-pack flood brake) ─────────────────────
# Max receive-pack requests (info/refs advertisement + push POST) per client
# IP per hour. 0 disables. Default 600.
GITLAWB_PUSH_RATE_LIMIT=600
# ── /ipfs/{cid} visibility-walk admission (#174) ──────────────────────────
# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to
# decide whether the caller may read a path-scoped blob. It is publicly reachable,
# so it is bounded to keep a permissionless caller from fanning out unbounded
# concurrent walks and exhausting blocking-pool threads + PIDs.
# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from
# the served-git pools). Over-cap sheds a 503. Default 32.
GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32
# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git
# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map).
# Default 4.
GITLAWB_IPFS_WALK_PER_SOURCE=4
# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a
# blob in a path-scoped repo costs a full-history walk). Over-cap repos are
# skipped without a verdict and the scan continues; if the object is then found
# nowhere the request sheds a retryable 503 instead of a false 404. Default 64.
GITLAWB_IPFS_MAX_REPOS_WALKED=64
# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each
# visit costs a repo acquire — on a Tigris cache miss a full archive download,
# so this is also the worst-case object-store fetch count per request — plus a
# cat-file probe. On exhaustion the scan stops and sheds a retryable 503.
# Default 1024.
GITLAWB_IPFS_MAX_REPO_VISITS=1024
# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk
# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire
# and walk stages to the remaining budget, and no stage starts once it is
# exhausted (the scan then sheds a retryable 503). The object-type probe and
# content-read cat-file subprocesses are budget-checked before starting AND each
# run under their own deadline (the lesser of GITLAWB_GIT_SERVICE_TIMEOUT_SECS and
# the remaining budget), reaped via process-group teardown, so a hung cat-file
# cannot hold the request's walk slot past it. One hang path is still unbounded:
# the probe's object-store readability check is a plain filesystem sweep with
# nothing to reap, so a wedged filesystem can hold the slot past the deadline.
# Must be 1..=3153600000 (100 years): the node derives an Instant deadline from
# this value, and a larger one cannot be represented. Default 600.
GITLAWB_IPFS_REQUEST_BUDGET_SECS=600
# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct
# from the concurrency caps above). 0 disables. Default 600.
GITLAWB_IPFS_RATE_LIMIT=600
# ── Creation rate limiting (repo/agent/issue/PR flood brake) ──────────────
# Max creation requests (POST /api/v1/repos, /api/register, fork, issues,
# pulls) per client IP per hour, in addition to the per-DID limit. The per-DID
+25 -13
View File
@@ -4,7 +4,7 @@
Gitlawb Node is the open-source node software behind the Gitlawb network. It lets anyone run a self-hosted node, publish repositories under a DID, sign writes with Ed25519 HTTP signatures, replicate git activity across peers, and move toward a resilient app-delivery network where code and build assets can be served closer to users.
Gitlawb is not trying to be only “another git host.” The long-term direction is:
Gitlawb is not trying to be only "another git host." The long-term direction is:
```txt
Decentralized GitHub
@@ -34,12 +34,12 @@ This is a Rust workspace with four crates:
Most git hosting today depends on a small number of centralized platforms. Gitlawb Nodes are designed for a different model:
- **Own your identity** — every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`.
- **Signed writes by default** — write requests use RFC 9421 HTTP Signatures instead of passwords.
- **Git-native transport** — repositories are still real git repositories served over smart HTTP.
- **Agent-native workflows** — the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents.
- **Peer-aware delivery** — nodes can announce, discover, gossip, and sync with each other.
- **App CDN direction** — the network can evolve from decentralized code storage into code + asset + app delivery.
- **Own your identity**: every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`.
- **Signed writes by default**: write requests use RFC 9421 HTTP Signatures instead of passwords.
- **Git-native transport**: repositories are still real git repositories served over smart HTTP.
- **Agent-native workflows**: the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents.
- **Peer-aware delivery**: nodes can announce, discover, gossip, and sync with each other.
- **App CDN direction**: the network can evolve from decentralized code storage into code + asset + app delivery.
---
@@ -189,14 +189,14 @@ For public-network use, make sure `GITLAWB_NODE` points to the node you want. Th
Public nodes (e.g. `node.gitlawb.com`) require two things on writes:
1. **RFC 9421 HTTP Signatures** — every write is signed by your identity key. `gl`
1. **RFC 9421 HTTP Signatures**: every write is signed by your identity key. `gl`
and the `git-remote-gitlawb` helper do this automatically. An old/unsigned CLI
fails with `401 not_an_agent`; `gl` will tell you to upgrade and register.
2. **An iCaptcha proof** on the spam-gated writes (**repo create, fork, register**).
`gl` solves this for you: on the node's `403 icaptcha_proof_required` it reads the
`x-icaptcha-url` / `x-icaptcha-level` hints, requests a challenge, solves it
locally (arithmetic / algebra / sequence), and **retries the same signed request**
with the `x-icaptcha-proof` header — no manual steps, no env vars.
with the `x-icaptcha-proof` header. No manual steps, no env vars.
```bash
gl identity new # create did:key identity
@@ -215,14 +215,14 @@ Notes:
- **Proofs are short-lived (~5 min TTL) and single-use.** If one expires between
solving and use, the client transparently solves a fresh one and retries.
- **What needs what:** create / fork / register are signed **and** iCaptcha-gated;
`git push` is **signed-only** (owner signature is the gate — no per-push challenge);
`git push` is **signed-only** (owner signature is the gate, no per-push challenge);
reads (clone / fetch / `repo info`) need no proof. A non-existent repo returns a
clear `404`, never a placeholder.
- **API-key iCaptcha deployments:** set `GITLAWB_ICAPTCHA_URL` to your iCaptcha
origin and `GITLAWB_ICAPTCHA_API_KEY` to its key. The client only talks to an
`https` origin whose host is allowlisted (that URL or the public default), and
sends the bearer token **only** to your configured origin — never to a URL a
node advertises — so a hostile node can't capture the key or redirect the solve.
sends the bearer token **only** to your configured origin, never to a URL a
node advertises, so a hostile node can't capture the key or redirect the solve.
---
@@ -343,7 +343,19 @@ Important node settings:
| `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. |
| `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. |
| `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. |
| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. |
| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. |
| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. |
| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. |
| `GITLAWB_MAX_CONCURRENT_GIT_PUSHES` | Max concurrent `git-receive-pack` POST operations, in a pool separate from the read pool. The anon receive-pack `info/refs` advertisement runs in a third pool of the same size, disjoint from both, so an advertisement flood cannot shed a push either. Two per-source push caps are derived from this value (`/8`, floor 1) and have no env var of their own. Over-cap sheds a 503 + Retry-After. Default 32. |
| `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER` | Max concurrent read ops a single caller may hold, so one caller cannot monopolize the read pool. Keyed on the resolved source IP, never the DID, and only as granular as `GITLAWB_TRUSTED_PROXY`: left unset, a node behind an edge or NAT keys every caller on the edge IP and this collapses to one global cap. Default 16. |
| `GITLAWB_MAX_CONCURRENT_PIN_TASKS` | Max post-push pin loops (IPFS + Pinata) running concurrently across all repos. This caps how many loops RUN at once, not how much object-id list memory the node retains: on the local IPFS path a loop parked waiting for a permit still holds its full list. Do not size memory from this knob alone. A loop over cap waits, never drops a pin. Default 8. |
| `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. |
| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. |
| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. |
| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. |
| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. |
| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. |
| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. |
| `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. |
| `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. |
| `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. |
+3
View File
@@ -77,3 +77,6 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] }
[dev-dependencies]
mockito = "1"
tempfile = "3"
# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert
# on which timer fired instead of on wall clock.
tokio = { workspace = true, features = ["test-util"] }
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+16
View File
@@ -520,6 +520,22 @@ mod tests {
sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)),
peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
shutdown_tx: tokio::sync::watch::channel(false).0,
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
encrypt_inflight: crate::state::EncryptInflight::new(),
repo_write_leases: crate::state::RepoWriteLeases::new(8),
git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
git_push_advert_per_caller:
crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_ipfs_walk_per_caller:
crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
git_bin: "git".to_string(),
}
}
+709 -8
View File
@@ -1,6 +1,33 @@
use clap::Parser;
use std::path::PathBuf;
/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds
/// (100 years).
///
/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs`
/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply
/// below, but it shares this ceiling because the defect class and the "set it very large
/// to disable" contract are the same.
///
/// The knob is not just stored, it is arithmetic input: the write path derives the
/// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into
/// `build_filtered_pack` and `blob_paths`, which each build a deadline as
/// `Instant::now() + Duration::from_secs(this)`. That addition panics on overflow in
/// RELEASE as well as debug, so an unbounded `u64` here turns an operator typo into a
/// serve-path crash rather than a very long timeout. Bounding at parse time keeps every
/// derived duration in range at once, instead of hardening each site as it is found.
///
/// The ceiling is representability, NOT a policy view of a sane timeout, and that
/// distinction is what sets the number. The help text has always told operators to set
/// this very large to disable the bound, so values like `999999999` (~31 years) are
/// working production "off" settings; a tighter, tidier cap would fail those nodes at
/// boot on upgrade over a value that was never the defect. 100 years clears every such
/// setting while staying a factor of about 5.85 under the ~584-year ceiling of a
/// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we
/// build for. Every value that worked before still parses; only the ones that would have
/// panicked are rejected.
pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 100 * 365 * 24 * 60 * 60;
#[derive(Parser, Debug, Clone)]
#[command(name = "gitlawb-node", about = "gitlawb node daemon", version)]
pub struct Config {
@@ -170,26 +197,54 @@ pub struct Config {
/// Maximum wall-clock time a single served git operation (upload-pack /
/// receive-pack through `run_git_service`) may run before it is aborted and
/// its process group torn down, in seconds. Bounds a git that neither
/// finishes nor disconnects. Must be positive; set it very large to
/// effectively disable the bound. Default: 600s (10 min), generous for large
/// clones. Does not cover the ref advertisement (`info/refs`) or the
/// withheld-blob fetch path (`upload_pack_excluding`, a blocking
/// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded.
/// finishes nor disconnects. Must be positive and at most
/// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (100 years, the largest value every derived
/// deadline can represent); setting it very large is still the way to disable the
/// bound. Default: 600s (10 min), generous for large clones. Also bounds the ref
/// advertisement
/// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s
/// pack-objects stage), which now share the same timeout + process-group
/// teardown (#174).
#[arg(
long,
env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS",
default_value_t = 600,
value_parser = clap::value_parser!(u64).range(1..)
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
)]
pub git_service_timeout_secs: u64,
/// Maximum wall-clock time the storage-acquisition phase of a served git
/// operation may run before the request is shed with a 503, in seconds. This
/// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris
/// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a
/// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool)
/// on a write acquire. A concurrency permit is taken BEFORE this phase, and
/// `git_service_timeout_secs` only starts once git spawns, so without this the
/// acquire phase is unbounded: a stalled backend pins the permit and drains the
/// pool until every later request 503s. On expiry the permit is released and a
/// bounded 503 + Retry-After is returned (fail-closed). Kept separate from
/// `git_service_timeout_secs` because acquisition and git execution are distinct
/// cost centers — one shared budget would let a slow acquire starve git. Must be
/// positive; set it very large to effectively disable the bound. Default: 30s.
#[arg(
long,
env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS",
default_value_t = 30,
value_parser = clap::value_parser!(u64).range(1..)
)]
pub git_acquire_timeout_secs: u64,
/// Maximum connections in the PostgreSQL pool. This is a cap, not a floor
/// (connections open lazily). Size against the database server's
/// max_connections, remembering admin tooling opens its own pool.
/// max_connections, remembering admin tooling opens its own pool. Each
/// concurrent write pins one pooled connection for its whole duration (the
/// advisory lock in `repo_store::acquire_write` is connection-affine), so this
/// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow
/// pushes starve every other DB path — enforced by `Config::validate`.
#[arg(
long,
env = "GITLAWB_DB_MAX_CONNECTIONS",
default_value_t = 20,
default_value_t = 48,
value_parser = clap::value_parser!(u32).range(1..)
)]
pub db_max_connections: u32,
@@ -234,6 +289,256 @@ pub struct Config {
value_parser = clap::value_parser!(u64).range(1..)
)]
pub db_retry_max_secs: u64,
/// Maximum number of served git operations (upload-pack / receive-pack /
/// info-refs) allowed to run concurrently. Beyond this the node sheds the
/// request with a clean 503 + Retry-After instead of spawning another git
/// subprocess and risking PID/thread exhaustion. Portable backstop: the
/// compose `pids_limit` is not present on Fly, whose connection-concurrency
/// cap is a different axis (500 connections each fan out to git +
/// pack-objects + threads). Size below the process budget with headroom.
///
/// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK
/// `info/refs` advertisement only. The authenticated push POST draws from a
/// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can
/// never reach, and each read caller is additionally bounded by
/// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual
/// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs`
/// advertisement draws from its OWN dedicated pool (sized like the write pool but
/// disjoint), so an advertisement flood can never occupy a permit the
/// authenticated push POST needs at admission (#174).
///
/// A permit is held for the whole op. Every git subprocess that STREAMS is
/// duration-bounded and reaps its process group on disconnect: upload-pack,
/// receive-pack, and both info/refs advertisements run under
/// `git_service_timeout_secs` with `process_group(0)` teardown, and the
/// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side
/// candidate-discovery children (`rev-list` / `cat-file`) now run under the same
/// bounded runner with process-group teardown, so a stuck git child no longer
/// holds its slot indefinitely (#174 closed the duration/cancellation gaps this
/// comment previously tracked).
///
/// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value
/// well under tokio's `Semaphore` permit limit so an oversized value is a
/// clean CLI error rather than a boot-time panic.
#[arg(
long,
env = "GITLAWB_MAX_CONCURRENT_GIT_OPS",
default_value_t = 128,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub max_concurrent_git_ops: usize,
/// Maximum number of concurrent `git-receive-pack` (push) operations. The
/// authenticated push POST draws from this dedicated pool, separate from
/// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an
/// authenticated push at admission (#174). The anon-reachable receive-pack
/// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived
/// from this knob), disjoint from this one, so an advertisement flood cannot
/// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 +
/// Retry-After.
///
/// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value
/// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI
/// error rather than a boot-time panic).
#[arg(
long,
env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES",
default_value_t = 32,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub max_concurrent_git_pushes: usize,
/// Maximum number of pushes that may be PARKED on one repo's in-process write lease
/// at once. Same-repo pushes are serialized (block-and-wait), and a parked push has
/// already had its entire pack buffered by axum, so an unbounded queue on a contended
/// repo is unbounded buffered memory held for up to `git_service_timeout_secs * 2 +
/// 60` (1260s at defaults). Past this cap the newest push sheds a clean 503 +
/// Retry-After instead of joining the queue.
///
/// The trade: raising it lets more same-repo pushes wait their turn (fewer 503s for a
/// hot repo, more memory pinned by waiters); lowering it sheds sooner. Only pushes to
/// the SAME repo count, and only ones parked right now, so the cap can never deny a
/// push to a different repo. The holder is deliberately not counted: a holder whose
/// cleanup never ran would otherwise pin a slot forever and wedge the repo, which is
/// the failure the `steal_after` reclaim exists to survive.
///
/// Default: 8, a quarter of the default `max_concurrent_git_pushes` (32). Raising the
/// push pool does not raise this; set it explicitly. Must be between 1 and 1_048_576.
#[arg(
long,
env = "GITLAWB_REPO_LEASE_MAX_WAITERS",
default_value_t = 8,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub repo_lease_max_waiters: usize,
/// Max concurrent post-push pin loops (`ipfs_pin` and `pinata`
/// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding
/// pin-task COUNT to one per repo, but each pin loop holds a full per-push
/// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it,
/// so N distinct repos could hold N such lists at once. This caps how many run
/// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits) and never drops,
/// since a dropped pin would lose the object's replication copy.
///
/// It does not cap the memory itself: the local IPFS path builds its list before
/// taking a permit, so tasks parked on this pool still hold theirs, and how many
/// park is capped only per repo. Lowering this knob bounds concurrent pinning, not
/// how much an actor pushing to many repos can retain.
///
/// Default: 8. Must be between 1 and 1_048_576.
#[arg(
long,
env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS",
default_value_t = 8,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub max_concurrent_pin_tasks: usize,
/// Maximum concurrent read operations (`upload-pack` and the upload-pack
/// `info/refs` advertisement) a single caller may hold at once, so one caller
/// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are
/// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a
/// caller off this cap, so an authenticated client cannot mint DIDs to escape it.
/// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`.
/// Left unset (the default), a node behind an edge/NAT keys all callers on the
/// edge IP, so this cap collapses to a single global cap rather than per-client.
/// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a
/// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a
/// caller sheds a clean 503 + Retry-After.
///
/// Default: 16. Must be between 1 and 1_048_576.
#[arg(
long,
env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER",
default_value_t = 16,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub max_concurrent_reads_per_caller: usize,
/// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their
/// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs
/// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history
/// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It
/// draws from THIS pool, not any served-git pool: a distinct public cost center
/// on a distinct surface, so sharing a git pool would let anonymous /ipfs
/// traffic shed authenticated git ops (the auth-boundary trap). A permit is
/// held for the whole request (across the repo loop) so it reflects real
/// blocking-thread occupancy, not merely the tokio wait. Beyond this the request
/// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling
/// keeps the value under tokio's `Semaphore` permit limit so an oversized value
/// is a clean CLI error rather than a boot-time panic. Default: 32.
#[arg(
long,
env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS",
default_value_t = 32,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub max_concurrent_ipfs_walks: usize,
/// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at
/// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174).
/// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`),
/// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no
/// admission step, so keying on the DID would let one host mint disposable DIDs to
/// multiply its budget. A request with no resolvable key (no trusted header, no
/// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a
/// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4.
#[arg(
long,
env = "GITLAWB_IPFS_WALK_PER_SOURCE",
default_value_t = 4,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub ipfs_walk_per_source: usize,
/// Upper bound on the number of EXPENSIVE visibility walks
/// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a
/// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a
/// path-scoped repo costs a walk, so the cap counts exactly those candidates
/// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead
/// (counting them here would starve a plain public copy past the cap out of
/// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict
/// and the scan continues; if the request then finds the object nowhere it
/// sheds a retryable 503 + Retry-After rather than misreport existing content
/// absent with a 404. The handler still short-circuits the moment it serves.
/// Must be between 1 and 1_048_576. Default: 64.
#[arg(
long,
env = "GITLAWB_IPFS_MAX_REPOS_WALKED",
default_value_t = 64,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub ipfs_max_repos_walked: usize,
/// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT —
/// pass the repo-level visibility gate into the acquire + `cat-file` probe.
/// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a
/// full repo-archive download from object storage, so the worst-case
/// object-store fetch count for one request equals this ceiling) plus a git
/// probe subprocess. On exhaustion the scan STOPS — unlike
/// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is
/// no cheaper way to keep scanning — and the request sheds a retryable 503 +
/// Retry-After rather than a false 404. Must be between 1 and 1_048_576.
/// Default: 1024.
#[arg(
long,
env = "GITLAWB_IPFS_MAX_REPO_VISITS",
default_value_t = 1024,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
)]
pub ipfs_max_repo_visits: usize,
/// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's
/// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how
/// MANY requests hold walk slots; this bounds how LONG one admitted request
/// may keep its slot. Without it, each repo iteration draws a fresh
/// `git_acquire_timeout_secs` and each expensive walk a fresh
/// `git_service_timeout_secs`, so one request scanning many repos could hold
/// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe,
/// visibility walk, content read) starts only while budget remains, and the
/// acquire wait and walk deadline are clamped to `min(their own timeout,
/// remaining budget)`; a stage is never started with zero remaining. On
/// exhaustion the scan stops without a verdict and the request sheds a
/// retryable 503 + Retry-After rather than a false 404. The clamps bound
/// only the acquire and walk stages (overshoot there is the walk watchdog's
/// SIGTERM grace + SIGKILL settle); the `object_type` /
/// `read_object_content` probe subprocesses are budget-checked before they
/// start AND each run under their own deadline (the lesser of
/// `git_service_timeout_secs` and the remaining budget), reaped by
/// process-group teardown, so a hung `cat-file` cannot hold the request's walk
/// slot past it. Still unbounded: the probe's `object_store_readable` check is a
/// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can
/// hold the slot past the deadline.
/// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is
/// representability, NOT a policy view of a sane budget: `get_by_cid` derives the
/// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition
/// is an explicit overflow check rather than a debug-only one, so a value near the top
/// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of
/// setting a very long budget. The ceiling sits well below where that starts, about a
/// factor of 5.85 (see the constant's own note), so it is a conservative margin rather than the
/// exact overflow point; rejecting at parse time keeps the unrepresentable values out
/// of every reachable configuration. Setting it very large is still the way to
/// effectively disable the budget, and the documented sentinels (`999999999`,
/// `1000000000`) are well inside the range.
/// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length
/// walk still fits.
#[arg(
long,
env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS",
default_value_t = 600,
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
)]
pub ipfs_request_budget_secs: u64,
/// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The
/// route is publicly reachable (`optional_signature`) and each request can drive
/// a full-history git walk, so it carries a per-IP flood brake in addition to the
/// concurrency cap above (a rate limit bounds request *rate*, the semaphore
/// bounds concurrent slow holds — different axes). Keyed on the resolved client
/// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600.
#[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)]
pub ipfs_rate_limit: usize,
}
impl Config {
@@ -250,6 +555,36 @@ impl Config {
}
PathBuf::from(&self.key_path)
}
/// DB connections reserved for everything other than held write-locks: auth
/// lookups, visibility-rule reads, the post-receive tail's own DB writes, and
/// admin tooling. A write pins one pooled connection for its whole duration, so
/// the pool must clear the concurrent-write cap by at least this margin.
pub const DB_POOL_APP_HEADROOM: u32 = 8;
/// Cross-field boot validation. Single-field ranges are enforced by clap; this
/// catches combinations that ship a denial-of-service under otherwise-valid
/// values. Call once at startup and fail fast on `Err`.
pub fn validate(&self) -> Result<(), String> {
// A write pins one pooled connection for its whole duration (the
// connection-affine advisory lock in repo_store::acquire_write), and
// concurrent writes are capped at max_concurrent_git_pushes. If the pool
// does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes
// drains every connection and every other DB path 503s. (#174 F1)
let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64);
if (self.db_max_connections as u64) < floor {
return Err(format!(
"GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \
+ {} headroom = {}: each concurrent write pins one pooled connection for its whole \
duration, so a smaller pool lets a burst of slow pushes starve every other DB path.",
self.db_max_connections,
self.max_concurrent_git_pushes,
Self::DB_POOL_APP_HEADROOM,
floor
));
}
Ok(())
}
}
#[cfg(test)]
@@ -272,4 +607,370 @@ mod tests {
Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err()
);
}
/// #174 (RED-before/GREEN-after): the upper bound is what keeps every duration
/// derived from this knob in range — the lease steal bound's `* 2 + 60` on the write
/// path, and the `Instant::now() + Duration::from_secs(..)` deadlines in
/// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in
/// release builds too. Checked at parse time so no reachable configuration can carry a
/// value those sites cannot represent.
///
/// The pre-existing "set it very large to disable the bound" settings are asserted
/// alongside the rejections on purpose. The bound exists to exclude unrepresentable
/// values, not to impose a view of a reasonable timeout, so a node that has been
/// running on ~31 years must not start failing at boot on upgrade.
#[test]
fn git_service_timeout_rejects_values_no_derived_duration_can_represent() {
let parse = |secs: u64| {
Config::try_parse_from([
"gitlawb-node",
"--git-service-timeout-secs",
&secs.to_string(),
])
};
// Large "disable the bound" values that predate the ceiling still parse.
for disable in [1_000_000_000, 999_999_999] {
assert_eq!(
parse(disable)
.unwrap_or_else(|e| panic!("{disable} was a working setting: {e}"))
.git_service_timeout_secs,
disable
);
}
// At the ceiling: accepted, and every derived duration still fits.
let at_max =
parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse");
assert_eq!(
at_max.git_service_timeout_secs,
GIT_SERVICE_TIMEOUT_SECS_MAX
);
assert!(at_max
.git_service_timeout_secs
.checked_mul(2)
.and_then(|v| v.checked_add(60))
.is_some());
assert!(std::time::Instant::now()
.checked_add(std::time::Duration::from_secs(
at_max.git_service_timeout_secs
))
.is_some());
// Past the ceiling, and the top of the u64 range clap used to accept — the value
// that panics `Instant::now() + Duration::from_secs(..)` outright.
for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] {
assert!(
parse(over).is_err(),
"{over} is past the representable ceiling and must be rejected at parse time"
);
}
assert!(std::time::Instant::now()
.checked_add(std::time::Duration::from_secs(u64::MAX))
.is_none());
}
#[test]
fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks,
8
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"])
.max_concurrent_pin_tasks,
2
);
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err()
);
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"])
.is_err()
);
}
#[test]
fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops,
128
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"])
.max_concurrent_git_ops,
8
);
// 0 permits would shed every served-git request with a 503; clap must reject it.
assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err());
// Above the ceiling would panic tokio's Semaphore::new at boot (permits >
// usize::MAX >> 3); clap must reject it as a clean CLI error instead.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"])
.is_err()
);
// The ceiling itself is accepted.
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"])
.max_concurrent_git_ops,
1_048_576
);
}
#[test]
fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes,
32
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"])
.max_concurrent_git_pushes,
8
);
// 0 permits would shed every push with a 503; clap must reject it.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err()
);
// Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"])
.is_err()
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"])
.max_concurrent_git_pushes,
1_048_576
);
}
#[test]
fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks,
32
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"])
.max_concurrent_ipfs_walks,
4
);
// 0 permits would shed every /ipfs walk with a 503; clap must reject it.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err()
);
// Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"])
.is_err()
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"])
.max_concurrent_ipfs_walks,
1_048_576
);
}
#[test]
fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() {
assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4);
assert_eq!(
Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"])
.ipfs_walk_per_source,
2
);
// 0 would shed every /ipfs walk from a keyed source; clap must reject it.
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err());
assert!(
Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err()
);
}
#[test]
fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked,
64
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"])
.ipfs_max_repos_walked,
8
);
// 0 would walk no repos (serve nothing); clap must reject it.
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err());
assert!(
Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err()
);
}
#[test]
fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits,
1024
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"])
.ipfs_max_repo_visits,
8
);
// 0 would visit no repos (serve nothing); clap must reject it.
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err());
assert!(
Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err()
);
}
#[test]
fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs,
600
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"])
.ipfs_request_budget_secs,
30
);
// 0 would expire every /ipfs request at its first stage (unconditional
// 503); clap must reject it.
assert!(
Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err()
);
}
/// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived
/// from this knob in range. `get_by_cid` builds the request budget as
/// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an
/// explicit overflow check rather than a debug-only one, so an oversized value aborts
/// every `/ipfs/{cid}` request in a release build instead of setting a very long budget.
/// The route is anon-reachable, so the failure is operator-triggered but publicly felt.
/// Checked at parse time so no reachable configuration can carry a value the deadline
/// cannot represent.
///
/// The large "disable the bound" settings are asserted alongside the rejections on
/// purpose, the same way the `git_service_timeout_secs` sibling does it. The ceiling
/// exists to exclude unrepresentable values, not to impose a view of a reasonable
/// budget, so a node already running on such a value must not start failing at boot on
/// upgrade. A test that only checked the boundary would pass with a far tighter cap.
#[test]
fn ipfs_request_budget_rejects_values_no_derived_duration_can_represent() {
let parse = |secs: u64| {
Config::try_parse_from([
"gitlawb-node",
"--ipfs-request-budget-secs",
&secs.to_string(),
])
};
// Large "disable the bound" values that predate the ceiling still parse.
for disable in [1_000_000_000, 999_999_999] {
assert_eq!(
parse(disable)
.unwrap_or_else(|e| panic!("{disable} was a working setting: {e}"))
.ipfs_request_budget_secs,
disable
);
}
// At the ceiling: accepted, and the derived deadline still fits. This knob feeds
// only the `Instant` addition (no multiply derivation like the lease steal bound),
// so there is no `checked_mul` clause to carry over from the sibling test.
let at_max =
parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse");
assert_eq!(
at_max.ipfs_request_budget_secs,
GIT_SERVICE_TIMEOUT_SECS_MAX
);
assert!(std::time::Instant::now()
.checked_add(std::time::Duration::from_secs(
at_max.ipfs_request_budget_secs
))
.is_some());
// Past the ceiling, and the top of the u64 range clap used to accept. Only the
// latter actually panics `Instant::now() + Duration::from_secs(..)`; the ceiling
// sits well below that, which is the conservative margin the constant documents.
for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] {
assert!(
parse(over).is_err(),
"{over} is past the representable ceiling and must be rejected at parse time"
);
}
assert!(std::time::Instant::now()
.checked_add(std::time::Duration::from_secs(u64::MAX))
.is_none());
}
#[test]
fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() {
assert_eq!(
Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller,
16
);
assert_eq!(
Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"])
.max_concurrent_reads_per_caller,
4
);
// 0 would shed every read from a keyed caller; clap must reject it.
assert!(
Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"])
.is_err()
);
assert!(Config::try_parse_from([
"gitlawb-node",
"--max-concurrent-reads-per-caller",
"1048577"
])
.is_err());
assert_eq!(
Config::parse_from([
"gitlawb-node",
"--max-concurrent-reads-per-caller",
"1048576"
])
.max_concurrent_reads_per_caller,
1_048_576
);
}
/// #174 F1: a connection-affine write lock pins a pooled connection per
/// concurrent write, so the pool must clear `max_concurrent_git_pushes` by
/// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path.
/// `validate()` must reject an under-sized pool at boot.
#[test]
fn db_pool_must_clear_the_git_push_cap() {
// Shipped defaults validate (48 >= 32 + 8).
Config::parse_from(["gitlawb-node"])
.validate()
.expect("default config must validate");
// An under-sized pool relative to the push cap is rejected (20 < 32 + 8).
let under = Config::parse_from([
"gitlawb-node",
"--db-max-connections",
"20",
"--max-concurrent-git-pushes",
"32",
]);
assert!(
under.validate().is_err(),
"db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected"
);
// Exactly at the floor validates (40 == 32 + 8).
let at_floor = Config::parse_from([
"gitlawb-node",
"--db-max-connections",
"40",
"--max-concurrent-git-pushes",
"32",
]);
assert!(
at_floor.validate().is_ok(),
"db_max_connections at the floor (pushes + headroom) must validate"
);
}
}
+1 -1
View File
@@ -170,7 +170,7 @@ pub async fn encrypt_and_pin(
continue;
}
};
let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope).await {
let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await {
Ok(c) if !c.is_empty() => c,
Ok(_) => {
tracing::warn!(oid = %oid, "pin_git_object returned no cid; skipping");
+30 -1
View File
@@ -47,6 +47,9 @@ pub enum AppError {
#[error("git service timed out: {0}")]
Timeout(String),
#[error("server overloaded: {0}")]
Overloaded(String),
#[error("database error: {0}")]
Db(#[from] sqlx::Error),
@@ -147,6 +150,12 @@ impl IntoResponse for AppError {
DB_UNAVAILABLE_CODE,
DB_UNAVAILABLE_MESSAGE.into(),
),
// 503 with a Retry-After (attached after this match — the shared tail
// can't carry per-variant headers). This is the single place Overloaded
// becomes a response, so it can never ship a 503 without the retry hint.
AppError::Overloaded(msg) => {
(StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone())
}
AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()),
AppError::Internal(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
@@ -160,7 +169,17 @@ impl IntoResponse for AppError {
"message": message,
}));
(status, body).into_response()
let mut resp = (status, body).into_response();
// Overloaded advertises when to retry. It rides the shared tail above for
// its body/status, so the header is attached here rather than in a bespoke
// early return — keeping the variant handled in exactly one place.
if matches!(self, AppError::Overloaded(_)) {
resp.headers_mut().insert(
axum::http::header::RETRY_AFTER,
axum::http::HeaderValue::from_static("1"),
);
}
resp
}
}
@@ -182,4 +201,14 @@ mod tests {
StatusCode::INTERNAL_SERVER_ERROR
);
}
#[test]
fn overloaded_maps_to_503_with_retry_after() {
let resp = AppError::Overloaded("x".into()).into_response();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
resp.headers().get("retry-after").unwrap().to_str().unwrap(),
"1"
);
}
}
+458 -96
View File
@@ -29,9 +29,9 @@
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant};
use anyhow::{bail, Context, Result};
use anyhow::Result;
/// Env var that forces the push path to always full-scan, bypassing the delta
/// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch
@@ -64,11 +64,21 @@ pub enum PinCandidates {
/// Fail-closed: any condition where the introduced set cannot be safely
/// determined returns [`PinCandidates::FullScanRequired`] rather than a partial
/// set, so the caller full-scans instead of silently under-pinning.
pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates {
// KTD7 kill-switch: force the (already-tested) full-scan fallback. The env
// read is split out from the pure logic so the resolver stays unit-testable
// without touching process-global state.
resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan())
pub fn resolve_push_delta(
repo_path: &Path,
new_tips: &[&str],
old_tips: &[&str],
git_bin: &str,
deadline: Instant,
) -> PinCandidates {
resolve_push_delta_inner(
repo_path,
new_tips,
old_tips,
force_full_scan(),
git_bin,
deadline,
)
}
/// Whether the force-full-scan kill-switch env var is set.
@@ -83,6 +93,8 @@ fn resolve_push_delta_inner(
new_tips: &[&str],
old_tips: &[&str],
force_full_scan: bool,
git_bin: &str,
deadline: Instant,
) -> PinCandidates {
if force_full_scan {
tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan");
@@ -102,7 +114,7 @@ fn resolve_push_delta_inner(
// commit). Bare `cat-file -t` returns `tag` for an annotated tag, and
// `for-each-ref %(*objecttype)` peels only one level — neither is correct.
for tip in new_tips {
match peeled_object_type(repo_path, tip) {
match peeled_object_type(repo_path, tip, git_bin, deadline) {
Some(t) if t == "commit" => {}
other => {
tracing::debug!(
@@ -115,7 +127,7 @@ fn resolve_push_delta_inner(
}
}
match rev_list_delta(repo_path, new_tips, old_tips) {
match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) {
Ok(oids) => PinCandidates::Delta(oids),
Err(e) => {
tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan");
@@ -126,41 +138,43 @@ fn resolve_push_delta_inner(
/// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`,
/// `blob`), or `None` if the object is missing/unpeelable or git errored.
fn peeled_object_type(repo_path: &Path, sha: &str) -> Option<String> {
let output = Command::new("git")
.args(["cat-file", "-t", &format!("{sha}^{{}}")])
.current_dir(repo_path)
.output()
.ok()?;
if !output.status.success() {
return None;
}
Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
fn peeled_object_type(
repo_path: &Path,
sha: &str,
git_bin: &str,
deadline: Instant,
) -> Option<String> {
let peel = format!("{sha}^{{}}");
let out = crate::git::visibility_pack::run_bounded_git(
git_bin,
&["cat-file", "-t", &peel],
repo_path,
b"",
deadline,
)
.ok()?;
Some(String::from_utf8_lossy(&out).trim().to_string())
}
/// Run `git rev-list --objects --no-object-names <new> --not <old>` and return
/// the bare OID set. Decides on `status.success()` *before* parsing stdout, so
/// a walk that prints a valid prefix then errors mid-walk is discarded.
fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result<Vec<String>> {
fn rev_list_delta(
repo_path: &Path,
new_tips: &[&str],
old_tips: &[&str],
git_bin: &str,
deadline: Instant,
) -> Result<Vec<String>> {
let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"];
args.extend_from_slice(new_tips);
if !old_tips.is_empty() {
args.push("--not");
args.extend_from_slice(old_tips);
}
let output = Command::new("git")
.args(&args)
.current_dir(repo_path)
.output()
.context("failed to run git rev-list for push delta")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
bail!("git rev-list failed: {stderr}");
}
let stdout = String::from_utf8_lossy(&output.stdout);
let out =
crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?;
let stdout = String::from_utf8_lossy(&out);
Ok(stdout
.lines()
.map(|l| l.trim().to_string())
@@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res
/// reconciliation sweep relies on. It returns *all* objects (including
/// unreachable/dangling ones), which is what the sweep needs to catch
/// stragglers — do not swap it for a reachability walk.
pub fn list_all_objects(repo_path: &Path) -> Result<Vec<String>> {
let output = Command::new("git")
.args([
pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<Vec<String>> {
let out = crate::git::visibility_pack::run_bounded_git(
git_bin,
&[
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname)",
])
.current_dir(repo_path)
.output()
.context("failed to run git cat-file")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
bail!("git cat-file failed: {stderr}");
}
let stdout = String::from_utf8_lossy(&output.stdout);
],
repo_path,
b"",
deadline,
)?;
let stdout = String::from_utf8_lossy(&out);
Ok(stdout
.lines()
.map(|l| l.trim().to_string())
@@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result<Vec<String>> {
/// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed
/// filter needs to tell blobs (content, withholdable) from commits/trees
/// (structural, never withheld) without typing the candidate list itself.
pub fn list_all_objects_with_type(repo_path: &Path) -> Result<Vec<(String, String)>> {
let output = Command::new("git")
.args([
pub fn list_all_objects_with_type(
repo_path: &Path,
git_bin: &str,
deadline: Instant,
) -> Result<Vec<(String, String)>> {
let out = crate::git::visibility_pack::run_bounded_git(
git_bin,
&[
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname) %(objecttype)",
])
.current_dir(repo_path)
.output()
.context("failed to run git cat-file")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
bail!("git cat-file failed: {stderr}");
}
let stdout = String::from_utf8_lossy(&output.stdout);
],
repo_path,
b"",
deadline,
)?;
let stdout = String::from_utf8_lossy(&out);
Ok(stdout
.lines()
.filter_map(|l| {
@@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result<Vec<(String, Strin
/// fail-closed pin filter drops any candidate blob absent from the reachable,
/// visibility-allowed set; a dangling private blob is in this set but not the
/// allowed set, so it never replicates (#99).
pub fn all_blob_oids(repo_path: &Path) -> Result<HashSet<String>> {
Ok(list_all_objects_with_type(repo_path)?
pub fn all_blob_oids(
repo_path: &Path,
git_bin: &str,
deadline: Instant,
) -> Result<HashSet<String>> {
Ok(list_all_objects_with_type(repo_path, git_bin, deadline)?
.into_iter()
.filter(|(_, ty)| ty == "blob")
.map(|(oid, _)| oid)
@@ -268,22 +282,68 @@ pub struct PinCandidateSet {
/// it can never leak because the withheld/fail-closed filter still runs on
/// whatever set is returned. `full_scan` rides on the returned set so the caller
/// knows when the dangling-inclusive filter is required.
///
/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`,
/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta
/// rev-list, and the full-scan fallback — run under ONE permit held for the
/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git
/// and never parks.
///
/// `force_full_scan` forces the full-scan fallback regardless of the tips — the
/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with
/// the env kill-switch (either forces), and it disqualifies the empty-tips fast
/// path: a forced-scan call with no tips must still enumerate the repo, never
/// silently resolve to an empty delta (which would pin nothing and lose the
/// drained pushes' work — the F5 bug shape).
pub async fn resolve_candidates_for_push(
scan_sem: std::sync::Arc<tokio::sync::Semaphore>,
repo_path: PathBuf,
new_tips: Vec<String>,
old_tips: Vec<String>,
git_bin: String,
timeout: Duration,
force_full_scan: bool,
) -> PinCandidateSet {
let force = force_full_scan || self::force_full_scan();
// No-git fast path: a deletion-only push resolves to an empty delta without
// spawning any git child, so it must not park on the scan pool. A forced
// full scan (caller flag or kill-switch) disqualifies it — the scan spawns git.
if new_tips.is_empty() && !force {
tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta");
return PinCandidateSet {
candidates: Vec::new(),
full_scan: false,
};
}
// Scan admission (#174 F4): DEFER, never shed — a dropped scan would
// silently under-pin this push. The permit moves into the blocking closure
// so a started scan always completes holding it. Residuals at
// `acquire_scan_permit`.
let permit =
crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await;
tokio::task::spawn_blocking(move || {
let _permit = permit;
// ONE shared deadline for the whole scan, per jatmn ("the same deadline").
let deadline = Instant::now() + timeout;
let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect();
let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect();
match resolve_push_delta(&repo_path, &new_refs, &old_refs) {
// `force` already folds in the env kill-switch, so the forced arm skips the
// delta machinery outright; the unforced arm goes through the normal
// resolver (whose own env read is false here by construction).
let resolved = if force {
tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)");
PinCandidates::FullScanRequired
} else {
resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline)
};
match resolved {
PinCandidates::Delta(objs) => {
tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta");
PinCandidateSet { candidates: objs, full_scan: false }
}
PinCandidates::FullScanRequired => {
tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback");
match list_all_objects(&repo_path) {
match list_all_objects(&repo_path, &git_bin, deadline) {
Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true },
Err(e) => {
tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)");
@@ -304,8 +364,13 @@ pub async fn resolve_candidates_for_push(
mod tests {
use super::*;
use std::collections::HashSet;
use std::process::Command;
use tempfile::TempDir;
fn td() -> std::time::Instant {
std::time::Instant::now() + std::time::Duration::from_secs(600)
}
/// Minimal git helper for building test repos.
struct Repo {
_td: TempDir,
@@ -363,9 +428,10 @@ mod tests {
let repo = Repo::new();
let c1 = repo.commit_file("a.txt", "one\n");
let c2 = repo.commit_file("b.txt", "two\n");
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1]))
.into_iter()
.collect();
let got: HashSet<String> =
delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td()))
.into_iter()
.collect();
// The new blob b.txt and commit c2 are in the delta; the old blob a.txt
// and commit c1 are not.
let new_blob = repo.rev("HEAD:b.txt");
@@ -383,7 +449,7 @@ mod tests {
// genuinely new objects, never fewer.
let repo = Repo::new();
let c1 = repo.commit_file("a.txt", "one\n");
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c1], &[]))
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td()))
.into_iter()
.collect();
assert!(got.contains(&c1));
@@ -399,9 +465,15 @@ mod tests {
// Rewrite history: reset to base, commit a different file.
repo.git(&["reset", "-q", "--hard", &base]);
let new_tip = repo.commit_file("c.txt", "three\n");
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip]))
.into_iter()
.collect();
let got: HashSet<String> = delta(resolve_push_delta(
&repo.path,
&[&new_tip],
&[&old_tip],
"git",
td(),
))
.into_iter()
.collect();
assert!(got.contains(&new_tip), "new tip in delta");
assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt"))));
// No error; force-push computes new-minus-old cleanly.
@@ -413,7 +485,7 @@ mod tests {
repo.commit_file("a.txt", "one\n");
// All updates were deletions => new_tips empty after the caller strips zeros.
assert_eq!(
resolve_push_delta(&repo.path, &[], &[ZERO]),
resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()),
PinCandidates::Delta(Vec::new())
);
}
@@ -424,7 +496,7 @@ mod tests {
repo.commit_file("a.txt", "one\n");
let blob = repo.rev("HEAD:a.txt");
assert_eq!(
resolve_push_delta(&repo.path, &[&blob], &[]),
resolve_push_delta(&repo.path, &[&blob], &[], "git", td()),
PinCandidates::FullScanRequired,
"a blob tip must force full scan (rev-list would exit 0 and walk it)"
);
@@ -436,7 +508,7 @@ mod tests {
repo.commit_file("a.txt", "one\n");
let tree = repo.rev("HEAD^{tree}");
assert_eq!(
resolve_push_delta(&repo.path, &[&tree], &[]),
resolve_push_delta(&repo.path, &[&tree], &[], "git", td()),
PinCandidates::FullScanRequired
);
}
@@ -449,7 +521,7 @@ mod tests {
repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]);
let tag = repo.rev("treetag");
assert_eq!(
resolve_push_delta(&repo.path, &[&tag], &[]),
resolve_push_delta(&repo.path, &[&tag], &[], "git", td()),
PinCandidates::FullScanRequired,
"annotated tag peeling to a tree must force full scan"
);
@@ -465,7 +537,7 @@ mod tests {
repo.git(&["tag", "-a", "t2", "-m", "x", &t1]);
let t2 = repo.rev("t2");
assert_eq!(
resolve_push_delta(&repo.path, &[&t2], &[]),
resolve_push_delta(&repo.path, &[&t2], &[], "git", td()),
PinCandidates::FullScanRequired
);
}
@@ -480,7 +552,7 @@ mod tests {
let c1 = repo.commit_file("a.txt", "one\n");
repo.git(&["tag", "-a", "rel", "-m", "release", &c1]);
let tag = repo.rev("rel");
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&tag], &[]))
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td()))
.into_iter()
.collect();
assert!(
@@ -501,7 +573,7 @@ mod tests {
let t1 = repo.rev("t1");
repo.git(&["tag", "-a", "t2", "-m", "x", &t1]);
let t2 = repo.rev("t2");
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&t2], &[]))
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td()))
.into_iter()
.collect();
assert!(
@@ -516,9 +588,16 @@ mod tests {
let repo = Repo::new();
let c1 = repo.commit_file("a.txt", "one\n");
let c2 = repo.commit_file("b.txt", "two\n");
let set =
resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()])
.await;
let set = resolve_candidates_for_push(
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
repo.path.clone(),
vec![c2.clone()],
vec![c1.clone()],
"git".to_string(),
std::time::Duration::from_secs(600),
false,
)
.await;
assert!(!set.full_scan, "happy-path delta is not a full scan");
let got: HashSet<String> = set.candidates.into_iter().collect();
let new_blob = repo.rev("HEAD:b.txt");
@@ -536,20 +615,206 @@ mod tests {
let repo = Repo::new();
repo.commit_file("a.txt", "one\n");
let blob = repo.rev("HEAD:a.txt");
let all: HashSet<String> = list_all_objects(&repo.path).unwrap().into_iter().collect();
let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await;
let all: HashSet<String> = list_all_objects(&repo.path, "git", td())
.unwrap()
.into_iter()
.collect();
let set = resolve_candidates_for_push(
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
repo.path.clone(),
vec![blob],
vec![],
"git".to_string(),
std::time::Duration::from_secs(600),
false,
)
.await;
assert!(set.full_scan, "non-commit tip is signalled as a full scan");
let got: HashSet<String> = set.candidates.into_iter().collect();
assert_eq!(got, all, "non-commit tip falls back to full repo scan");
}
/// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip
/// cat-file type probe + delta rev-list, and the full-scan fallback) run under a
/// scan-admission permit: with a zero-permit pool the call parks and spawns no
/// git; once a permit is available the SAME call runs (defer, not shed). On
/// ungated code the git runs regardless of the pool (RED).
#[cfg(unix)]
#[tokio::test]
async fn resolve_candidates_defers_when_scan_pool_exhausted() {
use std::os::unix::fs::PermissionsExt;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
let dir = tempfile::TempDir::new().unwrap();
let marker = dir.path().join("git.ran");
// Fake git records ANY invocation; cat-file reports a commit tip so the
// delta stage proceeds, rev-list yields an empty delta.
let fake = dir.path().join("fakegit");
std::fs::write(
&fake,
format!(
"#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n",
marker.display()
),
)
.unwrap();
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&fake, perm).unwrap();
let git_bin = fake.to_str().unwrap().to_string();
let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string();
let sem: Arc<Semaphore> = Arc::new(Semaphore::new(0));
let blocked = tokio::time::timeout(
Duration::from_millis(500),
resolve_candidates_for_push(
sem.clone(),
dir.path().to_path_buf(),
vec![tip.clone()],
vec![],
git_bin.clone(),
Duration::from_secs(5),
false,
),
)
.await;
assert!(
blocked.is_err(),
"the pin-candidate scan must defer (park on admission) when the pool is exhausted"
);
assert!(
!marker.exists(),
"the scan's git must not spawn while its admission permit is unavailable (F4)"
);
// Release admission: the SAME scan now runs (defer, not shed).
sem.add_permits(1);
let set = resolve_candidates_for_push(
sem,
dir.path().to_path_buf(),
vec![tip],
vec![],
git_bin,
Duration::from_secs(5),
false,
)
.await;
assert!(
marker.exists(),
"once admission is available the deferred scan runs its git"
);
assert!(!set.full_scan, "commit tip + empty rev-list is a delta");
assert!(set.candidates.is_empty());
}
/// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch
/// off) computes its empty delta without spawning ANY git child, so it must
/// complete without acquiring from the scan pool — even at zero permits. The
/// per-tip cat-file probe means every push with a non-empty new tip DOES spawn
/// git, so this is the only genuinely git-free stage.
#[cfg(unix)]
#[tokio::test]
async fn resolve_candidates_no_git_fast_path_skips_admission() {
use std::os::unix::fs::PermissionsExt;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
let dir = tempfile::TempDir::new().unwrap();
let marker = dir.path().join("git.ran");
let fake = dir.path().join("fakegit");
std::fs::write(
&fake,
format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()),
)
.unwrap();
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&fake, perm).unwrap();
let set = tokio::time::timeout(
Duration::from_millis(500),
resolve_candidates_for_push(
Arc::new(Semaphore::new(0)),
dir.path().to_path_buf(),
vec![],
vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()],
fake.to_str().unwrap().to_string(),
Duration::from_secs(5),
false,
),
)
.await
.expect("the no-new-tips fast path must not park on the scan pool");
assert_eq!(
set,
PinCandidateSet {
candidates: Vec::new(),
full_scan: false
}
);
assert!(!marker.exists(), "the fast path must spawn no git at all");
}
/// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit
/// `force_full_scan` flag, and a flagged call with NO tips must still enumerate
/// the whole repo (full_scan=true, non-empty candidates). The RED arm of the
/// encoding question: if the marker were encoded as a plain empty-tips call
/// (flag off — the fast path), the drain would resolve to an empty delta and
/// pin nothing, silently losing the coalesced pushes' work.
#[tokio::test]
async fn forced_full_scan_with_no_tips_enumerates_the_repo() {
let repo = Repo::new();
repo.commit_file("a.txt", "one\n");
let all: HashSet<String> = list_all_objects(&repo.path, "git", td())
.unwrap()
.into_iter()
.collect();
assert!(!all.is_empty(), "fixture repo has objects");
let set = resolve_candidates_for_push(
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
repo.path.clone(),
vec![],
vec![],
"git".to_string(),
std::time::Duration::from_secs(600),
true,
)
.await;
assert!(set.full_scan, "the forced call is signalled as a full scan");
let got: HashSet<String> = set.candidates.into_iter().collect();
assert_eq!(
got, all,
"a forced full scan with no tips enumerates the repo — never an empty delta"
);
// The discriminator: the SAME empty-tips call without the flag is the
// deletion-only fast path (empty delta, pin nothing). The two must differ,
// or the marker encoding has collapsed into the silent-loss shape.
let unforced = resolve_candidates_for_push(
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
repo.path.clone(),
vec![],
vec![],
"git".to_string(),
std::time::Duration::from_secs(600),
false,
)
.await;
assert!(!unforced.full_scan);
assert!(unforced.candidates.is_empty());
}
#[test]
fn missing_oid_tip_forces_full_scan() {
let repo = Repo::new();
repo.commit_file("a.txt", "one\n");
let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
assert_eq!(
resolve_push_delta(&repo.path, &[bogus], &[]),
resolve_push_delta(&repo.path, &[bogus], &[], "git", td()),
PinCandidates::FullScanRequired,
"a missing/corrupt tip OID must force full scan"
);
@@ -564,7 +829,7 @@ mod tests {
let c1 = repo.commit_file("a.txt", "one\n");
let c2 = repo.commit_file("b.txt", "two\n");
let old_tree = repo.rev(&format!("{c1}^{{tree}}"));
let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]);
let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td());
match result {
PinCandidates::FullScanRequired => {} // safe: caller full-scans
PinCandidates::Delta(objs) => {
@@ -583,10 +848,15 @@ mod tests {
// branch2 from base advances independently
repo.git(&["checkout", "-q", "-b", "branch2", &base]);
let b2 = repo.commit_file("c.txt", "three\n");
let got: HashSet<String> =
delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base]))
.into_iter()
.collect();
let got: HashSet<String> = delta(resolve_push_delta(
&repo.path,
&[&b1, &b2],
&[&base, &base],
"git",
td(),
))
.into_iter()
.collect();
assert!(got.contains(&b1), "branch1 new commit");
assert!(got.contains(&b2), "branch2 new commit");
}
@@ -595,7 +865,7 @@ mod tests {
fn empty_repo_no_tips_yields_empty_delta() {
let repo = Repo::new();
assert_eq!(
resolve_push_delta(&repo.path, &[], &[]),
resolve_push_delta(&repo.path, &[], &[], "git", td()),
PinCandidates::Delta(Vec::new())
);
}
@@ -609,12 +879,12 @@ mod tests {
let repo = Repo::new();
let c1 = repo.commit_file("a.txt", "one\n");
assert_eq!(
resolve_push_delta_inner(&repo.path, &[&c1], &[], true),
resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()),
PinCandidates::FullScanRequired
);
// And with the flag off, the same push yields a Delta.
assert!(matches!(
resolve_push_delta_inner(&repo.path, &[&c1], &[], false),
resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()),
PinCandidates::Delta(_)
));
}
@@ -624,7 +894,7 @@ mod tests {
let repo = Repo::new();
repo.commit_file("a.txt", "one\n");
repo.commit_file("b.txt", "two\n");
let all = list_all_objects(&repo.path).unwrap();
let all = list_all_objects(&repo.path, "git", td()).unwrap();
// 2 commits + 2 trees + 2 blobs = 6 objects.
assert_eq!(all.len(), 6, "got: {all:?}");
}
@@ -642,7 +912,7 @@ mod tests {
std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap();
let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]);
let blobs = all_blob_oids(&repo.path).unwrap();
let blobs = all_blob_oids(&repo.path, "git", td()).unwrap();
assert!(blobs.contains(&reachable_blob), "reachable blob present");
assert!(
blobs.contains(&dangling),
@@ -652,7 +922,7 @@ mod tests {
assert!(!blobs.contains(&tree), "tree is not a blob");
// The typed lister tags each object; spot-check the dangling blob's type.
let typed = list_all_objects_with_type(&repo.path).unwrap();
let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap();
assert!(
typed
.iter()
@@ -660,4 +930,96 @@ mod tests {
"dangling object is typed as a blob"
);
}
#[cfg(unix)]
#[test]
fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() {
use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, Instant};
let dir = tempfile::TempDir::new().unwrap();
// Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge).
let fake = dir.path().join("fakegit");
std::fs::write(
&fake,
"#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n",
)
.unwrap();
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&fake, perm).unwrap();
let git_bin = fake.to_str().unwrap().to_string();
let (tx, rx) = std::sync::mpsc::channel();
let path = dir.path().to_path_buf();
std::thread::spawn(move || {
let _ = tx.send(list_all_objects(
&path,
&git_bin,
Instant::now() + Duration::from_millis(150),
));
});
let res = rx.recv_timeout(Duration::from_secs(10)).expect(
"list_all_objects must return within the watchdog budget, not block on a hung git",
);
assert!(
res.is_err(),
"a hung git must make list_all_objects error out, not hang"
);
}
/// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file),
/// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are
/// the remaining candidate-discovery execs (the common push path). Each must
/// return within the watchdog budget on a hung git rather than block and pin the
/// write permit. Revert any one to a bare `Command::output()` and its arm blocks
/// past the recv budget (RED).
#[cfg(unix)]
#[test]
fn delta_path_exec_fns_time_out_on_a_hung_git() {
use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, Instant};
let dir = tempfile::TempDir::new().unwrap();
// Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken
// test can't leak a permanent orphan or wedge the suite).
let fake = dir.path().join("fakegit");
std::fs::write(
&fake,
"#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n",
)
.unwrap();
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&fake, perm).unwrap();
let git_bin = fake.to_str().unwrap().to_string();
let path = dir.path().to_path_buf();
// Run `f` on a thread and require it to RETURN (not block) within 10s.
fn returns_within<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> T {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let _ = tx.send(f());
});
rx.recv_timeout(Duration::from_secs(10)).expect(
"a delta-path exec fn must return within the watchdog budget, not block on a hung git",
)
}
let dl = || Instant::now() + Duration::from_millis(150);
let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
assert!(
returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(),
"peeled_object_type must time out to None on a hung cat-file"
);
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
assert!(
returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(),
"rev_list_delta must error out on a hung rev-list"
);
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
assert!(
returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(),
"list_all_objects_with_type must error out on a hung cat-file"
);
}
}
+845 -21
View File
@@ -13,7 +13,8 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
use sqlx::PgPool;
use sqlx::pool::PoolConnection;
use sqlx::{PgPool, Postgres};
use tokio::sync::Mutex;
use tracing::{debug, info, warn};
@@ -30,6 +31,12 @@ pub struct RepoStore {
/// Tracks repos already confirmed to exist in Tigris — avoids redundant
/// HEAD checks and background uploads for repos we've already migrated.
migrated: Arc<Mutex<HashSet<String>>>,
/// Test-only seam: armed here, copied into every `RepoWriteGuard` this store
/// hands out, so a test that only holds the `AppState` (not the guard) can
/// still park `release` at its pre-unlock point. See
/// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests.
#[cfg(test)]
pre_unlock_gate: Option<Arc<tokio::sync::Notify>>,
}
impl RepoStore {
@@ -40,15 +47,27 @@ impl RepoStore {
tigris: None,
pool,
migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
pre_unlock_gate: None,
}
}
/// Test-only: every guard from this store parks in `release` right before the
/// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future
/// while it is parked reproduces a client disconnect inside `release`.
#[cfg(test)]
pub fn with_pre_unlock_gate(mut self, gate: Arc<tokio::sync::Notify>) -> Self {
self.pre_unlock_gate = Some(gate);
self
}
pub fn new(repos_dir: PathBuf, tigris: Option<TigrisClient>, pool: PgPool) -> Self {
Self {
repos_dir,
tigris,
pool,
migrated: Arc::new(Mutex::new(HashSet::new())),
#[cfg(test)]
pre_unlock_gate: None,
}
}
@@ -157,13 +176,44 @@ impl RepoStore {
let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?;
let lock_key = advisory_lock_key(&owner_slug, repo_name);
// Acquire Postgres advisory lock with retry using pg_try_advisory_lock
// to avoid blocking indefinitely on stale locks from crashed connections.
// Pin a dedicated pooled connection and build the guard holding it BEFORE
// issuing the lock query. Session-level pg advisory locks are
// connection-affine (they can only be released on the session that took
// them), so the guard must own the locking connection; and building the
// guard first means any cancellation after the lock is taken — a
// `tokio::time::timeout` firing during the Tigris download below — drops a
// guard that CAN release, closing the leak the outer timeout otherwise
// opened (#174 F1).
let conn = self
.pool
.acquire()
.await
.context("acquiring db connection for the write advisory lock")?;
let mut guard = RepoWriteGuard {
owner_slug: owner_slug.clone(),
repo_name: repo_name.to_string(),
local_path: local_path.clone(),
lock_key,
conn: Some(conn),
locked: false,
released: false,
tigris: self.tigris.clone(),
#[cfg(test)]
test_pre_unlock_gate: self.pre_unlock_gate.clone(),
};
// Acquire the advisory lock with retry, through the guard's OWN connection,
// so the matching unlock (in release, or the Drop backstop) runs on the same
// session — pg_advisory_unlock on a different pooled connection is a no-op.
let mut acquired = false;
for attempt in 0..60 {
let c = guard
.conn
.as_deref_mut()
.expect("write guard holds its connection during acquisition");
let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(lock_key)
.fetch_one(&self.pool)
.fetch_one(&mut *c)
.await
.context("trying advisory lock")?;
if row.0 {
@@ -177,9 +227,11 @@ impl RepoStore {
if !acquired {
anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}");
}
guard.locked = true;
// Always download the latest from Tigris before writing.
// Local disk may be stale if another machine pushed since our last access.
// Always download the latest from Tigris before writing. Local disk may be
// stale if another machine pushed since our last access. The guard already
// owns the lock + its connection, so a cancellation here drops through Drop.
if let Some(ref tigris) = self.tigris {
if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) {
debug!(repo = %repo_name, "write acquire: downloading latest from tigris");
@@ -197,14 +249,7 @@ impl RepoStore {
}
}
Ok(RepoWriteGuard {
owner_slug,
repo_name: repo_name.to_string(),
local_path,
lock_key,
pool: self.pool.clone(),
tigris: self.tigris.clone(),
})
Ok(guard)
}
/// Initialize a new bare repo on local disk and upload to Tigris.
@@ -479,8 +524,62 @@ pub struct RepoWriteGuard {
repo_name: String,
pub local_path: PathBuf,
lock_key: i64,
pool: PgPool,
/// The pooled connection that took the advisory lock. Session-level pg
/// advisory locks are connection-affine, so the guard pins that connection
/// for its whole lifetime and unlocks on it (in `release`, or the `Drop`
/// backstop). `None` only after the connection has been taken, either to run
/// the detached unlock in `Drop` or to be closed when `release`'s unlock
/// errored (#174 F3b).
conn: Option<PoolConnection<Postgres>>,
/// Set once the advisory lock has actually been taken. A guard dropped
/// before the lock is held (or after `release`) performs no unlock.
locked: bool,
/// Set once `release` has run its unlock, making the `Drop` backstop inert.
released: bool,
tigris: Option<TigrisClient>,
/// Test-only seam: when set, `release` parks on this gate at the exact point
/// it is about to await `pg_advisory_unlock` (connection still owned, not yet
/// released). Dropping the `release` future while it is parked reproduces a
/// mid-unlock cancellation, so a test can assert the `Drop` backstop still
/// frees the session lock. Never set outside tests.
#[cfg(test)]
test_pre_unlock_gate: Option<Arc<tokio::sync::Notify>>,
}
/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`.
/// Long enough that a healthy socket always finishes well inside it, short enough that
/// a blackholed one does not pin admission resources for a TCP timeout.
const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// Await `close` under a deadline (#174 F3c).
///
/// `release` awaits this INLINE while the global write permit, the per-source permit
/// and the write lease are all still held, and sqlx puts no deadline on `close()`:
/// it writes Terminate and then tears the socket down. The branch that reaches here is
/// by definition a connection whose last statement errored, and a blackholed TCP path
/// to Postgres (a cloud failover that drops packets without an RST) is a plausible
/// cause, so an unbounded await here parks every later push to the repo behind three
/// pinned admission resources until the steal bound.
///
/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns.
/// Dropping it closes the socket, and closing the socket is what actually ends the
/// session and makes Postgres release the lock, so the deadline costs nothing the
/// graceful path was buying.
async fn close_conn_bounded(
repo_name: &str,
close: impl std::future::Future<Output = Result<(), sqlx::Error>>,
) {
match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await {
Ok(Ok(())) => {}
Ok(Err(e)) => {
warn!(repo = %repo_name, err = %e,
"closing the write-lock connection failed, the session teardown still frees the lock server-side");
}
Err(_) => {
warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(),
"closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side");
}
}
}
impl RepoWriteGuard {
@@ -494,7 +593,7 @@ impl RepoWriteGuard {
/// half-applied or otherwise inconsistent repo would propagate corruption to
/// Tigris (and to every node that later downloads it). The lock is always
/// released regardless, to avoid stale locks blocking future writes.
pub async fn release(self, success: bool) {
pub async fn release(mut self, success: bool) {
// Upload to Tigris only on success.
if success {
if let Some(ref tigris) = self.tigris {
@@ -509,11 +608,124 @@ impl RepoWriteGuard {
warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo");
}
// Release advisory lock
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(self.lock_key)
.execute(&self.pool)
.await;
// Release the advisory lock on the SAME connection that took it (session
// advisory locks are connection-affine). Unlock through the connection
// while it is STILL owned by `self` — do not `take()` it first. If this
// future is cancelled during the unlock await, `self` is dropped with
// `conn == Some(..)` and `released == false`, so the `Drop` backstop still
// runs the detached unlock. `released` is set only AFTER the await
// resolves, so a cancellation cannot make the backstop inert (#174 F4).
if self.locked {
#[cfg(test)]
let pre_unlock_gate = self.test_pre_unlock_gate.clone();
let unlock = if let Some(conn) = self.conn.as_deref_mut() {
// Test-only: park right before the unlock await so a test can drop
// this future mid-unlock (connection owned, not yet released).
#[cfg(test)]
if let Some(gate) = pre_unlock_gate {
gate.notified().await;
}
Some(
sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(self.lock_key)
.execute(&mut *conn)
.await,
)
} else {
None
};
// An unlock that ERRORS is a different failure from a cancellation: the
// await resolved, so `Drop` is about to be made inert by `released`
// below, but the session is still alive and still holds the lock
// (statement timeout, admin cancel, aborted transaction). Returning that
// `PoolConnection` to the pool would hand the next caller a connection
// holding a lock nobody tracks (#174 F3b). Connection disposal is the
// single mechanism here, and it is why we do not instead try to keep the
// `Drop` backstop armed: disposal needs `conn.take()`, and `Drop`
// early-returns on `conn == None`. Ending the session is what frees the
// lock, so `released = true` still holds.
if let Some(Err(e)) = unlock {
warn!(repo = %self.repo_name, err = %e,
"advisory unlock failed, closing the connection so the session ends and postgres drops the lock");
if let Some(conn) = self.conn.take() {
// `close()` over `detach()`: both consume the `PoolConnection` by
// value in sqlx 0.8.6, but we are in an async fn, so `close()`
// sends Terminate and waits for the socket to go down before
// `release` returns. `detach()` would only end the session
// whenever the returned `PgConnection` is dropped and its
// background close completes. If this future is cancelled during
// `close()`, the connection is dropped mid-close, which still
// tears the session down. That last point is also why the await is
// safe to bound: see `close_conn_bounded`, which gives it the
// deadline sqlx does not.
close_conn_bounded(&self.repo_name, conn.close()).await;
}
}
}
self.released = true;
}
}
impl Drop for RepoWriteGuard {
/// Cancellation-safe backstop: if the guard is dropped while still holding the
/// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a
/// handler future was dropped before `release`), unlock on the pinned
/// connection. This is NOT the backstop for an unlock that ran and returned an
/// error: that case is closed inside `release` by disposing of the connection,
/// because `Drop` early-returns on `conn == None` and the two mechanisms cannot
/// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the
/// same session (connection-affine). An off-runtime drop has nothing to spawn onto,
/// so it disposes of the connection instead. On runtime
/// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock
/// may not run — but shutdown tears down the pool, and closing the connection
/// releases the session-level advisory lock server-side, so this too is bounded.
fn drop(&mut self) {
if self.released || !self.locked {
return;
}
let Some(mut conn) = self.conn.take() else {
return;
};
let lock_key = self.lock_key;
let repo_name = self.repo_name.clone();
match tokio::runtime::Handle::try_current() {
Ok(handle) => {
handle.spawn(async move {
let unlock = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(lock_key)
.execute(&mut *conn)
.await;
// Same failure as `release`'s (#174 F3b), one level down: the await
// RESOLVED with an error, so the session is alive and still holds
// the lock. Letting this async block end here would drop `conn` and
// RETURN it to the pool, handing the next caller a connection
// holding a lock nobody tracks. Close it instead, which both keeps
// it out of the pool and ends the session that holds the lock.
if let Err(e) = unlock {
warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock");
close_conn_bounded(&repo_name, conn.close()).await;
}
});
}
Err(_) => {
// No runtime to spawn the unlock onto, and the connection is already
// out of the guard, so there is no path that unlocks on this session.
// Returning it to the pool would hand the next caller a connection
// still holding the lock. `PoolConnection`'s own drop also spawns its
// return-to-pool task, which panics with no runtime. `detach` gives up
// the pool slot and yields a plain `PgConnection`; dropping that closes
// the socket, which ends the session and is what frees the lock
// server-side. `Drop` cannot await, so this is the whole disposal:
// `close_conn_bounded` is not available here.
drop(conn.detach());
warn!(
repo = %repo_name,
"RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \
possible, so the pinned connection is disposed of instead: ending \
the session is what releases the advisory lock"
);
}
}
}
}
@@ -932,4 +1144,616 @@ mod tests {
);
}
}
// ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ──
/// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the
/// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when
/// it fires during the Tigris await — must still release the session advisory
/// lock. A checker connection is held OUT of the pool first, so `acquire_write`
/// is forced onto a distinct session; the checker (a different session) then
/// probes the lock, so advisory-lock re-entrancy cannot mask a leak.
///
/// Load-bearing: RED today (no `Drop` releases the lock → held by
/// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns
/// false). GREEN after the connection-affine `Drop` backstop.
#[sqlx::test]
async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA";
let name = "leaktest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
// Distinct session for the probe: hold it out of the pool BEFORE acquiring,
// so acquire_write cannot use it and a re-entrant probe cannot falsely read free.
let mut checker = pool.acquire().await.expect("checker connection");
let guard = store.acquire_write(owner, name).await.expect("acquire");
// The cancellation shape: drop without release().
drop(guard);
// Let the detached unlock task run.
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(
free,
"advisory lock must be released when the guard is dropped without release()"
);
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// F1 (latent non-affine release): `release()` must unlock on the SAME
/// session that locked, so the lock is freed regardless of which pooled
/// connection would service a fresh query. Observed from a distinct session.
#[sqlx::test]
async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB";
let name = "affinetest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
let mut checker = pool.acquire().await.expect("checker connection");
let guard = store.acquire_write(owner, name).await.expect("acquire");
guard.release(false).await;
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(
free,
"release() must free the advisory lock via connection-affine unlock"
);
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
// ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ──────────
/// F4 (P1): a cancellation DURING the unlock await must still free the session
/// advisory lock. `release` unlocks through the connection while `self` still
/// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some`
/// + `locked && !released` and runs its detached-unlock backstop. A test-only
/// gate parks `release` at the exact pre-unlock point; dropping the future
/// there reproduces the cancellation.
///
/// Load-bearing: RED on the original ordering (`self.conn.take()` before the
/// await → at cancellation `self.conn == None` → `Drop` skips → the local
/// connection returns to the pool with the session lock still held → the
/// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder.
#[sqlx::test]
async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC";
let name = "canceltest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
// Distinct session for the probe, held out of the pool before acquiring.
let mut checker = pool.acquire().await.expect("checker connection");
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
// Arm the pre-unlock gate; it is never notified, so `release` parks on it
// with the connection still owned and `released` still false.
let gate = Arc::new(tokio::sync::Notify::new());
guard.test_pre_unlock_gate = Some(gate);
// Box the future so we can drop it ourselves (tokio::pin! keeps it alive to
// end of scope, which would defer the guard's Drop past the assertions).
let mut fut = Box::pin(guard.release(false));
let parked =
tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await;
assert!(
parked.is_err(),
"release should park on the pre-unlock gate, not complete"
);
// Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop.
drop(fut);
// Let the Drop backstop's detached unlock task run.
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(
free,
"advisory lock must be freed when release is cancelled mid-unlock — the \
connection must stay owned by the guard so Drop's backstop can run"
);
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// F4: the ordinary success path still frees the lock, and a second write
/// acquire on the same repo returns promptly (no stale-lock retry loop).
#[sqlx::test]
async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD";
let name = "reltruetest";
let guard = store
.acquire_write(owner, name)
.await
.expect("first acquire");
guard.release(true).await;
let again = tokio::time::timeout(
std::time::Duration::from_secs(2),
store.acquire_write(owner, name),
)
.await
.expect("second acquire_write must not hit the ~60s stale-lock retry loop")
.expect("second acquire");
again.release(true).await;
}
// ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─
/// Put the guard's pinned connection into a failed-transaction state, so the
/// next statement on it errors while the SESSION stays alive and keeps holding
/// the session-level advisory lock (those survive a transaction abort; only
/// `pg_advisory_xact_lock` would not). This is the smallest injection that
/// reproduces F3b's shape: `pg_advisory_unlock` returning `Err` on a live,
/// still-locking session. No production seam is needed because the tests live
/// in this module and can reach `conn` directly.
async fn poison_guard_connection(guard: &mut RepoWriteGuard) {
let conn = guard
.conn
.as_deref_mut()
.expect("guard holds its connection before release");
sqlx::query("BEGIN")
.execute(&mut *conn)
.await
.expect("open a transaction on the guard connection");
let poisoned = sqlx::query("SELECT 1 / 0").execute(&mut *conn).await;
assert!(
poisoned.is_err(),
"the poison statement must fail so the transaction is aborted"
);
}
/// How long the polls below may wait for an asynchronous server-side effect.
/// Postgres frees a session advisory lock when the backend exits, which happens
/// asynchronously to our socket close, so these are polled to a generous deadline
/// rather than slept for a fixed interval: a constant sleep is a flake under load,
/// and one that is long enough to be safe is dead time on every run.
const RELEASE_POLL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20);
/// Poll `cond` until it holds, failing at the deadline so a regression fails the
/// test rather than hanging the suite.
async fn wait_until(mut cond: impl FnMut() -> bool, what: &str) {
let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE;
while !cond() {
assert!(
std::time::Instant::now() < deadline,
"timed out waiting for {what}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
}
/// Poll the advisory lock from `checker` until it is free. `pg_try_advisory_lock`
/// ACQUIRES on success, so a true result both answers the question and leaves the
/// checker session holding the lock; the caller unlocks it.
async fn wait_until_lock_free(checker: &mut sqlx::PgConnection, key: i64, what: &str) {
let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE;
loop {
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
if free {
return;
}
assert!(
std::time::Instant::now() < deadline,
"timed out waiting for {what}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
}
/// A second pool over the same test database with the idle reaper DISABLED.
///
/// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to
/// it is closed by the reaper about a second later, which ends the session and
/// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside
/// that window by luck; polling to a deadline long enough to be flake-proof would
/// land outside it and go green whether or not `release` disposed of the
/// connection, so the tests below would stop testing anything (measured: with the
/// reaper in play, the disposal shows up at ~2s even with the fix reverted). With
/// no reaper, `release` is the only thing that can end that session, so the poll
/// measures exactly the property these two tests exist for.
async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.idle_timeout(None)
.max_lifetime(None)
.connect_with(pool.connect_options().as_ref().clone())
.await
.expect("a second pool over the test database")
}
/// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED.
/// `release` awaits it inline while the global write permit, the per-source permit
/// and the write lease are all still held, and sqlx's `close()` carries no deadline
/// of its own, so a blackholed socket would park every later push to that repo
/// behind three pinned admission resources.
///
/// What this covers: the deadline itself. A close that never resolves still lets
/// `close_conn_bounded` return, which is the property `release` depends on. What it
/// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is
/// what stalls in production. Making a real `PgConnection::close` hang needs a
/// blackholed TCP path to Postgres, and the flip has to land after the unlock
/// statement round-trips but before the Terminate write, which is not a seam this
/// module exposes. A never-resolving future is the faithful stand-in for that
/// close, and the F3b tests above already cover that `release` really routes its
/// close through here.
///
/// Time is paused, so nothing here depends on wall clock: the runtime auto-advances
/// to the next timer, and the assertion is on which timer fired, not on elapsed
/// time. The outer bound is what turns a removed deadline into a failure rather
/// than a hung suite.
///
/// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the
/// inner future never resolves, so the outer bound fires and this fails.
#[tokio::test(start_paused = true)]
async fn unlock_error_connection_close_is_bounded() {
let hanging = std::future::pending::<Result<(), sqlx::Error>>();
let outcome = tokio::time::timeout(
UNLOCK_ERROR_CLOSE_TIMEOUT * 4,
close_conn_bounded("boundedclosetest", hanging),
)
.await;
assert!(
outcome.is_ok(),
"a connection close that never completes must not hold the write lease and \
both admission permits open-endedly: close_conn_bounded must give up and \
drop the connection"
);
}
/// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive
/// (statement timeout, admin cancel, aborted transaction), the lock must not
/// survive `release`. The old code discarded the error with `let _ =` and set
/// `released = true` anyway, so `Drop` early-returned and the `PoolConnection`
/// went back to the pool still holding the session lock.
///
/// Observed from a SEPARATE connection held out of the pool before acquiring:
/// session advisory locks are re-entrant and counted, so probing from the
/// holding session (or via a fresh `acquire_write` that may be handed the same
/// connection) would report free whether or not the fix is present.
///
/// Load-bearing: RED before the fix (lock still held → `pg_try_advisory_lock`
/// returns false), GREEN after (the errored connection is closed, so the
/// session ends and Postgres drops the lock).
#[sqlx::test]
async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store_pool = pool_without_idle_reaper(&pool).await;
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF";
let name = "unlockerrtest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
// Distinct session for the probe, held out of the pool before acquiring.
let mut checker = pool.acquire().await.expect("checker connection");
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
poison_guard_connection(&mut guard).await;
// Sanity: the poisoned session is still alive and still holds the lock, so
// the assertion below measures the release path and not a dead session.
let (free_before,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(
!free_before,
"the poisoned session must still hold the lock before release"
);
guard.release(false).await;
// Postgres drops the lock when the disposed session's backend exits, which is
// asynchronous to our socket close: poll for it rather than sleeping a
// constant. The deadline is what keeps this load-bearing: a release that
// leaves the lock held never satisfies the probe and fails here.
wait_until_lock_free(
&mut checker,
key,
"an errored pg_advisory_unlock must not leave the lock held: release must \
dispose of the connection so the session ends",
)
.await;
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// F3b: the connection that saw the unlock error must leave the pool entirely,
/// rather than being handed to the next caller while still holding the lock.
/// `PgPool::size()` counts the connections the pool owns, so closing the
/// errored one is observable as a drop in that count.
#[sqlx::test]
async fn write_guard_release_with_failing_unlock_does_not_return_the_connection(
pool: sqlx::PgPool,
) {
let dir = tempfile::TempDir::new().unwrap();
let store_pool = pool_without_idle_reaper(&pool).await;
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG";
let name = "unlockerrpooltest";
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
poison_guard_connection(&mut guard).await;
let size_before = store_pool.size();
assert!(size_before > 0, "the pool owns the guard's connection");
guard.release(false).await;
// The pool's size drops when the closed connection's slot is given up, which
// is not synchronous with `release` returning: poll rather than sleep.
wait_until(
|| store_pool.size() == size_before - 1,
"the connection that saw the unlock error to be closed rather than returned \
to the pool still holding the session lock",
)
.await;
}
/// F3b regression guard on the success path: a normal unlock keeps the
/// connection in the pool and marks the guard released, so the disposal branch
/// is confined to the error case.
#[sqlx::test]
async fn write_guard_release_success_keeps_the_connection_and_frees_the_lock(
pool: sqlx::PgPool,
) {
let dir = tempfile::TempDir::new().unwrap();
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
let owner = "did:key:z6MkUnlockOkProofHHHHHHHHHHHHHHHHHHHHHHHH";
let name = "unlockoktest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
let mut checker = pool.acquire().await.expect("checker connection");
let guard = store.acquire_write(owner, name).await.expect("acquire");
let size_before = pool.size();
guard.release(false).await;
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
assert_eq!(
pool.size(),
size_before,
"a successful unlock must leave the connection in the pool"
);
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(free, "the success path must still free the lock");
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
// ── the Drop backstop disposes its connection too (#174 U8) ─────────────
/// U8 (P2): the `Drop` backstop carries the same hazard F3b closed in `release`.
/// When the detached `pg_advisory_unlock` ERRORS on a live session, the async
/// block ends and drops the moved `PoolConnection`, which RETURNS it to the pool
/// while that session may still hold the lock, handing the next caller a
/// connection holding a lock nobody tracks. The errored connection must be closed
/// instead: that keeps it out of the pool and ends the session, which is what
/// frees the lock server-side.
///
/// Run against a pool with the idle reaper disabled, for the reason spelled out on
/// `pool_without_idle_reaper`: with the reaper in play the session dies on its own
/// about a second later and the assertion stops measuring the disposal.
///
/// Load-bearing: RED before the fix (the connection goes back to the pool, so the
/// size never drops and this times out), GREEN after.
#[sqlx::test]
async fn write_guard_drop_with_failing_unlock_does_not_return_the_connection(
pool: sqlx::PgPool,
) {
let dir = tempfile::TempDir::new().unwrap();
let store_pool = pool_without_idle_reaper(&pool).await;
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
let owner = "did:key:z6MkDropUnlockErrProofIIIIIIIIIIIIIIIIIIII";
let name = "dropunlockerrtest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
// Distinct session for the probe, held out of the store's pool entirely.
let mut checker = pool.acquire().await.expect("checker connection");
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
poison_guard_connection(&mut guard).await;
let size_before = store_pool.size();
assert!(size_before > 0, "the pool owns the guard's connection");
// The backstop shape: dropped without release(), with an unlock that errors.
drop(guard);
wait_until(
|| store_pool.size() == size_before - 1,
"the connection whose detached unlock errored to be closed rather than \
returned to the pool still holding the session lock",
)
.await;
wait_until_lock_free(
&mut checker,
key,
"an errored detached unlock must not leave the lock held: Drop must dispose \
of the connection so the session ends",
)
.await;
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// U8 regression guard on the success path: a detached unlock that SUCCEEDS must
/// still return the connection to the pool. Without this, "close the connection on
/// Drop" could be widened to "always close" and the test above would not notice.
#[sqlx::test]
async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store_pool = pool_without_idle_reaper(&pool).await;
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ";
let name = "dropunlockoktest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
let mut checker = pool.acquire().await.expect("checker connection");
let guard = store.acquire_write(owner, name).await.expect("acquire");
let size_before = store_pool.size();
assert!(size_before > 0, "the pool owns the guard's connection");
drop(guard);
// The connection goes back only once the detached unlock task has finished.
wait_until(
|| store_pool.num_idle() > 0,
"the detached unlock to finish and hand the connection back",
)
.await;
assert_eq!(
store_pool.size(),
size_before,
"a successful detached unlock must leave the connection in the pool"
);
wait_until_lock_free(
&mut checker,
key,
"the Drop backstop's successful unlock to free the lock",
)
.await;
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the
/// unlock onto, and the connection has already been taken out of the guard, so the
/// old code dropped it with no unlock attempted at all, back to the pool, session
/// lock still held. Dropping a `PoolConnection` off a runtime is worse than that:
/// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the
/// old arm also panicked in a destructor.
///
/// Reached by dropping the guard on a plain `std::thread`, where
/// `Handle::try_current()` fails.
///
/// Load-bearing: RED before the fix (the join sees the "requires a Tokio context"
/// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the
/// pool slot, so nothing is spawned and dropping the detached connection closes
/// the socket, which ends the session and frees the lock).
#[sqlx::test]
async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let store_pool = pool_without_idle_reaper(&pool).await;
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK";
let name = "dropoffruntimetest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
let mut checker = pool.acquire().await.expect("checker connection");
let guard = store.acquire_write(owner, name).await.expect("acquire");
let size_before = store_pool.size();
assert!(size_before > 0, "the pool owns the guard's connection");
let dropped = std::thread::spawn(move || drop(guard)).join();
assert!(
dropped.is_ok(),
"dropping a write guard off a Tokio runtime must not panic"
);
wait_until(
|| store_pool.size() == size_before - 1,
"the connection of a guard dropped off a runtime to be disposed of rather \
than returned to the pool with no unlock attempted",
)
.await;
wait_until_lock_free(
&mut checker,
key,
"a guard dropped off a runtime to end its session so postgres drops the lock",
)
.await;
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
/// F4: releasing a guard that never took the lock (`locked == false`, the state
/// acquire_write leaves after a failed acquisition) must not unlock or panic.
#[sqlx::test]
async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) {
let dir = tempfile::TempDir::new().unwrap();
let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE";
let name = "notlockedtest";
let slug = owner.replace([':', '/'], "_");
let key = advisory_lock_key(&slug, name);
let guard = RepoWriteGuard {
owner_slug: slug,
repo_name: name.to_string(),
local_path: dir.path().to_path_buf(),
lock_key: key,
conn: Some(pool.acquire().await.expect("conn")),
locked: false,
released: false,
tigris: None,
test_pre_unlock_gate: None,
};
// Must complete without panic and issue no unlock.
guard.release(false).await;
let mut checker = pool.acquire().await.expect("checker");
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
.bind(key)
.fetch_one(&mut *checker)
.await
.unwrap();
assert!(free, "release on an unlocked guard must not touch the key");
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
.bind(key)
.execute(&mut *checker)
.await;
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+19
View File
@@ -31,6 +31,25 @@ impl TigrisClient {
})
}
/// Test-only constructor with an explicit S3 endpoint, region, and static
/// credentials — no env-var reads, so parallel tests cannot race each other's
/// `AWS_*` environment the way the env-based `new` would. Lets a test point
/// the client at a non-routable endpoint to exercise acquire-stall paths.
#[cfg(test)]
pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self {
let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test");
let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.endpoint_url(endpoint_url)
.region(aws_config::Region::new("auto"))
.credentials_provider(creds)
.load()
.await;
Self {
s3: S3Client::new(&config),
bucket: bucket.to_string(),
}
}
/// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst`
fn repo_key(owner_slug: &str, repo_name: &str) -> String {
format!("repos/v1/{owner_slug}/{repo_name}.tar.zst")
+744 -94
View File
@@ -4,11 +4,321 @@
//! content is held back.
use crate::db::VisibilityRule;
use crate::git::store;
use crate::visibility::{visibility_check, Decision};
use anyhow::{Context, Result};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::path::Path;
use std::process::Stdio;
use std::sync::mpsc;
use std::time::{Duration, Instant};
/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3).
/// The walk is fast for a real repo; this bound exists to reap a hung or
/// pathologically slow git child so it cannot pin a served-git permit (the read
/// permit on the upload-pack serve path, the write permit on the receive-pack
/// post-push replication path) past the deadline. Every caller funnels through
/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers
/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this
/// fixed budget only backs the `git_bin`-less test wrappers.
#[cfg(test)]
const WALK_TIMEOUT: Duration = Duration::from_secs(600);
/// How long the process-group watchdog waits after SIGTERM before escalating to
/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only
/// paid on a timeout (already the exceptional path).
#[cfg(unix)]
const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1);
/// Run one git child under a shared `deadline` with process-group teardown,
/// BLOCKING, and return its stdout. The child runs in its own process group; a
/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs,
/// the whole group if the deadline passes before the child is reaped, so a hung or
/// slow git can pin neither a served-git permit nor a blocking thread past the
/// deadline (jatmn's "retain admission until they are reaped"). This is the
/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the
/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot
/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so
/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can
/// drive the teardown in tests without mutating the process-global PATH;
/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children).
/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT
/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child
/// waitable, so the caller's later `child.wait()` still collects the status and the
/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)`
/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually
/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a).
#[cfg(unix)]
fn child_terminated_without_reaping(pid: i32) -> bool {
// SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory;
// WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking.
let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() };
let rc = unsafe {
libc::waitid(
libc::P_PID,
pid as libc::id_t,
&mut info,
libc::WEXITED | libc::WNOHANG | libc::WNOWAIT,
)
};
// rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero
// si_pid means the child has entered a waitable, exited state. EINTR/other errors
// (rc != 0) are treated as "not yet terminated" and the caller re-polls.
rc == 0 && unsafe { info.si_pid() } != 0
}
#[cfg(unix)]
pub(crate) fn run_bounded_git_raw(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
use std::io::{Read, Write};
use std::os::unix::process::CommandExt;
use std::sync::mpsc::RecvTimeoutError;
let label = args.first().copied().unwrap_or("git");
let mut child = std::process::Command::new(git_bin)
.args(args)
.current_dir(repo_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.process_group(0)
.spawn()
.with_context(|| format!("failed to spawn git {label}"))?;
// With process_group(0) the child leads its own group, so pgid == its pid.
let pgid = child.id() as i32;
// Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace
// for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL
// SIGKILL of the group. It never stands down on leader-reap alone: a group member
// that ignores SIGTERM while the leader exits cleanly would otherwise escape the
// SIGKILL and keep running past the deadline (finding 3, #174). The main thread
// defers reaping the leader until this thread returns (see below), so the leader's
// pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have
// been recycled — which is why this no longer needs the old `reaped` short-circuit.
// Kept off the main thread because the main thread's stdout drain is exactly what
// blocks until a hung child is torn down.
let (done_tx, done_rx) = mpsc::channel::<()>();
let watchdog = std::thread::spawn(move || -> bool {
let wait = deadline.saturating_duration_since(Instant::now());
match done_rx.recv_timeout(wait) {
Ok(()) | Err(RecvTimeoutError::Disconnected) => false,
Err(RecvTimeoutError::Timeout) => {
// SAFETY: kill(2) takes only integers and borrows no Rust memory;
// ESRCH on an already-gone group is ignored.
unsafe { libc::kill(-pgid, libc::SIGTERM) };
// Fixed grace: because the main thread defers the leader's reap, a
// fully-exited group still shows a zombie leader here, so polling for
// ESRCH cannot detect early completion — just wait the grace, then
// SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op;
// on a SIGTERM-ignoring member it is what actually kills it.
std::thread::sleep(WATCHDOG_TERM_GRACE);
unsafe { libc::kill(-pgid, libc::SIGKILL) };
// Brief settle so the SIGKILL is delivered before the main thread
// reaps the leader and frees the pgid. A wedged (D-state) member
// survives even SIGKILL — the documented residual, as in smart_http.
std::thread::sleep(Duration::from_millis(20));
if unsafe { libc::kill(-pgid, 0) } == 0 {
tracing::warn!(
pgid,
"withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)"
);
}
true
}
}
});
// Feed stdin on a writer thread and drain stderr on a reader thread so the main
// thread can drain stdout concurrently; writing all of stdin (or draining one
// pipe) before the others can deadlock once a pipe buffer fills.
let mut stdin = child.stdin.take();
let input = stdin_bytes.to_vec();
let writer = std::thread::spawn(move || {
if let Some(mut s) = stdin.take() {
let _ = s.write_all(&input);
}
});
let mut stderr = child.stderr.take().context("git stderr was not piped")?;
let err_reader = std::thread::spawn(move || {
let mut err = Vec::new();
let _ = stderr.read_to_end(&mut err);
err
});
let mut stdout = child.stdout.take().context("git stdout was not piped")?;
let mut out = Vec::new();
// Blocking drain, unblocked by the child closing stdout on exit. The watchdog's
// SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible
// (D-state) I/O survives even SIGKILL, so this drain and the wait below can block
// until the kernel returns, pinning the walk thread and its permit. That residual
// is unreachable in userspace (no signal reaps a D-state process) and matches the
// async `reap_group_on_timeout`, which likewise only warns and gives up there.
let read_result = stdout.read_to_end(&mut out);
// The drain has returned, but that only means all stdout write ends are closed —
// NOT that the child has exited. A group member, or the leader itself, can close
// stdout and keep running; standing the watchdog down on the drain alone (as the
// old code did) would then let `child.wait()` block forever on that live child,
// past the deadline, pinning the walk thread and its permit (finding P1-a, #174).
// So stand the watchdog down only once the child has ACTUALLY terminated, detected
// WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its
// pgid un-recycled until the watchdog finishes and we join it below. Past the
// deadline the watchdog owns the teardown, so we stop polling and let it run the
// full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every
// `kill(-pgid)` firing while the pid is still unreaped and guarantees a
// stdout-closing-then-hanging member has been SIGKILLed rather than left running.
loop {
if child_terminated_without_reaping(pgid) {
let _ = done_tx.send(());
break;
}
if Instant::now() >= deadline {
break;
}
std::thread::sleep(Duration::from_millis(2));
}
let killed = watchdog.join().unwrap_or(false);
let status = child.wait().context("git wait failed")?;
let err = err_reader.join().unwrap_or_default();
let _ = writer.join();
read_result.context("failed to read git stdout")?;
// The watchdog runs off a wall clock that can race a child finishing right at the
// deadline. A child that exited on its own (success) is not a timeout even if the
// watchdog fired late; only a child that did not exit successfully is a genuine
// timeout, which keeps a walk completing at its budget from a spurious 504.
if killed && !status.success() {
return Err(crate::git::smart_http::GitServiceTimeout.into());
}
Ok((status, out, err))
}
/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin
/// wrapper the walk callers use. Probes that must distinguish exit classes —
/// `git cat-file` absence vs an object-store access failure — call
/// [`run_bounded_git_raw`] and classify the status/stderr themselves.
#[cfg(unix)]
pub(crate) fn run_bounded_git(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<Vec<u8>> {
let label = args.first().copied().unwrap_or("git");
let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?;
if !status.success() {
anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err));
}
Ok(out)
}
/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets
/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only),
/// so this bounds a single child on its own: threads feed stdin and drain stderr
/// while the main thread drains stdout, and a watchdog thread kills the child at the
/// deadline (which closes stdout and unblocks the drain). The child is shared with
/// the watchdog behind a mutex that the main thread does NOT hold while draining, so
/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct
/// child, not a descendant group — which is why the hardened, group-aware path above
/// is gated to Unix, the only target the served node actually runs on (the Windows
/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with
/// the Unix version's signature and result semantics so every caller compiles on all
/// targets (#174).
#[cfg(not(unix))]
pub(crate) fn run_bounded_git_raw(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
use std::io::{Read, Write};
use std::sync::mpsc::RecvTimeoutError;
let label = args.first().copied().unwrap_or("git");
let mut child = std::process::Command::new(git_bin)
.args(args)
.current_dir(repo_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.with_context(|| format!("failed to spawn git {label}"))?;
let mut stdin = child.stdin.take();
let input = stdin_bytes.to_vec();
let writer = std::thread::spawn(move || {
if let Some(mut s) = stdin.take() {
let _ = s.write_all(&input);
}
});
let mut stderr = child.stderr.take().context("git stderr was not piped")?;
let err_reader = std::thread::spawn(move || {
let mut err = Vec::new();
let _ = stderr.read_to_end(&mut err);
err
});
let mut stdout = child.stdout.take().context("git stdout was not piped")?;
// Share the child with the watchdog. The main thread drains stdout WITHOUT
// holding this lock, so the watchdog can always acquire it to kill on timeout;
// killing closes stdout and unblocks the drain below.
let child = std::sync::Arc::new(std::sync::Mutex::new(child));
let (done_tx, done_rx) = mpsc::channel::<()>();
let watchdog = {
let child = child.clone();
std::thread::spawn(move || -> bool {
let wait = deadline.saturating_duration_since(Instant::now());
match done_rx.recv_timeout(wait) {
Ok(()) | Err(RecvTimeoutError::Disconnected) => false,
Err(RecvTimeoutError::Timeout) => {
if let Ok(mut c) = child.lock() {
let _ = c.kill();
}
true
}
}
})
};
let mut out = Vec::new();
let read_result = stdout.read_to_end(&mut out);
// The drain has returned (child exited or was killed), so taking the lock here
// cannot deadlock against the watchdog.
let status = child
.lock()
.expect("git child mutex poisoned")
.wait()
.context("git wait failed")?;
let _ = done_tx.send(());
let killed = watchdog.join().unwrap_or(false);
let err = err_reader.join().unwrap_or_default();
let _ = writer.join();
read_result.context("failed to read git stdout")?;
if killed && !status.success() {
return Err(crate::git::smart_http::GitServiceTimeout.into());
}
Ok((status, out, err))
}
/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics.
#[cfg(not(unix))]
pub(crate) fn run_bounded_git(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<Vec<u8>> {
let label = args.first().copied().unwrap_or("git");
let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?;
if !status.success() {
anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err));
}
Ok(out)
}
/// Fail closed unless every ref ultimately resolves to a commit (a ref pointing
/// directly at a blob or tree, or an annotated tag — even a nested one — of such
@@ -21,19 +331,15 @@ use std::path::Path;
/// Full peeling is why this is not `for-each-ref %(*objecttype)`, which
/// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a-
/// commit as a non-commit.
fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
let refs = std::process::Command::new("git")
.args(["for-each-ref", "--format=%(refname)"])
.current_dir(repo_path)
.output()
.context("git for-each-ref failed")?;
if !refs.status.success() {
anyhow::bail!(
"git for-each-ref failed: {}",
String::from_utf8_lossy(&refs.stderr)
);
}
let refs_stdout = String::from_utf8_lossy(&refs.stdout);
fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> {
let refs_out = run_bounded_git(
git_bin,
&["for-each-ref", "--format=%(refname)"],
repo_path,
b"",
deadline,
)?;
let refs_stdout = String::from_utf8_lossy(&refs_out);
let refnames: Vec<&str> = refs_stdout
.lines()
.map(str::trim)
@@ -43,59 +349,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
return Ok(());
}
// Peel every ref in one `git cat-file --batch-check` pass: one
// `<refname>^{}` query per line, one output line per input line, in order.
// The stdin write runs on a separate thread so this thread can drain stdout
// concurrently. cat-file echoes the full query on a `<query> missing` line,
// so output scales with refname length (not a fixed size per ref); writing
// all of stdin before reading any stdout would deadlock both pipes once the
// child's stdout buffer fills. Dropping `stdin` at the end of the closure
// sends EOF.
// Peel every ref in one `git cat-file --batch-check` pass: one `<refname>^{}`
// query per line, one output line per input line, in order. cat-file echoes the
// full query on a `<query> missing` line, so output scales with refname length;
// run_bounded_git drains stdout concurrently with the stdin write, so the pipe
// cannot deadlock, and the whole peel is bounded by the shared walk deadline.
let queries = refnames
.iter()
.map(|r| format!("{r}^{{}}"))
.collect::<Vec<_>>()
.join("\n");
use std::io::Write;
let mut child = std::process::Command::new("git")
.args(["cat-file", "--batch-check=%(objecttype)"])
.current_dir(repo_path)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.context("failed to spawn git cat-file")?;
// Feed stdin on a writer thread so this thread can drain stdout via
// wait_with_output concurrently; a None handle (the pipe vanished) becomes a
// broken-pipe write error. wait_with_output reaps the child unconditionally
// before any error is surfaced, so no path drops it unwaited (#53), and the
// writer is joined only after the drain so the join cannot deadlock.
let writer = child
.stdin
.take()
.map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes())));
let peel_result = child.wait_with_output();
let write_result = match writer {
Some(handle) => handle
.join()
.map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?,
None => Err(std::io::Error::new(
std::io::ErrorKind::BrokenPipe,
"git cat-file stdin unavailable",
)),
};
// Surface a write error only if the process didn't already fail with a
// clearer status.
let peel = peel_result.context("git cat-file failed")?;
if !peel.status.success() {
anyhow::bail!(
"git cat-file --batch-check failed: {}",
String::from_utf8_lossy(&peel.stderr)
);
}
write_result.context("failed to write to git cat-file stdin")?;
let peel_out = run_bounded_git(
git_bin,
&["cat-file", "--batch-check=%(objecttype)"],
repo_path,
queries.as_bytes(),
deadline,
)?;
let peel_stdout = String::from_utf8_lossy(&peel.stdout);
let peel_stdout = String::from_utf8_lossy(&peel_out);
let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect();
// A short read means at least one ref went unclassified — fail closed.
if types.len() != refnames.len() {
@@ -147,47 +419,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
/// Fails closed: if commit enumeration or any tree walk fails, returns an error so
/// the caller aborts the serve/pin rather than producing a partial (under-withheld)
/// set.
fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
assert_all_refs_are_commits(repo_path)?;
fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result<Vec<(String, String)>> {
// One deadline spans the whole walk (the ref check, the HEAD probe, rev-list,
// and every per-commit ls-tree), so a slow or hung walk is bounded as a unit
// rather than granting each git child a fresh timeout.
let deadline = Instant::now() + timeout;
assert_all_refs_are_commits(repo_path, git_bin, deadline)?;
// Enumerate every reachable commit, not just ref tips. `--all` walks all refs;
// append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no
// ref) is still classified. When HEAD does not resolve (unborn branch on an
// empty repo) `--all` alone yields nothing, which is correct — no objects exist.
let head = store::head_commit(repo_path).context("resolve HEAD failed")?;
// empty repo) `--all` alone yields nothing, which is correct: no objects exist.
// The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means
// HEAD resolves), replacing the previously unbounded `store::head_commit` child.
let head_resolves = run_bounded_git(
git_bin,
&["rev-parse", "--verify", "HEAD"],
repo_path,
b"",
deadline,
)
.is_ok();
let mut rev_args = vec!["rev-list", "--all"];
if head.is_some() {
if head_resolves {
rev_args.push("HEAD");
}
let commits = std::process::Command::new("git")
.args(&rev_args)
.current_dir(repo_path)
.output()
.context("git rev-list --all failed")?;
if !commits.status.success() {
anyhow::bail!(
"git rev-list --all failed: {}",
String::from_utf8_lossy(&commits.stderr)
);
}
let commits_stdout = String::from_utf8_lossy(&commits.stdout);
let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?;
let commits_stdout = String::from_utf8_lossy(&commits_out);
let mut out: HashSet<(String, String)> = HashSet::new();
for commit in commits_stdout.lines() {
let commit = commit.trim();
if commit.is_empty() {
continue;
}
let listing = std::process::Command::new("git")
.args(["ls-tree", "-rz", commit])
.current_dir(repo_path)
.output()
.context("git ls-tree -rz failed")?;
if !listing.status.success() {
anyhow::bail!(
"git ls-tree -rz {commit} failed: {}",
String::from_utf8_lossy(&listing.stderr)
);
}
let listing_out = run_bounded_git(
git_bin,
&["ls-tree", "-rz", commit],
repo_path,
b"",
deadline,
)?;
// `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r`
// C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes
// "secret/caf\303\251.txt"), and that quoted literal would not match a
@@ -198,7 +469,7 @@ fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
// path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string
// would no longer match its deny rule — the same under-withholding class, one
// layer down. Fail closed instead so the caller aborts rather than leaks.
let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else {
let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else {
anyhow::bail!(
"git ls-tree -rz {commit} returned a non-UTF-8 path; \
refusing to produce a partial (under-withheld) set"
@@ -229,6 +500,7 @@ fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
///
/// The whole-repo "/" gate is handled by the caller before this function runs:
/// if "/" denies, the caller gets a 404 and never reaches the filtered serve.
#[cfg(test)]
pub fn withheld_blob_oids(
repo_path: &Path,
rules: &[VisibilityRule],
@@ -236,7 +508,33 @@ pub fn withheld_blob_oids(
owner_did: &str,
caller: Option<&str>,
) -> Result<HashSet<String>> {
let pairs = blob_paths(repo_path)?;
withheld_blob_oids_bounded(
repo_path,
"git",
WALK_TIMEOUT,
rules,
is_public,
owner_did,
caller,
)
}
/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served
/// handlers call this with the operator-configured git binary and
/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same
/// budget as the other served-git ops and a fake `git` can drive its teardown in
/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the
/// classification tests that run against real git.
pub fn withheld_blob_oids_bounded(
repo_path: &Path,
git_bin: &str,
timeout: Duration,
rules: &[VisibilityRule],
is_public: bool,
owner_did: &str,
caller: Option<&str>,
) -> Result<HashSet<String>> {
let pairs = blob_paths(repo_path, git_bin, timeout)?;
Ok(withheld_from_pairs(
&pairs, rules, is_public, owner_did, caller,
))
@@ -304,6 +602,7 @@ pub fn replicable_objects(all: Vec<String>, withheld: &HashSet<String>) -> Vec<S
/// at an allowed path is included even when also denied elsewhere (its content
/// is public elsewhere). A dangling blob is absent from the reachable walk, so
/// it is never in this set and the fail-closed filter drops it (#99).
#[cfg(test)]
pub fn replicable_blob_set(
repo_path: &Path,
rules: &[VisibilityRule],
@@ -313,6 +612,21 @@ pub fn replicable_blob_set(
allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None)
}
/// [`replicable_blob_set`] with an injectable `git_bin` and walk `timeout`, for the
/// fail-closed full-scan pin path on the receive-pack side.
pub fn replicable_blob_set_bounded(
repo_path: &Path,
git_bin: &str,
timeout: Duration,
rules: &[VisibilityRule],
is_public: bool,
owner_did: &str,
) -> Result<HashSet<String>> {
allowed_blob_set_for_caller_bounded(
repo_path, git_bin, timeout, rules, is_public, owner_did, None,
)
}
/// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The
/// caller-aware generalization of `replicable_blob_set` (which is the anonymous
/// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against
@@ -325,6 +639,7 @@ pub fn replicable_blob_set(
/// elsewhere (its content is readable to this caller elsewhere). Trees and
/// commits are NOT included here; the caller decides per object type whether
/// the allow-set applies (it does not for trees/commits — KTD3).
#[cfg(test)]
pub fn allowed_blob_set_for_caller(
repo_path: &Path,
rules: &[VisibilityRule],
@@ -332,7 +647,29 @@ pub fn allowed_blob_set_for_caller(
owner_did: &str,
caller: Option<&str>,
) -> Result<HashSet<String>> {
let pairs = blob_paths(repo_path)?;
allowed_blob_set_for_caller_bounded(
repo_path,
"git",
WALK_TIMEOUT,
rules,
is_public,
owner_did,
caller,
)
}
/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`,
/// for the `GET /ipfs/{cid}` gate.
pub fn allowed_blob_set_for_caller_bounded(
repo_path: &Path,
git_bin: &str,
timeout: Duration,
rules: &[VisibilityRule],
is_public: bool,
owner_did: &str,
caller: Option<&str>,
) -> Result<HashSet<String>> {
let pairs = blob_paths(repo_path, git_bin, timeout)?;
let mut allowed = HashSet::new();
for (oid, path) in &pairs {
if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow {
@@ -365,14 +702,28 @@ pub fn replicable_objects_fail_closed(
/// owner plus any reader DID that `visibility_check` Allows at some path the
/// blob appears at. Least-privilege: a reader of one private subtree is not a
/// recipient of a blob that only lives in another.
#[cfg(test)]
pub fn withheld_blob_recipients(
repo_path: &Path,
rules: &[VisibilityRule],
is_public: bool,
owner_did: &str,
) -> Result<HashMap<String, BTreeSet<String>>> {
withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did)
}
/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for
/// the receive-pack encrypt-then-pin path.
pub fn withheld_blob_recipients_bounded(
repo_path: &Path,
git_bin: &str,
timeout: Duration,
rules: &[VisibilityRule],
is_public: bool,
owner_did: &str,
) -> Result<HashMap<String, BTreeSet<String>>> {
// One history walk feeds both the withheld set and the recipient mapping.
let pairs = blob_paths(repo_path)?;
let pairs = blob_paths(repo_path, git_bin, timeout)?;
let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None);
if withheld.is_empty() {
return Ok(HashMap::new());
@@ -402,6 +753,300 @@ pub fn withheld_blob_recipients(
#[cfg(test)]
mod tests {
use super::*;
/// Write an executable fake `git` shell script into `dir` and return its path,
/// so a test can drive the walk's process-group teardown without a real git and
/// without mutating the process-global PATH (the crate's only injection seam).
#[cfg(unix)]
fn write_fake_git(dir: &Path, body: &str) -> String {
use std::os::unix::fs::PermissionsExt;
let p = dir.join("fakegit");
std::fs::write(&p, body).unwrap();
let mut perm = std::fs::metadata(&p).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&p, perm).unwrap();
p.to_str().unwrap().to_string()
}
/// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so
/// a hung git child cannot pin the caller's permit past the deadline. A fake git
/// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout`
/// within the watchdog budget (not block for the child's lifetime), and the
/// child's process group must be reaped (its recorded leader PID gone). Every
/// caller (upload-pack serve, receive-pack replication) funnels through
/// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize
/// the watchdog SIGTERM and this hangs past the recv budget (RED).
#[cfg(unix)]
#[test]
fn blob_paths_times_out_and_reaps_a_hung_walk() {
use std::time::Duration;
let tmp = TempDir::new().unwrap();
// Fast on every stage except rev-list, which records its own (group-leader)
// PID and then hangs. `sleep 30` bounds the worst case if the watchdog is
// ever broken, so a regression cannot wedge the suite for 300s.
let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n";
let git_bin = write_fake_git(tmp.path(), body);
// Run the walk on a thread with a short budget; the recv_timeout succeeding
// is itself proof the walk did not block on the hung child.
let (tx, rx) = mpsc::channel();
let path = tmp.path().to_path_buf();
std::thread::spawn(move || {
let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200)));
});
let result = rx.recv_timeout(Duration::from_secs(10)).expect(
"blob_paths must return within the watchdog budget, not hang on a stuck git child",
);
let err = result.expect_err("a hung rev-list must abort the walk with an error");
assert!(
err.downcast_ref::<crate::git::smart_http::GitServiceTimeout>()
.is_some(),
// `{err:#}` prints the whole anyhow chain. Plain `{err}` shows only the top
// context ("failed to spawn git for-each-ref") and drops the underlying io
// error, which left a real beta-lane CI failure undiagnosable.
"a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err:#}"
);
// The recorded process-group leader must be gone: the watchdog reaps the
// whole group before blob_paths returns, so no orphaned git lingers.
let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid"))
.expect("the fake git must have recorded its rev-list PID")
.trim()
.parse()
.expect("recorded PID must parse");
let mut gone = false;
for _ in 0..200 {
// SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means
// the process is gone. Borrows no Rust memory.
if unsafe { libc::kill(pid, 0) } != 0 {
gone = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
assert!(
gone,
"the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts"
);
}
/// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is
/// never reported as a timeout even when the watchdog fires, so a walk finishing
/// right at its deadline is not a spurious 504. The fake only exits when signalled
/// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always
/// reaches its kill path (killed == true) yet the child's status is success.
/// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED).
#[cfg(unix)]
#[test]
fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() {
use std::time::{Duration, Instant};
let tmp = TempDir::new().unwrap();
let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n";
let git_bin = write_fake_git(tmp.path(), body);
let out = run_bounded_git(
&git_bin,
&["rev-list"],
tmp.path(),
b"",
Instant::now() + Duration::from_millis(100),
);
assert!(
out.is_ok(),
"a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}"
);
}
/// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped
/// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its
/// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still
/// return (via SIGKILL at the grace step) with a timeout error and the group must
/// be gone. (A truly uninterruptible D-state child, which no signal can reap, is
/// the documented residual this teardown, like the async twin, cannot cover.)
#[cfg(unix)]
#[test]
fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() {
use std::time::{Duration, Instant};
let tmp = TempDir::new().unwrap();
let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n";
let git_bin = write_fake_git(tmp.path(), body);
let (tx, rx) = std::sync::mpsc::channel();
let path = tmp.path().to_path_buf();
std::thread::spawn(move || {
let _ = tx.send(run_bounded_git(
&git_bin,
&["rev-list"],
&path,
b"",
Instant::now() + Duration::from_millis(100),
));
});
let out = rx
.recv_timeout(Duration::from_secs(10))
.expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child");
assert!(
out.is_err(),
"a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}"
);
let pid: i32 = std::fs::read_to_string(tmp.path().join("pid"))
.unwrap()
.trim()
.parse()
.unwrap();
let mut gone = false;
for _ in 0..300 {
if unsafe { libc::kill(pid, 0) } != 0 {
gone = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
assert!(
gone,
"the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running"
);
}
/// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must
/// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The
/// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its
/// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores
/// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the
/// group, the leader exits, its stdout closes, the main drain unblocks, and the
/// leader is reaped — the exact window a `reaped`-gated watchdog stands down in,
/// before escalating to SIGKILL. The descendant must be dead when run_bounded_git
/// returns; a teardown that stands down on leader-reap leaves it running (RED).
#[cfg(unix)]
#[test]
fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() {
use std::time::{Duration, Instant};
let tmp = TempDir::new().unwrap();
// Both loops are bounded (~30s) so a broken teardown cannot leak a permanent
// orphan or wedge the suite; the assertion fires well before then.
let body = "#!/bin/sh\n\
case \"$1\" in\n\
rev-list)\n\
sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\
trap 'exit 0' TERM\n\
i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\
*) : ;;\n\
esac\n";
let git_bin = write_fake_git(tmp.path(), body);
let (tx, rx) = std::sync::mpsc::channel();
let path = tmp.path().to_path_buf();
std::thread::spawn(move || {
let _ = tx.send(run_bounded_git(
&git_bin,
&["rev-list"],
&path,
b"",
Instant::now() + Duration::from_millis(100),
));
});
let _ = rx
.recv_timeout(Duration::from_secs(10))
.expect("run_bounded_git must return within the watchdog budget");
// Wait for the descendant to record its OWN pid, then assert it is gone.
let desc_pid_path = tmp.path().join("desc.pid");
let mut desc: Option<i32> = None;
for _ in 0..200 {
if let Some(p) = std::fs::read_to_string(&desc_pid_path)
.ok()
.and_then(|s| s.trim().parse::<i32>().ok())
{
desc = Some(p);
break;
}
std::thread::sleep(Duration::from_millis(10));
}
let desc = desc.expect("the fake leader must have spawned and recorded a descendant");
let mut gone = false;
for _ in 0..300 {
if unsafe { libc::kill(desc, 0) } != 0 {
gone = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
// Kill it regardless so a RED run leaks no orphan.
unsafe { libc::kill(desc, libc::SIGKILL) };
assert!(
gone,
"a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned"
);
}
/// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own
/// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code
/// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down
/// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()`
/// then blocks on the still-alive leader — pinning the walk thread and its read/
/// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS.
/// This is distinct from the descendant case above: there the leader sleeps until
/// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of
/// the deadline. The fix keeps the watchdog armed until the child is actually
/// reaped, so the deadline SIGTERM still fires and the call returns within budget.
/// A pre-fix build blocks on `child.wait()` past the recv budget (RED).
#[cfg(unix)]
#[test]
fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() {
use std::time::{Duration, Instant};
let tmp = TempDir::new().unwrap();
// rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs
// immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst
// case so a RED run cannot wedge the suite; the recv budget fires first.
//
// The deadline below is deliberately far larger than the work it bounds: the
// watchdog tears the whole group down at the deadline, so a deadline tight
// enough to race `/bin/sh` reaching its first statement kills the leader
// BEFORE it records its pid, and the `leader.pid` read below then fails with
// NotFound. That is a harness race, not a product failure, and it is exactly
// what a loaded CI runner reproduced (test (beta), 2026-08-04). Sizing it at
// 2s keeps the property under test intact, since the pre-fix build blocks on
// `child.wait()` for the child's full `sleep 30` and still trips the 10s recv
// budget below (RED), while leaving a 20x margin over the ~100ms that a
// healthy start actually needs.
let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n";
let git_bin = write_fake_git(tmp.path(), body);
let (tx, rx) = std::sync::mpsc::channel();
let path = tmp.path().to_path_buf();
std::thread::spawn(move || {
let _ = tx.send(run_bounded_git(
&git_bin,
&["rev-list"],
&path,
b"",
Instant::now() + Duration::from_secs(2),
));
});
let out = rx.recv_timeout(Duration::from_secs(10)).expect(
"run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()",
);
assert!(
out.is_err(),
"a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}"
);
let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid"))
.unwrap()
.trim()
.parse()
.unwrap();
let mut gone = false;
for _ in 0..300 {
if unsafe { libc::kill(pid, 0) } != 0 {
gone = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
// Kill it regardless so a RED run leaks no orphan.
unsafe { libc::kill(pid, libc::SIGKILL) };
assert!(
gone,
"the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running"
);
}
use crate::db::VisibilityMode;
use chrono::Utc;
use std::process::Command;
@@ -735,7 +1380,12 @@ mod tests {
String::from_utf8_lossy(&out.stdout).trim().to_string()
};
let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap();
let all_blobs = crate::git::push_delta::all_blob_oids(
&work,
"git",
std::time::Instant::now() + std::time::Duration::from_secs(600),
)
.unwrap();
assert!(
all_blobs.contains(&dangling_oid),
"precondition: the dangling blob is in the all-objects universe"
File diff suppressed because it is too large. Load diff
+142 -10
View File
@@ -74,6 +74,13 @@ async fn main() -> Result<()> {
// bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS.
bootstrap::merge_seeds(&mut config);
// Fail fast on config combinations that are individually in-range but jointly
// unsafe — notably a DB pool too small for the concurrent-write cap, which
// would let a push burst starve every other DB path (#174 F1).
config
.validate()
.map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?;
if !config.public_read {
warn!(
"GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha"
@@ -378,7 +385,81 @@ async fn main() -> Result<()> {
sync_trigger_rate_limiter,
peer_write_rate_limiter,
shutdown_tx: shutdown_tx.clone(),
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)),
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(
config.max_concurrent_git_pushes,
)),
// Anon receive-pack advertisements get their OWN pool, same size as the
// write pool but disjoint, so filling it (which takes many source IPs, each
// capped by git_push_advert_per_caller) never occupies a permit the
// authenticated POST needs (#174).
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(
config.max_concurrent_git_pushes,
)),
// Bounds concurrent detached post-push encryption walks, sized from the push
// pool (no separate knob — Q1): completed pushes cannot outnumber active
// encryption walks past this (#174 P1-e).
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(
config.max_concurrent_git_pushes,
)),
// Bounds how many post-push pin loops run concurrently across all repos (#174 F6),
// independent of the per-repo encrypt-task coalescing below. Not a bound on the
// MB-scale object-id lists themselves: parked tasks still hold theirs (see the
// field doc on AppState::pin_semaphore).
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)),
// Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher
// cannot grow the outstanding parked-waiter set past one task per repo (#174
// P2-2). No knob: it is a natural cap (one entry per distinct repo), not a
// sized pool.
encrypt_inflight: crate::state::EncryptInflight::new(),
// Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg
// advisory lock so a disconnected push's still-reaping git group can't be raced
// by a second same-node push. The map is naturally capped (one entry per contended
// repo, freed when unreferenced); the sized knob is how many pushes may PARK on
// one repo, since each parked push holds a fully buffered pack.
repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters),
git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
config.max_concurrent_reads_per_caller,
),
// Per-source cap on the receive-pack advertisement, sized to an eighth of the
// write pool (min 1): one resolved client key (rate_limit::client_key) can hold
// at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore,
// disjoint from the write pool), so saturating that pool takes ~8 distinct keys
// (#174). That bounds an IPv4 or single-address caller; a caller controlling many
// addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since
// client_key uses the full IP with no prefix folding. Narrowing the keying is a
// deferred design call, not something these caps claim to solve. Sized off the
// write pool only because the advert pool is created at the same size; an advert
// flood cannot touch a write permit.
git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
rate_limit::per_source_push_cap(config.max_concurrent_git_pushes),
),
// Per-source cap on the authenticated receive-pack POST, sized like the advert
// cap: one resolved client key can hold at most this many write-pool slots, so
// monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as
// above: keys are full IPs, so a caller with many addresses has many caps.
git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
rate_limit::per_source_push_cap(config.max_concurrent_git_pushes),
),
// Bounds concurrent /ipfs visibility walks — a distinct public cost center, so
// its own pool + per-source sub-cap + per-IP rate limiter, never a git pool
// (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15).
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(
config.max_concurrent_ipfs_walks,
)),
git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
config.ipfs_walk_per_source,
),
ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded(
config.ipfs_rate_limit,
std::time::Duration::from_secs(3600),
200_000,
),
git_bin: "git".to_string(),
};
if config.ipfs_rate_limit == 0 {
tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled");
}
// Periodic peer-count poll for the metrics gauge. If p2p is disabled
// we still set the gauge to 0 so dashboards don't show "no data".
@@ -408,22 +489,14 @@ async fn main() -> Result<()> {
// Periodic cleanup of expired rate limit entries + consumed-proof ledger
{
let rl = state.rate_limiter.clone();
let create_ip_rl = state.create_ip_rate_limiter.clone();
let push_rl = state.push_rate_limiter.clone();
let sync_trigger_rl = state.sync_trigger_rate_limiter.clone();
let peer_write_rl = state.peer_write_rate_limiter.clone();
let sweep_state = state.clone();
let db = state.db.clone();
let mut shutdown_rx = state.subscribe_shutdown();
tokio::spawn(async move {
loop {
tokio::select! {
_ = tokio::time::sleep(std::time::Duration::from_secs(300)) => {
rl.cleanup().await;
create_ip_rl.cleanup().await;
push_rl.cleanup().await;
sync_trigger_rl.cleanup().await;
peer_write_rl.cleanup().await;
sweep_rate_limiters(&sweep_state).await;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
@@ -959,6 +1032,65 @@ async fn gossip_task(
}
}
#[cfg(test)]
mod rate_limiter_sweep_tests {
use crate::rate_limit::RateLimiter;
use std::time::Duration;
// Every per-key limiter the router mounts must be swept by the periodic
// task, the `/ipfs` one included: a limiter left out keeps expired keys
// until its map fills and the inline capacity sweep fires. Fails on the
// pre-fix sweeper, which skipped `ipfs_rate_limiter`.
#[tokio::test]
async fn sweep_evicts_expired_keys_from_every_limiter() {
let window = Duration::from_millis(30);
let mut state = crate::test_support::test_state_lazy();
state.rate_limiter = RateLimiter::new(10, window);
state.create_ip_rate_limiter = RateLimiter::new(10, window);
state.push_rate_limiter = RateLimiter::new(10, window);
state.sync_trigger_rate_limiter = RateLimiter::new(10, window);
state.peer_write_rate_limiter = RateLimiter::new(10, window);
state.ipfs_rate_limiter = RateLimiter::new(10, window);
let limiters = |s: &crate::state::AppState| {
[
s.rate_limiter.clone(),
s.create_ip_rate_limiter.clone(),
s.push_rate_limiter.clone(),
s.sync_trigger_rate_limiter.clone(),
s.peer_write_rate_limiter.clone(),
s.ipfs_rate_limiter.clone(),
]
};
for l in limiters(&state) {
assert!(l.check("1.2.3.4").await);
assert_eq!(l.tracked_keys().await, 1);
}
tokio::time::sleep(window * 3).await;
super::sweep_rate_limiters(&state).await;
for (i, l) in limiters(&state).into_iter().enumerate() {
assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept");
}
}
}
/// Evict expired entries from every per-key rate limiter on the state.
///
/// Named and driven off `AppState` so the periodic sweeper stays in step with
/// the limiters the router actually mounts: adding a limiter field and
/// forgetting it here leaves its keys pinned until the map hits `max_keys` and
/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way).
async fn sweep_rate_limiters(state: &AppState) {
state.rate_limiter.cleanup().await;
state.create_ip_rate_limiter.cleanup().await;
state.push_rate_limiter.cleanup().await;
state.sync_trigger_rate_limiter.cleanup().await;
state.peer_write_rate_limiter.cleanup().await;
state.ipfs_rate_limiter.cleanup().await;
}
/// Build the shared node HTTP client used for every outbound fan-out (sync
/// trigger, profile/repo fetches, gossip announce + peer pings).
///
+742 -9
View File
@@ -7,6 +7,7 @@
//! no-op, so nodes without Pinata backing work fine.
use anyhow::Result;
use std::time::{Duration, Instant};
/// Pin a single git object's raw bytes on Pinata (v3 API).
///
@@ -72,25 +73,77 @@ pub async fn pin_object(
/// `object_list` is the already-withheld-filtered OID set to pin: the caller
/// applies `visibility_pack::replicable_objects` on the delta path or the
/// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is
/// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors
/// this shape — change both in lockstep. Objects already recorded with a
/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly
/// pinned object.
/// still needed to read each object's bytes, and `git_bin` names the binary those
/// reads run: the production caller passes the literal `"git"`, and a test passes a
/// fake so the loop's own bound can be driven with a git that never answers.
/// Objects already recorded with a `pinata_cid` are skipped. Returns
/// `(sha_hex, cid)` pairs for each newly pinned object.
///
/// # What `batch_budget` does and does not bound
///
/// The loop runs under a `pin_semaphore` permit and that pool defers rather than
/// sheds, so the hold has to be bounded by something other than the pusher's object
/// count. Two things here are:
///
/// - this loop's own wall-clock: the deadline is taken once at loop start and
/// checked at the top of every iteration, so no object's work begins with less
/// than the read floor left. It is a gate, not a hard ceiling, since a started
/// iteration still runs to completion;
/// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the
/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_pinata_cid`
/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL
/// process-group teardown, so a hung `git cat-file` costs this batch its remaining
/// budget plus one watchdog teardown instead of holding the permit for the child's
/// whole lifetime and blocking a runtime worker while it does.
///
/// So the LOOP's hold is bounded by roughly `batch_budget`, plus one watchdog
/// teardown and one upload (the shared client's whole-request timeout bounds the
/// upload; `pin_object` takes no per-request override). The PERMIT's hold is NOT
/// bounded by any of this: `api::repos` acquires the permit and then re-derives the
/// object list with `pinata_object_list_for_refs` BEFORE this function is entered,
/// and that walk carries no aggregate deadline. The DB round-trips
/// (`has_pinata_cid`, `record_pinata_cid`) are untimed inside the budgeted region
/// too.
///
/// The twin in `ipfs_pin.rs` shares the budget gate and the bounded read with this
/// loop, so both are at parity again. Change them in lockstep: the skip-if-pinned
/// check, the fault arms, the returned pairs, and the budget handling.
// Eight arguments, one over clippy's threshold: the two the budget adds (`git_bin`,
// `batch_budget`) are what put the read under test injection and under a deadline, and
// grouping them into a struct would only move the same values behind a name the twin in
// `ipfs_pin.rs` does not use. Same allow as the sibling call sites in `api::repos`.
#[allow(clippy::too_many_arguments)]
pub async fn pin_new_objects(
client: &reqwest::Client,
upload_url: &str,
jwt: &str,
repo_path: &std::path::Path,
git_bin: &str,
object_list: Vec<String>,
db: &crate::db::Db,
batch_budget: Duration,
) -> Vec<(String, String)> {
if jwt.is_empty() {
return vec![];
}
let deadline = Instant::now() + batch_budget;
let total = object_list.len();
let mut pinned = Vec::new();
for sha in object_list {
for (attempted, sha) in object_list.into_iter().enumerate() {
// Top of the iteration, before any of this object's work: an object is never
// started with a remainder too small to cover a bounded read's teardown. The
// gate is shared with the IPFS loop so the two cannot drift apart in how they
// report a truncated batch. Consumed as a guard only: the read below runs against
// the absolute batch deadline, and `pin_object` takes no per-request override, so
// the remainder has no other consumer here.
if crate::ipfs_pin::batch_budget_gate("Pinata", deadline, pinned.len(), total - attempted)
.is_none()
{
break;
}
match db.has_pinata_cid(&sha).await {
Ok(true) => continue,
Ok(false) => {}
@@ -100,13 +153,73 @@ pub async fn pin_new_objects(
}
}
let data = match crate::git::store::read_object(repo_path, &sha) {
Ok(Some((_kind, bytes))) => bytes,
Ok(None) => continue,
Err(e) => {
// Read raw object content, bounded and reaped, under `spawn_blocking`: this is
// synchronous blocking work (child spawn, pipe drain, watchdog join), so calling
// it from the runtime task would block a worker thread for its whole duration.
// Placement mirrors the `/ipfs` serve path and the IPFS pin loop.
//
// The read runs against the ABSOLUTE batch deadline, not against the remainder
// measured at the top of the iteration: the `has_pinata_cid` round-trip above sits
// between the two, so `Instant::now() + budget_left` would land past `deadline` by
// however long the DB took, and under a saturated pool that is the dominant term.
// A slow DB check must not push the read's own bound out.
let read_deadline = deadline;
let read_path = repo_path.to_path_buf();
let read_sha = sha.clone();
let read_git = git_bin.to_string();
let read = tokio::task::spawn_blocking(move || {
crate::git::store::read_object_bounded(&read_git, &read_path, &read_sha, read_deadline)
})
.await;
let data = match read {
Ok(Ok(Some((_kind, bytes)))) => bytes,
// A verified absence, and the only outcome that is not a fault.
Ok(Ok(None)) => continue,
// A Transient fault does NOT by itself mean the store is gone. It also covers
// a spawn or watchdog-timeout failure of the reaped child, an unaffordable
// confirming re-probe, and, because readability is judged FOR one oid, a
// single unreadable `objects/<xx>` fan-out, which is 1/256 of the store. So
// re-check store-wide before deciding what the fault costs.
Ok(Err(e @ crate::git::store::ProbeError::Transient(_))) => {
if !crate::git::store::object_store_readable_store_wide(repo_path) {
// Genuinely store-wide: every remaining object fails identically, and
// continuing would spawn one doomed bounded child per object and spend
// the batch budget reaping them.
tracing::warn!(
sha = %sha,
err = %e,
unattempted = total - attempted,
"object store unreadable while pinning to Pinata; stopping the batch"
);
break;
}
// The store still reads store-wide, so the fault is object-scoped or
// transient to this read. Breaking would forfeit a healthy store's
// remaining objects permanently: the documented recovery re-derives the
// same list and breaks at the same index.
tracing::warn!(
sha = %sha,
err = %e,
"transient fault reading git object for Pinata; the object store is \
still readable store-wide, so this costs only this object"
);
continue;
}
// The store is readable and git still failed: a corrupt object, or a
// repo-wide fault git reports immediately. Either way it is per-object work
// that stays inside the budget, and breaking would forfeit a healthy store's
// remaining objects over one bad one, permanently (a later full-scan push
// re-offers the same object and breaks in the same place).
Ok(Err(e)) => {
tracing::warn!(sha = %sha, err = %e, "failed to read git object for Pinata");
continue;
}
// A panic in the read closure leaves no evidence that the failure is
// object-scoped, so fail toward the conservative arm.
Err(e) => {
tracing::warn!(sha = %sha, err = %e, "bounded git read task failed; stopping the batch");
break;
}
};
match pin_object(client, upload_url, jwt, &sha, &data).await {
@@ -132,6 +245,626 @@ pub async fn pin_new_objects(
mod tests {
use super::*;
/// Write `n` loose blobs into a fresh bare repo and return their oids. The read
/// path shells to `git cat-file`, so the objects must genuinely exist on disk: a
/// fabricated oid would `continue` past the upload and the loop scenarios below
/// would prove nothing. Copied from `ipfs_pin.rs`'s test mod rather than shared,
/// since test mods are private.
fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec<String> {
crate::git::store::init_bare(repo_path).expect("init bare repo");
(0..n)
.map(|i| {
let mut cmd = std::process::Command::new("git");
cmd.args(["hash-object", "-w", "--stdin"])
.current_dir(repo_path)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped());
let mut child = cmd.spawn().expect("spawn git hash-object");
{
use std::io::Write;
child
.stdin
.as_mut()
.expect("stdin")
.write_all(format!("pinata loop object {i}\n").as_bytes())
.expect("write stdin");
}
let out = child.wait_with_output().expect("hash-object output");
assert!(
out.status.success(),
"git hash-object: {}",
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).trim().to_string()
})
.collect()
}
/// A live Pinata-shaped endpoint that answers each upload with a v3 `data.cid`
/// body after `delays[i]` for the i-th request it accepts (the last entry
/// repeats), counting the requests it received.
///
/// Hand rolled rather than driven with `mockito` like the `pin_object` tests
/// above: mockito has no per-response delay primitive, and the batch-budget test
/// needs uploads that are slow enough to exhaust the budget partway. Drains the
/// full request, headers plus the declared `Content-Length` body, before
/// sleeping: answering early and closing would surface as a write failure on the
/// client and turn a slow-but-healthy upload into a different failure shape.
/// Same fixture shape as `ipfs_pin.rs`'s `delaying_endpoint`.
async fn delaying_pinata_endpoint(
delays: Vec<Duration>,
requests: std::sync::Arc<std::sync::atomic::AtomicUsize>,
) -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
let mut seen = 0usize;
while let Ok((mut sock, _)) = listener.accept().await {
let delay = *delays
.get(seen)
.or_else(|| delays.last())
.unwrap_or(&Duration::ZERO);
seen += 1;
requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
tokio::spawn(async move {
let mut acc = Vec::new();
let mut buf = [0u8; 4096];
loop {
let n = match sock.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(n) => n,
};
acc.extend_from_slice(&buf[..n]);
if let Some(hdr_end) =
acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4)
{
let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase();
let len: usize = headers
.lines()
.find_map(|l| l.strip_prefix("content-length:"))
.and_then(|v| v.trim().parse().ok())
.unwrap_or(0);
if acc.len() >= hdr_end + len {
break;
}
}
}
tokio::time::sleep(delay).await;
let body = br#"{"data":{"cid":"QmPinataBatchTestCid","name":"git.bin"}}"#;
let head = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n",
body.len()
);
let _ = sock.write_all(head.as_bytes()).await;
let _ = sock.write_all(body).await;
let _ = sock.flush().await;
});
}
});
endpoint
}
/// A `tracing` sink a test can read back, so the truncation warn and its sink
/// label can be asserted on rather than assumed. Installed with `set_default`,
/// which is thread-local and scoped to the guard, so it cannot bleed into any
/// other test in the binary.
#[derive(Clone, Default)]
struct CapturedLogs(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl CapturedLogs {
fn text(&self) -> String {
String::from_utf8_lossy(&self.0.lock().unwrap()).to_string()
}
}
impl std::io::Write for CapturedLogs {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
type Writer = CapturedLogs;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_writer(logs.clone())
.with_max_level(tracing::Level::WARN)
.with_ansi(false)
.finish();
let guard = tracing::subscriber::set_default(subscriber);
(logs, guard)
}
/// Write an executable `/bin/sh` script. Copied per module rather than shared:
/// `store.rs`, `visibility_pack.rs` and `ipfs_pin.rs` each keep their own, since
/// their test mods are private and not reachable from here.
#[cfg(unix)]
fn write_script(path: &std::path::Path, body: &str) {
use std::os::unix::fs::PermissionsExt;
std::fs::write(path, body).expect("write fake git");
let mut perm = std::fs::metadata(path).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(path, perm).unwrap();
}
/// The permit-hold bound on this branch. `pin_new_objects` runs under the same
/// deferring `pin_semaphore` as the IPFS loop, so without a batch deadline the
/// hold is O(N) with N chosen by the pusher. Five objects against an endpoint
/// that takes 2s each, under a 5.5s budget, must stop partway: the batch is
/// truncated and the remainder is left unattempted with exactly one warn naming
/// how many, labelled for this sink.
///
/// The windows are deliberately loose. Four pins would need every upload to
/// answer in under 1.4s, which the endpoint's own 2s sleep forbids, and one pin
/// needs only the first upload to land inside 5.5s, so both bounds hold with
/// more than a second of slack on a loaded box.
#[sqlx::test]
async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) {
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("slow.git");
let oids = seed_loose_blobs(&repo_path, 5);
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let endpoint = delaying_pinata_endpoint(
vec![Duration::from_secs(2)],
std::sync::Arc::clone(&requests),
)
.await;
let (logs, _guard) = capture_logs();
let client = reqwest::Client::new();
let pinned = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"test-jwt",
&repo_path,
"git",
oids,
&db,
Duration::from_millis(5500),
),
)
.await
.expect("wedge guard: a 5.5s budget cannot take 30s");
assert!(
(1..=3).contains(&pinned.len()),
"the batch must stop partway, not pin all five and not stall on the first: pinned {}",
pinned.len()
);
assert_eq!(
requests.load(std::sync::atomic::Ordering::SeqCst),
pinned.len(),
"no upload may be issued for an object the budget stopped short of"
);
let text = logs.text();
let warns: Vec<&str> = text
.lines()
.filter(|l| l.contains("pin batch deadline reached"))
.collect();
assert_eq!(
warns.len(),
1,
"the deadline must be reported exactly once for the batch, not per object: {text}"
);
assert!(
warns[0].contains("Pinata"),
"the truncation warn must name this sink, not the twin's: {}",
warns[0]
);
let unattempted: usize = warns[0]
.split("unattempted=")
.nth(1)
.and_then(|s| {
s.split(|c: char| !c.is_ascii_digit())
.next()
.and_then(|d| d.parse().ok())
})
.unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}"));
assert!(
unattempted >= 1 && unattempted + pinned.len() <= 5,
"unattempted={unattempted} with {} pinned is not a partial batch of five",
pinned.len()
);
}
/// #174 F3 on this branch: the git read runs while the `pin_semaphore` permit is
/// held, so a wedged `git cat-file` used to hold that permit for as long as the
/// child lived, with no deadline and no reaping, on a path a pusher drives. With
/// the read bounded, a git that never answers costs the batch its budget plus one
/// watchdog teardown and no more.
///
/// The fake traps SIGTERM and sleeps a BOUNDED 30s, following the fixture in
/// `visibility_pack.rs`: with the deadline neutralized the read would otherwise
/// leave the blocking closure and its child alive long after the test-level
/// timeout fires, wedging the run instead of reporting a failure. The endpoint is
/// never reached, since no object's bytes are ever produced.
///
/// The batch ends on the BUDGET, not on the fault arm: the repo is a healthy bare
/// store, so the timeout's `Transient` verdict is object-scoped and the loop moves on,
/// only to find the budget spent. Capturing that warn is not decoration. `tracing`
/// caches a callsite's interest globally the first time it is hit, and a hit from a
/// thread with no subscriber caches it as never-interested for the whole binary, which
/// silently blinds the deadline tests running beside this one.
#[cfg(unix)]
#[sqlx::test]
async fn pin_new_objects_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) {
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("wedged.git");
let oids = seed_loose_blobs(&repo_path, 3);
let fake = tmp.path().join("hanging-git");
write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n");
let (logs, _guard) = capture_logs();
let client = reqwest::Client::new();
let started = std::time::Instant::now();
let pinned = tokio::time::timeout(
Duration::from_secs(25),
pin_new_objects(
&client,
"http://127.0.0.1:9",
"test-jwt",
&repo_path,
fake.to_str().unwrap(),
oids,
&db,
Duration::from_secs(2),
),
)
.await
.expect(
"a wedged git must not hold the pin permit past the batch budget: the read is \
bounded and reaped, so this cannot reach the outer timeout",
);
let elapsed = started.elapsed();
assert!(
pinned.is_empty(),
"a git that never answers cannot produce a pinned object: {pinned:?}"
);
assert!(
elapsed < Duration::from_secs(20),
"elapsed {elapsed:?} must stay inside the budget plus one watchdog teardown"
);
let text = logs.text();
assert_eq!(
text.lines()
.filter(|l| l.contains("pin batch deadline reached"))
.count(),
1,
"one wedged read must spend the whole budget and stop the batch there, exactly \
once: {text}"
);
// The child's process group must be gone once the call returns; a bounded read
// that leaves the child running has only moved the hold somewhere else.
let pid: i32 = std::fs::read_to_string(repo_path.join("pid"))
.expect("the fake git must have recorded its pid, or it was never on the read path")
.trim()
.parse()
.unwrap();
let mut gone = false;
for _ in 0..200 {
// SAFETY: kill(2) with signal 0 only probes existence; ESRCH means gone.
if unsafe { libc::kill(pid, 0) } != 0 {
gone = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
assert!(
gone,
"the reaped fake git ({pid}) must not outlive the call"
);
}
/// A `git_bin` wrapper that records every invocation's arguments and then execs the
/// real git, so a test can tell which objects the loop actually attempted. The returned
/// pin list cannot: it is empty both when the loop broke after one object and when it
/// continued past all of them. Copied from `ipfs_pin.rs`'s test mod, like the fixtures
/// above, since test mods are private.
#[cfg(unix)]
fn counting_git(dir: &std::path::Path, log: &std::path::Path) -> String {
let fake = dir.join("counting-git");
write_script(
&fake,
&format!(
"#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n",
log.display()
),
);
fake.to_str().unwrap().to_string()
}
/// How many objects the loop actually attempted, read off the invocation log.
///
/// Counts `--batch-check` invocations, not log lines and not oid occurrences: the type
/// probe carries its oid on stdin rather than in argv, so an oid appears in the log only
/// once an object has already got past its probe, and a healthy object costs two
/// invocations to a faulting one's one.
fn objects_attempted(log: &std::path::Path) -> usize {
std::fs::read_to_string(log)
.unwrap_or_default()
.lines()
.filter(|l| l.contains("--batch-check"))
.count()
}
/// The store-wide fault arm, the twin of the IPFS loop's. When the object store cannot
/// be read at all every remaining object fails identically, so continuing would spawn
/// one doomed bounded child per object and burn the batch budget on reaping them.
///
/// The fixture looks wrong and is not: with the objects LOOSE and only `objects/pack`
/// unreadable, git still resolves each object, but it prints an `error:` diagnostic
/// that the probe routes to a fault before the present/missing parse, so the read
/// reaches the fault classification and (the store being unreadable) returns
/// `Transient`, which the store-wide re-check then confirms really is store-wide.
#[cfg(unix)]
#[sqlx::test]
async fn pin_new_objects_breaks_the_batch_on_an_unreadable_store(pool: sqlx::PgPool) {
use std::os::unix::fs::PermissionsExt;
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("unreadable.git");
let oids = seed_loose_blobs(&repo_path, 5);
let log = tmp.path().join("calls.log");
let git_bin = counting_git(tmp.path(), &log);
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let endpoint =
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
let client = reqwest::Client::new();
let pack_dir = repo_path.join("objects").join("pack");
let chmod = |mode: u32| {
let mut perms = std::fs::metadata(&pack_dir).unwrap().permissions();
perms.set_mode(mode);
std::fs::set_permissions(&pack_dir, perms).unwrap();
};
chmod(0o000);
// Root bypasses permission bits, so witness the exact operation the probe performs
// and skip rather than falsely fail.
let genuinely_unreadable = std::fs::read_dir(&pack_dir).is_err();
let pinned = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"test-jwt",
&repo_path,
&git_bin,
oids,
&db,
Duration::from_secs(60),
),
)
.await
.expect("an immediately-faulting store cannot take 30s");
let attempted = objects_attempted(&log);
chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up
if genuinely_unreadable {
assert!(
pinned.is_empty(),
"nothing can be pinned through a store that cannot be read: {pinned:?}"
);
assert_eq!(
attempted, 1,
"a store-wide fault must break the batch after the first object, not spawn \
one doomed bounded child per object: {attempted} of 5 objects were read"
);
assert_eq!(
requests.load(std::sync::atomic::Ordering::SeqCst),
0,
"no upload may be issued for an object whose bytes were never read"
);
}
}
/// The must-not direction of the arm above, the twin of the IPFS loop's. One corrupt
/// loose object among healthy ones is a `Deterministic` fault (the store is readable,
/// git still fails), and the documented recovery path cannot repair it: a later
/// full-scan push re-offers the same object and would break at the same place, so
/// breaking here stops the repo replicating permanently.
///
/// Deliberately not the bad-config corruption, which is repo-wide: all five objects
/// would fault and the test would pin the store-wide case rather than the object-scoped
/// one this arm rests on.
#[cfg(unix)]
#[sqlx::test]
async fn pin_new_objects_continues_past_a_deterministic_fault(pool: sqlx::PgPool) {
use std::os::unix::fs::PermissionsExt;
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("corrupt.git");
let oids = seed_loose_blobs(&repo_path, 5);
let log = tmp.path().join("calls.log");
let git_bin = counting_git(tmp.path(), &log);
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let endpoint =
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
let client = reqwest::Client::new();
// Overwrite exactly one loose object with non-zlib garbage (0o444 by default).
let victim = repo_path
.join("objects")
.join(&oids[0][0..2])
.join(&oids[0][2..]);
assert!(victim.is_file(), "fixture must leave the blob loose");
let mut perms = std::fs::metadata(&victim).unwrap().permissions();
perms.set_mode(0o644);
std::fs::set_permissions(&victim, perms).unwrap();
std::fs::write(&victim, b"garbage not a zlib stream").unwrap();
let pinned = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"test-jwt",
&repo_path,
&git_bin,
oids,
&db,
Duration::from_secs(60),
),
)
.await
.expect("an immediate endpoint and four healthy objects cannot take 30s");
assert_eq!(
objects_attempted(&log),
5,
"an object-scoped fault must not stop the batch: every object must be read"
);
assert_eq!(
pinned.len(),
4,
"one corrupt object must cost only itself: the other four must still pin"
);
assert_eq!(
requests.load(std::sync::atomic::Ordering::SeqCst),
4,
"exactly the readable objects may reach the endpoint"
);
}
/// The normal direction, and the dedup branch driven both ways: on a healthy
/// store with a healthy endpoint and a generous budget every object pins and the
/// CID is recorded, and a second call over the same list uploads nothing because
/// `has_pinata_cid` now answers true for all of them.
#[sqlx::test]
async fn pin_new_objects_pins_every_object_then_skips_the_recorded_ones(pool: sqlx::PgPool) {
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("healthy.git");
let oids = seed_loose_blobs(&repo_path, 3);
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let endpoint =
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
let client = reqwest::Client::new();
let pinned = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"test-jwt",
&repo_path,
"git",
oids.clone(),
&db,
Duration::from_secs(60),
),
)
.await
.expect("an immediate endpoint and three healthy objects cannot take 30s");
assert_eq!(pinned.len(), 3, "every healthy object must pin: {pinned:?}");
for (i, (sha, cid)) in pinned.iter().enumerate() {
assert_eq!(sha, &oids[i], "the pairs must carry the objects' own oids");
assert_eq!(cid, "QmPinataBatchTestCid");
assert!(
db.has_pinata_cid(sha).await.unwrap(),
"a pinned object must be recorded so the next batch skips it"
);
}
assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 3);
let again = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"test-jwt",
&repo_path,
"git",
oids,
&db,
Duration::from_secs(60),
),
)
.await
.expect("a fully deduped batch cannot take 30s");
assert!(again.is_empty(), "already-recorded objects must be skipped");
assert_eq!(
requests.load(std::sync::atomic::Ordering::SeqCst),
3,
"the skip must happen before the upload, not after it"
);
}
/// The no-op configuration still short-circuits with the budgeted signature: an
/// empty JWT must return before any git child is spawned and before any request
/// is issued. The `git_bin` here records every invocation, so "git was never
/// touched" is observed rather than assumed.
#[cfg(unix)]
#[sqlx::test]
async fn pin_new_objects_with_an_empty_jwt_touches_neither_git_nor_the_endpoint(
pool: sqlx::PgPool,
) {
let db = crate::db::Db::for_testing(pool);
db.run_migrations().await.expect("migrations");
let tmp = tempfile::TempDir::new().unwrap();
let repo_path = tmp.path().join("unconfigured.git");
let oids = seed_loose_blobs(&repo_path, 2);
let log = tmp.path().join("calls.log");
let fake = tmp.path().join("counting-git");
write_script(
&fake,
&format!(
"#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n",
log.display()
),
);
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let endpoint =
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
let client = reqwest::Client::new();
let pinned = tokio::time::timeout(
Duration::from_secs(30),
pin_new_objects(
&client,
&endpoint,
"",
&repo_path,
fake.to_str().unwrap(),
oids,
&db,
Duration::from_secs(60),
),
)
.await
.expect("an unconfigured sink returns immediately");
assert!(pinned.is_empty(), "an empty JWT pins nothing");
assert!(
!log.exists(),
"no git child may be spawned when the sink is not configured"
);
assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 0);
}
#[tokio::test]
async fn test_pin_skipped_when_jwt_empty() {
let client = reqwest::Client::new();
+168
View File
@@ -121,6 +121,13 @@ impl RateLimiter {
true
}
/// Number of keys currently tracked. Tests use it to observe what a sweep
/// reclaimed; there is no production reader.
#[cfg(test)]
pub async fn tracked_keys(&self) -> usize {
self.state.lock().await.len()
}
pub async fn cleanup(&self) {
let now = Instant::now();
let mut state = self.state.lock().await;
@@ -132,6 +139,110 @@ impl RateLimiter {
}
}
/// Per-source concurrency cap derived from the write-pool size: one resolved client
/// key (see [`client_key`]) may hold at most an eighth of the pool, so saturating it
/// takes ~8 distinct keys. Real for an IPv4 or single-address caller; a caller with a
/// routed IPv6 /64 has 2^64 keys, because `client_key` returns the full address with
/// no prefix folding. Narrowing that keying is a deferred decision, so this cap is a
/// bound per key, not a bound per operator.
///
/// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0
/// would shed EVERY receive-pack advertisement and break all pushes. The floor is
/// load-bearing at the minimum write-pool size (1), which integer-divides to 0.
pub(crate) fn per_source_push_cap(max_concurrent_git_pushes: usize) -> usize {
(max_concurrent_git_pushes / 8).max(1)
}
/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which
/// caps request RATE. Each caller key may hold at most `per_caller` in-flight
/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None`
/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or
/// DID) monopolizing the served-git read pool (#174).
///
/// The key map is self-bounding: a key is removed the instant its in-flight count
/// reaches zero, so it never holds more keys than there are concurrently-active
/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert
/// backstop guarantees a key farm can never grow the map even if that invariant
/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15).
///
/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the
/// permit's `Drop` must release synchronously; the critical section holds no await.
#[derive(Clone)]
pub struct PerCallerConcurrency {
state: Arc<std::sync::Mutex<HashMap<String, usize>>>,
per_caller: usize,
max_keys: usize,
}
/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements
/// the caller's in-flight count and removes the key when it reaches zero.
pub struct PerCallerPermit {
state: Arc<std::sync::Mutex<HashMap<String, usize>>>,
key: String,
}
impl PerCallerConcurrency {
pub fn new(per_caller: usize, max_keys: usize) -> Self {
Self {
state: Arc::new(std::sync::Mutex::new(HashMap::new())),
per_caller: per_caller.max(1),
max_keys: max_keys.max(1),
}
}
/// Convenience constructor with the default key bound.
pub fn with_default_max_keys(per_caller: usize) -> Self {
Self::new(per_caller, DEFAULT_MAX_KEYS)
}
/// `Some(permit)` when the caller is under its cap and the map has room;
/// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is
/// rejected without allocating.
pub fn try_acquire(&self, key: &str) -> Option<PerCallerPermit> {
// Recover from a poisoned lock rather than panicking: the critical section
// is pure counter arithmetic and cannot itself panic, so a poisoned mutex
// would only ever come from an unrelated abort, and a slightly-off count
// self-heals as permits drop. A panic here would instead brick the limiter
// for every caller (each subsequent lock re-panics).
let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner());
match map.get_mut(key) {
Some(count) => {
if *count >= self.per_caller {
return None;
}
*count += 1;
}
None => {
if map.len() >= self.max_keys {
return None;
}
map.insert(key.to_string(), 1);
}
}
Some(PerCallerPermit {
state: self.state.clone(),
key: key.to_string(),
})
}
#[cfg(test)]
pub fn tracked_keys(&self) -> usize {
self.state.lock().unwrap().len()
}
}
impl Drop for PerCallerPermit {
fn drop(&mut self) {
let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner());
if let Some(count) = map.get_mut(&self.key) {
*count -= 1;
if *count == 0 {
map.remove(&self.key);
}
}
}
}
pub async fn rate_limit_by_did(request: Request, next: Next) -> Response {
let limiter = request.extensions().get::<RateLimiter>().cloned();
@@ -288,6 +399,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response {
mod tests {
use super::*;
#[test]
fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() {
let lim = PerCallerConcurrency::new(2, 100);
let p1 = lim.try_acquire("did:key:zA").expect("first under cap");
let p2 = lim.try_acquire("did:key:zA").expect("second under cap");
assert!(
lim.try_acquire("did:key:zA").is_none(),
"a third in-flight op for the same caller sheds (over the per-caller cap)"
);
// A DIFFERENT caller is unaffected — the cap is per-caller, not global.
let _other = lim
.try_acquire("did:key:zB")
.expect("a different caller has its own budget");
drop(p1);
assert!(
lim.try_acquire("did:key:zA").is_some(),
"freeing one in-flight slot lets the same caller back in"
);
drop(p2);
}
#[test]
fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() {
// Self-bounding: acquire+drop many distinct keys — the map never grows
// because a key is removed the instant its in-flight count hits zero.
let lim = PerCallerConcurrency::new(4, 3);
for i in 0..50 {
let _p = lim.try_acquire(&format!("k{i}"));
}
assert_eq!(
lim.tracked_keys(),
0,
"keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty"
);
// Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds
// WITHOUT growing the map past the cap (INV-15 — a rejected request never
// allocates an entry).
let held: Vec<_> = (0..3)
.map(|i| lim.try_acquire(&format!("h{i}")).unwrap())
.collect();
assert_eq!(
lim.tracked_keys(),
3,
"three distinct callers held concurrently"
);
assert!(
lim.try_acquire("h3").is_none(),
"a new key at max_keys is rejected"
);
assert_eq!(
lim.tracked_keys(),
3,
"the rejected new key did not allocate an entry (reject-before-insert)"
);
drop(held);
}
#[tokio::test]
async fn allows_within_limit() {
let limiter = RateLimiter::new(3, Duration::from_secs(60));
+11
View File
@@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router {
// identity and can apply per-repo visibility (#110); anonymous callers stay
// anonymous and still read genuinely public content. `/api/v1/ipfs/pins`
// stays unsigned — gating the pin index is tracked separately (#121).
// `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each
// request can drive a full-history git walk, so the per-IP rate limiter is the
// outermost layer (rejects a flood before the walk-admission work), mirroring the
// push/create routers. The extension MUST be attached or rate_limit_by_ip is a
// silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before.
let ipfs_limiter = rate_limit::IpRateLimiter {
limiter: state.ipfs_rate_limiter.clone(),
trust: state.push_limiter_trust,
};
let ipfs_routes = Router::new()
.route("/ipfs/{cid}", get(ipfs::get_by_cid))
.layer(middleware::from_fn(auth::optional_signature))
.layer(middleware::from_fn(rate_limit::rate_limit_by_ip))
.layer(axum::Extension(ipfs_limiter))
.merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins)));
// ── Arweave permanent anchors ──────────────────────────────────────────
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -605,7 +605,7 @@ async fn replicate_encrypted_blobs(
continue;
}
};
match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope).await {
match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope, None).await {
Ok(cid) if !cid.is_empty() => {
if cid != blob.cid {
warn!(oid = %blob.oid, expected = %blob.cid, got = %cid, "replicated envelope CID mismatch; skipping record");
+255 -10
View File
@@ -82,6 +82,26 @@ fn build_state(db: Arc<crate::db::Db>, pool: PgPool) -> AppState {
sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)),
peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
shutdown_tx: tokio::sync::watch::channel(false).0,
// Generous — no test drives the handler-level shed (git_permit is unit-tested).
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
encrypt_inflight: crate::state::EncryptInflight::new(),
repo_write_leases: crate::state::RepoWriteLeases::new(8),
git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(
8,
),
git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
// Generous — a test that drives the /ipfs walk shed overrides these directly.
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(
16,
),
ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
git_bin: "git".to_string(),
}
}
@@ -109,6 +129,25 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body
.expect("request builder")
}
/// A local endpoint whose TCP accept succeeds instantly but that never writes an
/// HTTP response, so any request against it stalls deterministically until the
/// caller's own timeout. (A non-routable address hangs only if the network
/// blackholes the SYN — a fast RST would end the stall early and make a timeout
/// test pass for the wrong reason.) The accepted sockets are parked in the
/// spawned task, which dies with the test's runtime, so the peer never sees a
/// close mid-test.
pub(crate) async fn silent_http_endpoint() -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((sock, _)) = listener.accept().await {
held.push(sock);
}
});
endpoint
}
#[cfg(test)]
mod tests {
use super::*;
@@ -187,6 +226,207 @@ mod tests {
);
}
/// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the
/// DB. The held `git_permit` acquire now sits after the per-source cap, so the
/// cheap early shed is carried by an explicit `available_permits() == 0` check at
/// the top of the handler (the held permit remains the authoritative bound further
/// down). That check is a permit-less snapshot: it spares a request's DB work once
/// the pool is ALREADY saturated, which is the case this test drives, and it does
/// not bound the DB window in general. DB-free here because an exhausted semaphore
/// sheds before any DB/disk access, so a lazy state works. Remove the early-shed block
/// from git_info_refs and this goes red (the request falls through to the DB and
/// returns something other than 503).
#[tokio::test]
async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() {
let mut state = test_state_lazy();
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
let router = Router::new()
.route(
"/{owner}/{repo}/info/refs",
axum::routing::get(crate::api::repos::git_info_refs),
)
.with_state(state);
let resp = router
.oneshot(anon_get(
"/alice/repo.git/info/refs?service=git-upload-pack",
))
.await
.unwrap();
assert_eq!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"an exhausted git semaphore must shed info/refs with 503 before touching the DB"
);
assert_eq!(
resp.headers()
.get("retry-after")
.and_then(|h| h.to_str().ok()),
Some("1"),
"the 503 shed must carry Retry-After"
);
}
/// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same
/// explicit `available_permits() == 0` early check at the top, so an ALREADY
/// exhausted semaphore must shed the request with a 503 before its DB/disk work.
/// That is the case the permit-less snapshot does deliver; it is not an admission
/// bound on the DB window. Anonymous-reachable, so no auth injection is needed.
/// Remove the early-shed block from git_upload_pack and this goes red.
#[tokio::test]
async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() {
let mut state = test_state_lazy();
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
let router = Router::new()
.route(
"/{owner}/{repo}/git-upload-pack",
axum::routing::post(crate::api::repos::git_upload_pack),
)
.with_state(state);
let req = Request::builder()
.method(Method::POST)
.uri("/alice/repo.git/git-upload-pack")
.body(Body::from(&b"0000"[..]))
.unwrap();
let resp = router.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB"
);
assert_eq!(
resp.headers()
.get("retry-after")
.and_then(|h| h.to_str().ok()),
Some("1"),
"the 503 shed must carry Retry-After"
);
}
/// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed
/// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174),
/// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk
/// work (the case the permit-less snapshot delivers, not an admission bound on the
/// DB window), while the write pool (reserved for authenticated POSTs) is left
/// free here.
/// Flip the pool selection back to the write pool, or remove the early-shed
/// block, and this goes red.
#[tokio::test]
async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() {
let mut state = test_state_lazy();
state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
let router = Router::new()
.route(
"/{owner}/{repo}/info/refs",
axum::routing::get(crate::api::repos::git_info_refs),
)
.with_state(state);
let resp = router
.oneshot(anon_get(
"/alice/repo.git/info/refs?service=git-receive-pack",
))
.await
.unwrap();
assert_eq!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB"
);
assert_eq!(
resp.headers()
.get("retry-after")
.and_then(|h| h.to_str().ok()),
Some("1"),
"the 503 shed must carry Retry-After"
);
}
/// PR3 (#62) sibling for the push path: git-receive-pack requires an
/// AuthenticatedDid extension (production: require_signature injects it), so the
/// request carries one via signed_request_as — without it the Extension
/// extractor 500s before the handler body reaches the shed. What sits at the top of
/// the handler is a permit-less `available_permits() == 0` peek, NOT the permit
/// itself: the authoritative held acquire is taken after the per-repo lease, so a
/// lease-blocked waiter pins no write slot. An ALREADY exhausted pool is the case
/// the peek delivers, so the request sheds 503 before its DB work here. Remove the
/// early-shed block from git_receive_pack and this goes red.
#[tokio::test]
async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() {
let mut state = test_state_lazy();
state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
let router = Router::new()
.route(
"/{owner}/{repo}/git-receive-pack",
axum::routing::post(crate::api::repos::git_receive_pack),
)
.with_state(state);
let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA";
let resp = router
.oneshot(signed_request_as(
owner,
Method::POST,
"/alice/repo.git/git-receive-pack",
Body::from(&b"0000"[..]),
))
.await
.unwrap();
assert_eq!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"an exhausted write pool must shed git-receive-pack with 503 before touching the DB"
);
assert_eq!(
resp.headers()
.get("retry-after")
.and_then(|h| h.to_str().ok()),
Some("1"),
"the 503 shed must carry Retry-After"
);
}
/// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an
/// authenticated push — the write pool is a separate budget. Read pool at zero,
/// write pool with capacity: the push proceeds PAST admission (it then errors on
/// the placeholder DB, but crucially it is not a 503). Route git-receive-pack
/// back to the read pool and this goes red — that is the isolation proof.
#[tokio::test]
async fn git_receive_pack_not_shed_by_exhausted_read_pool() {
let mut state = test_state_lazy();
// Read pool exhausted as if a flood of anonymous clones held every slot.
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
// Write pool keeps its default capacity from test_state_lazy.
let router = Router::new()
.route(
"/{owner}/{repo}/git-receive-pack",
axum::routing::post(crate::api::repos::git_receive_pack),
)
.with_state(state);
let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA";
let resp = router
.oneshot(signed_request_as(
owner,
Method::POST,
"/alice/repo.git/git-receive-pack",
Body::from(&b"0000"[..]),
))
.await
.unwrap();
assert_ne!(
resp.status(),
StatusCode::SERVICE_UNAVAILABLE,
"an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)"
);
}
/// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner
/// before any git work (so no on-disk repo is needed for the rejection).
#[sqlx::test]
@@ -3285,7 +3525,10 @@ mod tests {
/// the handler skips the whole repo rather than serving. Asserts no leak of the
/// withheld blob AND that even the *public* blob in that repo is withheld — the
/// latter distinguishes fail-closed-skip from normal per-blob withholding and
/// would serve 200 if the error arm wrongly proceeded.
/// would serve 200 if the error arm wrongly proceeded. The skip carries no
/// VERDICT (F2), so the response is the retryable truncation 503, not a 404
/// claiming the object is absent — never-serve-unproven and never-404-unproven
/// hold together.
#[sqlx::test]
async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) {
use crate::db::VisibilityMode;
@@ -3330,7 +3573,8 @@ mod tests {
.await
.expect("deny rule");
// Withheld secret CID under a walk error → 404, no leak.
// Withheld secret CID under a walk error → the repo is skipped without a
// verdict, so the scan is truncated (503), and nothing leaks.
let (st, body) = cid_parts(
cid_router(&state)
.oneshot(cid_anon(&secret_cid))
@@ -3340,17 +3584,17 @@ mod tests {
.await;
assert_eq!(
st,
StatusCode::NOT_FOUND,
"walk error must not serve the withheld blob"
StatusCode::SERVICE_UNAVAILABLE,
"walk error must not serve the withheld blob — the unproven skip sheds 503"
);
assert!(
!body.contains("TOP SECRET"),
"walk-error 404 must not leak the secret"
"walk-error 503 must not leak the secret"
);
// The PUBLIC blob in the same repo is also 404: the walk error fails closed
// by skipping the whole repo, not by serving. Without the fail-closed arm
// this would serve 200, so this assertion is the load-bearing discriminator.
// The PUBLIC blob in the same repo is also not served: the walk error fails
// closed by skipping the whole repo. Without the fail-closed arm this would
// serve 200, so this assertion is the load-bearing discriminator.
let (st, _) = cid_parts(
cid_router(&state)
.oneshot(cid_anon(&public_cid))
@@ -3360,8 +3604,9 @@ mod tests {
.await;
assert_eq!(
st,
StatusCode::NOT_FOUND,
"walk error fails closed: repo skipped, even the public blob is not served"
StatusCode::SERVICE_UNAVAILABLE,
"walk error fails closed: repo skipped without a verdict, even the public \
blob is not served and the scan sheds 503"
);
}
+489
View File
@@ -0,0 +1,489 @@
//! #174 U6 — INV-22 completeness guard (rung-raising).
//!
//! INV-22: a permit held per op recovers only if every path that holds it is also
//! duration-bounded and reaps the process group before releasing admission, and every
//! detached git/blocking task carries its own admission. PR #174 fixed five paths
//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together
//! those form the five-revert matrix. This guard adds the missing piece: a source-scan
//! tripwire that fails when a NEW site reintroduces the class, or when one of the five
//! gates is removed.
//!
//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same
//! file it lives in would match its own identifier literals and pass vacuously. Here
//! the scanned `src/` files never contain this file's literals, so each check is
//! load-bearing — reverting the named gate turns the assertion red.
//!
//! These are deliberately coarse structural checks, not a parser. They cannot prove a
//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and
//! not bypassed*, which is what stops the class from silently regressing.
use std::path::Path;
fn src(rel: &str) -> String {
let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel);
std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display()))
}
#[test]
fn inv22_concurrency_gates_present_and_not_bypassed() {
let repos = src("api/repos.rs");
let smart_http = src("git/smart_http.rs");
let vis = src("git/visibility_pack.rs");
// U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the
// child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise
// a child that closes stdout then hangs pins the permit past the deadline. The
// probe is defined and called, so >= 2 occurrences; reverting the fix removes both.
assert!(
vis.matches("child_terminated_without_reaping").count() >= 2,
"U1/P1-a gate missing: run_bounded_git must confirm child exit via \
child_terminated_without_reaping before signalling the watchdog"
);
// U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper
// that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned
// via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the
// timeout path already has an async context and never calls it).
assert!(
smart_http.contains("Handle::try_current"),
"U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \
TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM"
);
// U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the
// global write permit. The acquire reads `state.git_write_per_caller`; comments name
// the field without the `state.` prefix, so this targets the real acquire site.
assert!(
repos.contains("state.git_write_per_caller"),
"U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \
(state.git_write_per_caller) before the global write permit"
);
// U5 / P1-e: the detached post-push encryption walk must run through the
// admission-gated helper, which is wired to the shared encrypt pool.
assert!(
repos.contains("fn withheld_recipients_gated")
&& repos.contains("state.git_encrypt_semaphore"),
"U5/P1-e gate missing: the encryption walk must run through \
withheld_recipients_gated, which acquires git_encrypt_semaphore"
);
// P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere
// but inside withheld_recipients_gated. A second call site (count > 1) is a new
// detached git walk that skips the admission gate — exactly the class U5 closed.
assert_eq!(
repos.matches("withheld_blob_recipients_bounded").count(),
1,
"P1-e bypass: the bounded recipients walk must be invoked only inside \
withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap"
);
// U4 / P2-2: the detached post-push encryption task must be gated by the per-repo
// coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is
// bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked
// waiters (the unbounded set U4 closed); the semaphore only caps active walks.
assert!(
repos.contains("state.encrypt_inflight.try_begin"),
"U4/P2-2 gate missing: the detached post-push encryption spawn must consult \
encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)"
);
// F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own
// pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and
// the task must loop-drain them (`finish_or_take_pending`) before releasing it.
// Removing the drain reverts to the silent loss: a coalesced push's pins and
// recovery copies are absent until an unrelated later push. Scan only the
// production half of the file — the u5 tests in its `mod tests` also name the
// drain call, and matching them would make this check vacuous.
let repos_production = repos
.split("#[cfg(test)]")
.next()
.expect("split always yields a first chunk");
assert!(
repos_production.contains("finish_or_take_pending"),
"F5 gate missing: the post-push encryption task must loop-drain coalesced \
pushes via finish_or_take_pending before releasing its repo key"
);
// F4: every post-receive scan helper admits itself to the shared scan pool via
// `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a
// push burst cannot accumulate unbounded concurrent scans once the write permit
// is released. Two halves, both load-bearing: the helper body must actually
// acquire the pool (state.rs sits the helper at the end of the file, so the
// definition tail contains no other `acquire_owned` to match vacuously), and
// within each scan helper the first qualified call precedes the first
// `spawn_blocking`. Severing a call site pushes the next occurrence past the
// helper's own `spawn_blocking` (or off the end of the file), turning the
// assertion red; comments name the helper without the `crate::state::` prefix,
// so this targets the real call sites.
let state_rs = src("state.rs");
let helper_def = state_rs
.find("fn acquire_scan_permit")
.expect("F4 gate missing: state.rs no longer defines acquire_scan_permit");
assert!(
state_rs[helper_def..].contains("acquire_owned"),
"F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned"
);
let push_delta = src("git/push_delta.rs");
for (file_src, file, helper) in [
(&repos, "api/repos.rs", "fn replication_withheld_set"),
(&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"),
(
&push_delta,
"git/push_delta.rs",
"fn resolve_candidates_for_push",
),
] {
let start = file_src
.find(helper)
.unwrap_or_else(|| panic!("{file}: `{helper}` not found"));
let tail = &file_src[start..];
let acquire = tail
.find("crate::state::acquire_scan_permit(")
.unwrap_or_else(|| {
panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit")
});
let spawn = tail.find("spawn_blocking").unwrap_or_else(|| {
panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard")
});
assert!(
acquire < spawn,
"F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \
permit BEFORE dispatching the blocking git scan"
);
}
}
/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release`
/// must await `pg_advisory_unlock` while `self` still owns the pooled connection,
/// and must not mark itself `released` until that await resolves. Either shape,
/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection
/// early leaves `Drop` with `conn == None`, and setting `released = true` early
/// leaves the `Drop` backstop inert — both strand the session lock on cancellation.
///
/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the
/// connection and unlocks, so a whole-file scan would match it and read as a false
/// pass. Reverting either ordering turns this red (proven load-bearing).
#[test]
fn f4_release_keeps_conn_owned_until_unlock_resolves() {
let repo_store = src("git/repo_store.rs");
let rel_start = repo_store
.find("pub async fn release(mut self")
.expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release");
let rel_end = repo_store[rel_start..]
.find("impl Drop for RepoWriteGuard")
.map(|off| rel_start + off)
.expect("F4 gate: release fn / Drop impl markers moved — update this guard");
let release_body = &repo_store[rel_start..rel_end];
let unlock = release_body
.find("pg_advisory_unlock")
.expect("F4 gate: release must still issue pg_advisory_unlock");
let before_unlock = &release_body[..unlock];
// (a) the connection must still be owned by `self` at the unlock await.
assert!(
!before_unlock.contains("self.conn.take()"),
"F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \
pg_advisory_unlock. A cancellation during the unlock await then strands the \
session advisory lock (Drop sees conn == None and skips its backstop). \
Unlock through the still-owned connection instead."
);
// (b) `released` must not be set before the unlock await — the other
// reintroduction shape a single-reorder check on (a) alone is blind to.
assert!(
!before_unlock.contains("released = true"),
"F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \
pg_advisory_unlock. A cancellation during the await then leaves the Drop \
backstop inert (it early-returns on released). Set released only AFTER the \
unlock await resolves."
);
}
/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires
/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial
/// metadata queries, and the per-repo loop's first budget gate runs only later. So
/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the
/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot
/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs`
/// (the `mod tests` half names the same calls in its own harness and would make the
/// check vacuous) and asserts each query call is immediately preceded by a
/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven
/// load-bearing).
#[test]
fn f6_ipfs_metadata_queries_are_deadline_wrapped() {
let ipfs = src("api/ipfs.rs");
let production = ipfs
.split("#[cfg(test)]")
.next()
.expect("split always yields a first chunk");
for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] {
assert_eq!(
production.matches(call).count(),
1,
"F6 guard stale: `{call}` must appear exactly once in the production half \
of api/ipfs.rs (the deadline-wrapped handler call) — update this guard"
);
let idx = production
.find(call)
.unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`"));
// The wrapper opens a few lines above the call (match tokio::time::timeout( ...
// remaining budget ..., <query> )); a 240-char lookback covers it without
// reaching the previous statement.
let window = &production[idx.saturating_sub(240)..idx];
assert!(
window.contains("tokio::time::timeout("),
"F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \
clamped to the remaining request budget. An unwrapped await pins the held \
walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS."
);
}
}
/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue
/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the
/// worker once a pin slot frees — it must NOT move the full per-push object list into
/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes
/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so
/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is
/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained
/// data, not the task count.
///
/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the
/// `mod tests` half names the same identifiers in its own harness and would make the
/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form —
/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red.
/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata
/// pin permit is acquired, so the object list is materialized only inside the pin-bounded
/// section and a parked task holds O(ref tuples).
#[test]
fn f2_pinata_enqueues_refs_not_retained_object_lists() {
let repos = src("api/repos.rs");
let production = repos
.split("#[cfg(test)]")
.next()
.expect("split always yields a first chunk");
// (a) the retained-list form must be gone from production.
assert!(
!production.contains("object_list_pinata"),
"F2/KTD-3 regression: the Pinata task retains a full per-push object list \
(`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \
tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)."
);
// (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the
// Pinata pin-admission clone so the window is the Pinata task, not the sibling
// IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives).
let anchor = production
.find("let pin_sem_pinata")
.expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved");
let tail = &production[anchor..];
let acquire = tail
.find(".acquire_owned()")
.expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)");
let rederive = tail.find("pinata_object_list_for_refs(").expect(
"F2 gate missing: the Pinata task must re-derive its object set via \
pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure",
);
assert!(
acquire < rederive,
"F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \
permit is acquired, so a parked task never holds the MB-scale object list"
);
// The re-derivation is driven by the push's ref tuples (the enqueued unit), not a
// retained object list — tie "enqueue ref_updates" to the call explicitly.
assert!(
tail[rederive..].starts_with("pinata_object_list_for_refs(")
&& tail[rederive..]
.get(..400)
.map(|w| w.contains("ref_updates_clone"))
.unwrap_or(false),
"F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \
(ref_updates_clone), the small unit the parked task retains"
);
}
/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's
/// git group is reaped): `git_receive_pack` must take the per-repo in-process write
/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the
/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is
/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at
/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the
/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test
/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this
/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs`
/// (the `mod tests` half names these identifiers in its own harness).
#[test]
fn f3_second_writer_leased_until_reap() {
let repos = src("api/repos.rs");
let smart_http = src("git/smart_http.rs");
let repos_production = repos
.split("#[cfg(test)]")
.next()
.expect("split always yields a first chunk");
// The lease is acquired, then carried by the AdmissionGuard (.with_lease), before
// receive_pack runs the write. Severing any of the three turns this red.
let lease_acquire = repos_production.find("repo_write_leases").expect(
"F3 gate missing: git_receive_pack no longer takes the per-repo write lease \
(state.repo_write_leases)",
);
let with_lease = repos_production.find(".with_lease(").expect(
"F3 gate missing: the write-path AdmissionGuard no longer carries the lease \
(.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \
NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).",
);
let receive = repos_production
.find("smart_http::receive_pack(")
.expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack");
assert!(
lease_acquire < with_lease && with_lease < receive,
"F3 gate bypassed: the write lease must be acquired, then carried by the \
AdmissionGuard (.with_lease), BEFORE receive_pack runs the write"
);
// The AdmissionGuard must actually hold the lease (Option<RepoWriteLease>) via a
// with_lease setter, so it travels into the detached reaper. Removing the field or
// method turns this red.
assert!(
smart_http.contains("_lease: Option<crate::state::RepoWriteLease>")
&& smart_http.contains("pub fn with_lease("),
"F3 gate missing: AdmissionGuard must hold an Option<RepoWriteLease> set via \
with_lease, so the lease rides the guard into KillGroupOnDrop's reaper"
);
}
/// #174 U1 — every blocking walk in the `/ipfs` scan carries the request's walk
/// admission.
///
/// The admission is an `Arc<WalkAdmission>` cloned into each `spawn_blocking`
/// closure, so the global + per-source permits release only when the last holder
/// drops: a client disconnect leaves the abandoned closure holding the slot, and a
/// panicking closure leaves the handler holding it.
///
/// Only the walk site runs under `state.git_bin`, so only it can be pinned by the
/// fake-git harness and mutation-verified dynamically (see
/// `get_by_cid_walk_permit_held_through_blocking_walk`). The probe and the content
/// read deliberately shell to the real `git`, which is why this structural check
/// exists: it binds all three sites, and any blocking site added to this loop
/// later, without reversing that deliberate independence.
///
/// MUTATION (RED): delete any one `Arc::clone(&admission)` binding, or drop the
/// clone from inside its closure, and the count falls below three.
#[test]
fn inv22_ipfs_walk_admission_reaches_every_blocking_site() {
let ipfs = src("api/ipfs.rs");
// The shared owner must exist and be built once per request.
assert!(
ipfs.contains("struct WalkAdmission") && ipfs.contains("Arc::new(WalkAdmission {"),
"U1 gate missing: the /ipfs walk admission must be a shared WalkAdmission, \
not a handler-local permit pair"
);
// Every blocking site in the scan takes its own clone...
let clones = ipfs.matches("Arc::clone(&admission)").count();
assert!(
clones >= 3,
"U1 gate bypassed: expected an admission clone for each of the three \
/ipfs spawn_blocking sites (probe, walk, read); found {clones}. A blocking \
walk that does not hold the admission lets a disconnect or a panic free the \
slot while its git child is still running."
);
// ...and each clone is actually moved INTO the blocking closure, not merely
// created in the async frame (which would hold nothing across the join).
let held = ipfs.matches("let _admission = ").count();
assert!(
held >= 3,
"U1 gate bypassed: each admission clone must be bound inside its \
spawn_blocking closure so the blocking work owns it; found {held} of 3."
);
// The count of blocking sites is itself the thing being covered: if a fourth
// appears, it needs an admission clone too and this gate must be revisited.
let sites = ipfs.matches("spawn_blocking(move ||").count();
assert_eq!(
sites, 3,
"the /ipfs scan grew or lost a spawn_blocking site ({sites} found, expected 3); \
give any new blocking walk its own Arc::clone(&admission) and update this gate"
);
}
/// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY,
/// which is the moment receive-pack returns success, not the end of the handler and
/// not after `guard.release()`.
///
/// The tail owes this push its pins, recovery copy, and announcements. Everything
/// below the spawn stays in the cancellable request future, so anything the tail is
/// spawned after is a window where a client disconnect drops that work while the pack
/// is already durable on disk. `guard.release()` is such a window: on success it
/// awaits the Tigris upload and then the advisory unlock.
///
/// The lower bound matters just as much as the upper one: `release` runs on failure
/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing
/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be
/// what gates it, so the success check is explicit and this gate binds it: the spawn
/// must sit inside `if push_succeeded`, and `release` must consume the same flag so
/// the two cannot drift apart.
///
/// This is an ordering check rather than a cancellation-race test on purpose: it is
/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which
/// drives the actual disconnect through a parked `release`. Same instrument the F3
/// gate above uses.
///
/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below
/// `guard.release(` and the ordering assertion fails; take it out of the
/// `if push_succeeded` block and the failed-push assertion fails.
#[test]
fn inv22_replication_tail_spawns_at_the_durability_boundary() {
let repos = src("api/repos.rs");
// Production half only — the tests below name these identifiers too.
let production = repos
.split("#[cfg(test)]")
.next()
.expect("split always yields a first chunk");
let success_flag = production
.find("let push_succeeded = receive_result.is_ok();")
.expect(
"U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()",
);
let gate_open = production
.find("if push_succeeded {")
.expect("U5 gate missing: the tail spawn must be gated on the push having succeeded");
let spawn = production
.find("tokio::spawn(post_receive_replication_tail(")
.expect("U5 gate missing: the replication tail must be spawned by git_receive_pack");
let release = production
.find("guard.release(push_succeeded)")
.expect("U5 gate stale: release must consume the same success flag as the tail gate");
let touch = production
.find("state.db.touch_repo(")
.expect("U5 gate stale: git_receive_pack no longer calls touch_repo");
let webhook = production
.find("webhooks::fire_event(")
.expect("U5 gate stale: git_receive_pack no longer fires push webhooks");
assert!(
success_flag < gate_open && gate_open < spawn,
"U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \
rejected push spawns a tail that pins and announces a half-applied repo"
);
// Still inside that block: no `}` may close it between the gate and the spawn.
assert!(
!production[gate_open + "if push_succeeded {".len()..spawn].contains('}'),
"U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \
rejected push now spawns a tail"
);
assert!(
spawn < release && spawn < touch && spawn < webhook,
"U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \
and the webhook fan-out, so a disconnect in any of those windows cannot drop \
this push's pins, recovery copy, and announcements"
);
}