mirror of
https://github.com/Twigpine/node.git
synced 2026-11-08 07:38:18 -10:00
Merge pull request #174 from Gitlawb/fix/served-git-concurrency-cap
feat(node,git): cap concurrent served git ops with a 503 load-shed (#62)
This commit is contained in:
24 files changed
+18665
-630
No files matched your search
+127
-4
@@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb
|
||||
# ── Database pool & startup resilience ────────────────────────────────────
|
||||
# Maximum connections in the PostgreSQL pool. A cap, not a floor —
|
||||
# connections open lazily. Size against the DB server's max_connections,
|
||||
# remembering admin tooling opens its own pool.
|
||||
GITLAWB_DB_MAX_CONNECTIONS=20
|
||||
# remembering admin tooling opens its own pool. Each concurrent write pins one
|
||||
# connection for its whole duration (the connection-affine advisory lock), so the
|
||||
# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8
|
||||
# headroom — keep this comfortably above that (default 48 for pushes 32).
|
||||
GITLAWB_DB_MAX_CONNECTIONS=48
|
||||
# Seconds a request waits for a pool connection before failing with 503.
|
||||
GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5
|
||||
# Upper bound on each startup connect+migrate attempt, in seconds. Keep it
|
||||
@@ -109,15 +112,135 @@ GITLAWB_MAX_PACK_BYTES=2147483648
|
||||
|
||||
# Max seconds a served git upload-pack / receive-pack (clone / push) may run
|
||||
# before it is aborted with a 504. Bounds a hung git that would otherwise pin a
|
||||
# worker and, on push, the repo write lock. Does NOT cover the info/refs
|
||||
# advertisement or the withheld-blob path, which remain unbounded. Default 600.
|
||||
# worker and, on push, the repo write lock. Also bounds both info/refs
|
||||
# advertisements, the withheld-blob pack build, and the push-side candidate
|
||||
# discovery (rev-list / cat-file), all reaped via process-group teardown (#174).
|
||||
# On the path-scoped upload-pack path the withheld-blob classification walk and
|
||||
# the pack serve share ONE deadline, so this value bounds their COMBINED duration
|
||||
# rather than giving each stage a full budget: a walk that consumes it leaves the
|
||||
# serve nothing and the clone gets a 504. Serving large path-scoped repos may
|
||||
# need a higher value here than when each stage was budgeted separately.
|
||||
# Must be 1..=3153600000 (100 years): the node derives deadlines from this value,
|
||||
# and a larger one cannot be represented. Default 600.
|
||||
GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600
|
||||
|
||||
# Max seconds the storage-ACQUISITION phase of a served git op may run before the
|
||||
# request is shed with a 503, separate from the git-run timeout above. A
|
||||
# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS
|
||||
# only starts once git spawns, so without this a stalled backend (a hung Tigris
|
||||
# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains
|
||||
# the pool until every later request 503s. On expiry the permit is released
|
||||
# (fail-closed). Kept separate because acquisition and git execution are distinct
|
||||
# cost centers. Must be positive; set very large to effectively disable. Default 30.
|
||||
GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30
|
||||
|
||||
# Max concurrent git READ ops (upload-pack + the upload-pack info/refs
|
||||
# advertisement) served at once, a global pool separate from the push pool below.
|
||||
# The anon receive-pack info/refs advertisement has its OWN pool (see below), not
|
||||
# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from
|
||||
# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one
|
||||
# caller cannot monopolize the pool. Default 128.
|
||||
GITLAWB_MAX_CONCURRENT_GIT_OPS=128
|
||||
|
||||
# Max concurrent git-receive-pack (push) POST operations, in a pool separate from
|
||||
# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an
|
||||
# authenticated push at admission. The anon receive-pack info/refs advertisement
|
||||
# runs in a SEPARATE pool of the same size (disjoint from this one), so an
|
||||
# advertisement flood cannot shed a push either. Over-cap sheds a 503 +
|
||||
# Retry-After. Default 32.
|
||||
GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32
|
||||
|
||||
# Max pushes PARKED at once waiting for the per-repo write lease of the SAME repo.
|
||||
# A parked push holds its already-buffered pack body in memory, so this bounds that
|
||||
# memory for a hot repo. Past the cap the newest push sheds a clean 503 +
|
||||
# Retry-After ("repo is busy with another push") instead of joining the queue; a
|
||||
# push to a DIFFERENT repo is never affected. The lease holder is not counted.
|
||||
# Raising GITLAWB_MAX_CONCURRENT_GIT_PUSHES does not raise this; set it explicitly.
|
||||
# Default 8.
|
||||
GITLAWB_REPO_LEASE_MAX_WAITERS=8
|
||||
|
||||
# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all
|
||||
# repos. The per-repo encrypt-task set caps the pin-task COUNT; this caps how many
|
||||
# of those loops RUN CONCURRENTLY, and so how much MB-scale object-id list memory is
|
||||
# held WHILE BEING PINNED. It does not bound retained memory in general: on the local
|
||||
# IPFS path the caller builds its list before this pool is entered, so a loop parked
|
||||
# waiting for a permit still holds its full list, and how many park is capped only
|
||||
# per repo. Do not size memory from this knob alone. A loop DEFERS (waits) when the
|
||||
# pool is full, never drops a pin. Default 8.
|
||||
GITLAWB_MAX_CONCURRENT_PIN_TASKS=8
|
||||
|
||||
# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement)
|
||||
# a single caller may hold, so one caller cannot monopolize the read pool. Keyed
|
||||
# on the resolved SOURCE IP, never the DID: a signature does not move a caller off
|
||||
# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below:
|
||||
# left unset, a node behind an edge/NAT keys all callers on the edge IP and this
|
||||
# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a
|
||||
# high-fanout caller (CI behind one NAT) then needs the operator to raise this.
|
||||
# Default 16.
|
||||
GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16
|
||||
|
||||
# Two further per-source concurrency caps exist on the PUSH side but have NO
|
||||
# environment variable of their own: both are DERIVED as
|
||||
# GITLAWB_MAX_CONCURRENT_GIT_PUSHES / 8, with a floor of 1. They are not settable
|
||||
# independently, so raising the push pool raises both proportionally, and the
|
||||
# default push pool of 32 gives each a cap of 4.
|
||||
# * The anon receive-pack info/refs advertisement cap: one source IP may hold at
|
||||
# most this many slots in the dedicated advert pool, so saturating that pool
|
||||
# takes ~8 distinct source IPs.
|
||||
# * The authenticated receive-pack POST cap: one source IP may hold at most this
|
||||
# many WRITE-pool slots. This one is load-bearing for push availability, since
|
||||
# it is acquired before the global write permit: without it, one host minting
|
||||
# disposable did:key identities could open enough slow pushes to monopolize the
|
||||
# write pool and 503 every other source (owner enforcement defaults off, and
|
||||
# the push rate limiter caps arrival rate, not in-flight concurrency).
|
||||
# Keyed on the resolved source IP, never the DID, so a DID farm does not defeat
|
||||
# them; keying granularity follows GITLAWB_TRUSTED_PROXY like the read cap above.
|
||||
|
||||
# ── Push rate limiting (git-receive-pack flood brake) ─────────────────────
|
||||
# Max receive-pack requests (info/refs advertisement + push POST) per client
|
||||
# IP per hour. 0 disables. Default 600.
|
||||
GITLAWB_PUSH_RATE_LIMIT=600
|
||||
|
||||
# ── /ipfs/{cid} visibility-walk admission (#174) ──────────────────────────
|
||||
# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to
|
||||
# decide whether the caller may read a path-scoped blob. It is publicly reachable,
|
||||
# so it is bounded to keep a permissionless caller from fanning out unbounded
|
||||
# concurrent walks and exhausting blocking-pool threads + PIDs.
|
||||
# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from
|
||||
# the served-git pools). Over-cap sheds a 503. Default 32.
|
||||
GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32
|
||||
# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git
|
||||
# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map).
|
||||
# Default 4.
|
||||
GITLAWB_IPFS_WALK_PER_SOURCE=4
|
||||
# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a
|
||||
# blob in a path-scoped repo costs a full-history walk). Over-cap repos are
|
||||
# skipped without a verdict and the scan continues; if the object is then found
|
||||
# nowhere the request sheds a retryable 503 instead of a false 404. Default 64.
|
||||
GITLAWB_IPFS_MAX_REPOS_WALKED=64
|
||||
# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each
|
||||
# visit costs a repo acquire — on a Tigris cache miss a full archive download,
|
||||
# so this is also the worst-case object-store fetch count per request — plus a
|
||||
# cat-file probe. On exhaustion the scan stops and sheds a retryable 503.
|
||||
# Default 1024.
|
||||
GITLAWB_IPFS_MAX_REPO_VISITS=1024
|
||||
# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk
|
||||
# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire
|
||||
# and walk stages to the remaining budget, and no stage starts once it is
|
||||
# exhausted (the scan then sheds a retryable 503). The object-type probe and
|
||||
# content-read cat-file subprocesses are budget-checked before starting AND each
|
||||
# run under their own deadline (the lesser of GITLAWB_GIT_SERVICE_TIMEOUT_SECS and
|
||||
# the remaining budget), reaped via process-group teardown, so a hung cat-file
|
||||
# cannot hold the request's walk slot past it. One hang path is still unbounded:
|
||||
# the probe's object-store readability check is a plain filesystem sweep with
|
||||
# nothing to reap, so a wedged filesystem can hold the slot past the deadline.
|
||||
# Must be 1..=3153600000 (100 years): the node derives an Instant deadline from
|
||||
# this value, and a larger one cannot be represented. Default 600.
|
||||
GITLAWB_IPFS_REQUEST_BUDGET_SECS=600
|
||||
# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct
|
||||
# from the concurrency caps above). 0 disables. Default 600.
|
||||
GITLAWB_IPFS_RATE_LIMIT=600
|
||||
|
||||
# ── Creation rate limiting (repo/agent/issue/PR flood brake) ──────────────
|
||||
# Max creation requests (POST /api/v1/repos, /api/register, fork, issues,
|
||||
# pulls) per client IP per hour, in addition to the per-DID limit. The per-DID
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Gitlawb Node is the open-source node software behind the Gitlawb network. It lets anyone run a self-hosted node, publish repositories under a DID, sign writes with Ed25519 HTTP signatures, replicate git activity across peers, and move toward a resilient app-delivery network where code and build assets can be served closer to users.
|
||||
|
||||
Gitlawb is not trying to be only “another git host.” The long-term direction is:
|
||||
Gitlawb is not trying to be only "another git host." The long-term direction is:
|
||||
|
||||
```txt
|
||||
Decentralized GitHub
|
||||
@@ -34,12 +34,12 @@ This is a Rust workspace with four crates:
|
||||
|
||||
Most git hosting today depends on a small number of centralized platforms. Gitlawb Nodes are designed for a different model:
|
||||
|
||||
- **Own your identity** — every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`.
|
||||
- **Signed writes by default** — write requests use RFC 9421 HTTP Signatures instead of passwords.
|
||||
- **Git-native transport** — repositories are still real git repositories served over smart HTTP.
|
||||
- **Agent-native workflows** — the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents.
|
||||
- **Peer-aware delivery** — nodes can announce, discover, gossip, and sync with each other.
|
||||
- **App CDN direction** — the network can evolve from decentralized code storage into code + asset + app delivery.
|
||||
- **Own your identity**: every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`.
|
||||
- **Signed writes by default**: write requests use RFC 9421 HTTP Signatures instead of passwords.
|
||||
- **Git-native transport**: repositories are still real git repositories served over smart HTTP.
|
||||
- **Agent-native workflows**: the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents.
|
||||
- **Peer-aware delivery**: nodes can announce, discover, gossip, and sync with each other.
|
||||
- **App CDN direction**: the network can evolve from decentralized code storage into code + asset + app delivery.
|
||||
|
||||
---
|
||||
|
||||
@@ -189,14 +189,14 @@ For public-network use, make sure `GITLAWB_NODE` points to the node you want. Th
|
||||
|
||||
Public nodes (e.g. `node.gitlawb.com`) require two things on writes:
|
||||
|
||||
1. **RFC 9421 HTTP Signatures** — every write is signed by your identity key. `gl`
|
||||
1. **RFC 9421 HTTP Signatures**: every write is signed by your identity key. `gl`
|
||||
and the `git-remote-gitlawb` helper do this automatically. An old/unsigned CLI
|
||||
fails with `401 not_an_agent`; `gl` will tell you to upgrade and register.
|
||||
2. **An iCaptcha proof** on the spam-gated writes (**repo create, fork, register**).
|
||||
`gl` solves this for you: on the node's `403 icaptcha_proof_required` it reads the
|
||||
`x-icaptcha-url` / `x-icaptcha-level` hints, requests a challenge, solves it
|
||||
locally (arithmetic / algebra / sequence), and **retries the same signed request**
|
||||
with the `x-icaptcha-proof` header — no manual steps, no env vars.
|
||||
with the `x-icaptcha-proof` header. No manual steps, no env vars.
|
||||
|
||||
```bash
|
||||
gl identity new # create did:key identity
|
||||
@@ -215,14 +215,14 @@ Notes:
|
||||
- **Proofs are short-lived (~5 min TTL) and single-use.** If one expires between
|
||||
solving and use, the client transparently solves a fresh one and retries.
|
||||
- **What needs what:** create / fork / register are signed **and** iCaptcha-gated;
|
||||
`git push` is **signed-only** (owner signature is the gate — no per-push challenge);
|
||||
`git push` is **signed-only** (owner signature is the gate, no per-push challenge);
|
||||
reads (clone / fetch / `repo info`) need no proof. A non-existent repo returns a
|
||||
clear `404`, never a placeholder.
|
||||
- **API-key iCaptcha deployments:** set `GITLAWB_ICAPTCHA_URL` to your iCaptcha
|
||||
origin and `GITLAWB_ICAPTCHA_API_KEY` to its key. The client only talks to an
|
||||
`https` origin whose host is allowlisted (that URL or the public default), and
|
||||
sends the bearer token **only** to your configured origin — never to a URL a
|
||||
node advertises — so a hostile node can't capture the key or redirect the solve.
|
||||
sends the bearer token **only** to your configured origin, never to a URL a
|
||||
node advertises, so a hostile node can't capture the key or redirect the solve.
|
||||
|
||||
---
|
||||
|
||||
@@ -343,7 +343,19 @@ Important node settings:
|
||||
| `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. |
|
||||
| `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. |
|
||||
| `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. |
|
||||
| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. |
|
||||
| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. |
|
||||
| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. |
|
||||
| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. |
|
||||
| `GITLAWB_MAX_CONCURRENT_GIT_PUSHES` | Max concurrent `git-receive-pack` POST operations, in a pool separate from the read pool. The anon receive-pack `info/refs` advertisement runs in a third pool of the same size, disjoint from both, so an advertisement flood cannot shed a push either. Two per-source push caps are derived from this value (`/8`, floor 1) and have no env var of their own. Over-cap sheds a 503 + Retry-After. Default 32. |
|
||||
| `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER` | Max concurrent read ops a single caller may hold, so one caller cannot monopolize the read pool. Keyed on the resolved source IP, never the DID, and only as granular as `GITLAWB_TRUSTED_PROXY`: left unset, a node behind an edge or NAT keys every caller on the edge IP and this collapses to one global cap. Default 16. |
|
||||
| `GITLAWB_MAX_CONCURRENT_PIN_TASKS` | Max post-push pin loops (IPFS + Pinata) running concurrently across all repos. This caps how many loops RUN at once, not how much object-id list memory the node retains: on the local IPFS path a loop parked waiting for a permit still holds its full list. Do not size memory from this knob alone. A loop over cap waits, never drops a pin. Default 8. |
|
||||
| `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. |
|
||||
| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. |
|
||||
| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. |
|
||||
| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. |
|
||||
| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. |
|
||||
| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. |
|
||||
| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. |
|
||||
| `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. |
|
||||
| `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. |
|
||||
| `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. |
|
||||
|
||||
@@ -77,3 +77,6 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] }
|
||||
[dev-dependencies]
|
||||
mockito = "1"
|
||||
tempfile = "3"
|
||||
# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert
|
||||
# on which timer fired instead of on wall clock.
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
+2647
-33
File diff suppressed because it is too large.
Load diff
+6899
-175
File diff suppressed because it is too large.
Load diff
@@ -520,6 +520,22 @@ mod tests {
|
||||
sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)),
|
||||
peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
|
||||
shutdown_tx: tokio::sync::watch::channel(false).0,
|
||||
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
encrypt_inflight: crate::state::EncryptInflight::new(),
|
||||
repo_write_leases: crate::state::RepoWriteLeases::new(8),
|
||||
git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
|
||||
git_push_advert_per_caller:
|
||||
crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
|
||||
git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
|
||||
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_ipfs_walk_per_caller:
|
||||
crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
|
||||
ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
|
||||
git_bin: "git".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,33 @@
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds
|
||||
/// (100 years).
|
||||
///
|
||||
/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs`
|
||||
/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply
|
||||
/// below, but it shares this ceiling because the defect class and the "set it very large
|
||||
/// to disable" contract are the same.
|
||||
///
|
||||
/// The knob is not just stored, it is arithmetic input: the write path derives the
|
||||
/// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into
|
||||
/// `build_filtered_pack` and `blob_paths`, which each build a deadline as
|
||||
/// `Instant::now() + Duration::from_secs(this)`. That addition panics on overflow in
|
||||
/// RELEASE as well as debug, so an unbounded `u64` here turns an operator typo into a
|
||||
/// serve-path crash rather than a very long timeout. Bounding at parse time keeps every
|
||||
/// derived duration in range at once, instead of hardening each site as it is found.
|
||||
///
|
||||
/// The ceiling is representability, NOT a policy view of a sane timeout, and that
|
||||
/// distinction is what sets the number. The help text has always told operators to set
|
||||
/// this very large to disable the bound, so values like `999999999` (~31 years) are
|
||||
/// working production "off" settings; a tighter, tidier cap would fail those nodes at
|
||||
/// boot on upgrade over a value that was never the defect. 100 years clears every such
|
||||
/// setting while staying a factor of about 5.85 under the ~584-year ceiling of a
|
||||
/// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we
|
||||
/// build for. Every value that worked before still parses; only the ones that would have
|
||||
/// panicked are rejected.
|
||||
pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 100 * 365 * 24 * 60 * 60;
|
||||
|
||||
#[derive(Parser, Debug, Clone)]
|
||||
#[command(name = "gitlawb-node", about = "gitlawb node daemon", version)]
|
||||
pub struct Config {
|
||||
@@ -170,26 +197,54 @@ pub struct Config {
|
||||
/// Maximum wall-clock time a single served git operation (upload-pack /
|
||||
/// receive-pack through `run_git_service`) may run before it is aborted and
|
||||
/// its process group torn down, in seconds. Bounds a git that neither
|
||||
/// finishes nor disconnects. Must be positive; set it very large to
|
||||
/// effectively disable the bound. Default: 600s (10 min), generous for large
|
||||
/// clones. Does not cover the ref advertisement (`info/refs`) or the
|
||||
/// withheld-blob fetch path (`upload_pack_excluding`, a blocking
|
||||
/// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded.
|
||||
/// finishes nor disconnects. Must be positive and at most
|
||||
/// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (100 years, the largest value every derived
|
||||
/// deadline can represent); setting it very large is still the way to disable the
|
||||
/// bound. Default: 600s (10 min), generous for large clones. Also bounds the ref
|
||||
/// advertisement
|
||||
/// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s
|
||||
/// pack-objects stage), which now share the same timeout + process-group
|
||||
/// teardown (#174).
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS",
|
||||
default_value_t = 600,
|
||||
value_parser = clap::value_parser!(u64).range(1..)
|
||||
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
|
||||
)]
|
||||
pub git_service_timeout_secs: u64,
|
||||
|
||||
/// Maximum wall-clock time the storage-acquisition phase of a served git
|
||||
/// operation may run before the request is shed with a 503, in seconds. This
|
||||
/// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris
|
||||
/// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a
|
||||
/// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool)
|
||||
/// on a write acquire. A concurrency permit is taken BEFORE this phase, and
|
||||
/// `git_service_timeout_secs` only starts once git spawns, so without this the
|
||||
/// acquire phase is unbounded: a stalled backend pins the permit and drains the
|
||||
/// pool until every later request 503s. On expiry the permit is released and a
|
||||
/// bounded 503 + Retry-After is returned (fail-closed). Kept separate from
|
||||
/// `git_service_timeout_secs` because acquisition and git execution are distinct
|
||||
/// cost centers — one shared budget would let a slow acquire starve git. Must be
|
||||
/// positive; set it very large to effectively disable the bound. Default: 30s.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS",
|
||||
default_value_t = 30,
|
||||
value_parser = clap::value_parser!(u64).range(1..)
|
||||
)]
|
||||
pub git_acquire_timeout_secs: u64,
|
||||
|
||||
/// Maximum connections in the PostgreSQL pool. This is a cap, not a floor
|
||||
/// (connections open lazily). Size against the database server's
|
||||
/// max_connections, remembering admin tooling opens its own pool.
|
||||
/// max_connections, remembering admin tooling opens its own pool. Each
|
||||
/// concurrent write pins one pooled connection for its whole duration (the
|
||||
/// advisory lock in `repo_store::acquire_write` is connection-affine), so this
|
||||
/// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow
|
||||
/// pushes starve every other DB path — enforced by `Config::validate`.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_DB_MAX_CONNECTIONS",
|
||||
default_value_t = 20,
|
||||
default_value_t = 48,
|
||||
value_parser = clap::value_parser!(u32).range(1..)
|
||||
)]
|
||||
pub db_max_connections: u32,
|
||||
@@ -234,6 +289,256 @@ pub struct Config {
|
||||
value_parser = clap::value_parser!(u64).range(1..)
|
||||
)]
|
||||
pub db_retry_max_secs: u64,
|
||||
|
||||
/// Maximum number of served git operations (upload-pack / receive-pack /
|
||||
/// info-refs) allowed to run concurrently. Beyond this the node sheds the
|
||||
/// request with a clean 503 + Retry-After instead of spawning another git
|
||||
/// subprocess and risking PID/thread exhaustion. Portable backstop: the
|
||||
/// compose `pids_limit` is not present on Fly, whose connection-concurrency
|
||||
/// cap is a different axis (500 connections each fan out to git +
|
||||
/// pack-objects + threads). Size below the process budget with headroom.
|
||||
///
|
||||
/// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK
|
||||
/// `info/refs` advertisement only. The authenticated push POST draws from a
|
||||
/// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can
|
||||
/// never reach, and each read caller is additionally bounded by
|
||||
/// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual
|
||||
/// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs`
|
||||
/// advertisement draws from its OWN dedicated pool (sized like the write pool but
|
||||
/// disjoint), so an advertisement flood can never occupy a permit the
|
||||
/// authenticated push POST needs at admission (#174).
|
||||
///
|
||||
/// A permit is held for the whole op. Every git subprocess that STREAMS is
|
||||
/// duration-bounded and reaps its process group on disconnect: upload-pack,
|
||||
/// receive-pack, and both info/refs advertisements run under
|
||||
/// `git_service_timeout_secs` with `process_group(0)` teardown, and the
|
||||
/// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side
|
||||
/// candidate-discovery children (`rev-list` / `cat-file`) now run under the same
|
||||
/// bounded runner with process-group teardown, so a stuck git child no longer
|
||||
/// holds its slot indefinitely (#174 closed the duration/cancellation gaps this
|
||||
/// comment previously tracked).
|
||||
///
|
||||
/// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value
|
||||
/// well under tokio's `Semaphore` permit limit so an oversized value is a
|
||||
/// clean CLI error rather than a boot-time panic.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_MAX_CONCURRENT_GIT_OPS",
|
||||
default_value_t = 128,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub max_concurrent_git_ops: usize,
|
||||
|
||||
/// Maximum number of concurrent `git-receive-pack` (push) operations. The
|
||||
/// authenticated push POST draws from this dedicated pool, separate from
|
||||
/// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an
|
||||
/// authenticated push at admission (#174). The anon-reachable receive-pack
|
||||
/// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived
|
||||
/// from this knob), disjoint from this one, so an advertisement flood cannot
|
||||
/// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 +
|
||||
/// Retry-After.
|
||||
///
|
||||
/// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value
|
||||
/// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI
|
||||
/// error rather than a boot-time panic).
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES",
|
||||
default_value_t = 32,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub max_concurrent_git_pushes: usize,
|
||||
|
||||
/// Maximum number of pushes that may be PARKED on one repo's in-process write lease
|
||||
/// at once. Same-repo pushes are serialized (block-and-wait), and a parked push has
|
||||
/// already had its entire pack buffered by axum, so an unbounded queue on a contended
|
||||
/// repo is unbounded buffered memory held for up to `git_service_timeout_secs * 2 +
|
||||
/// 60` (1260s at defaults). Past this cap the newest push sheds a clean 503 +
|
||||
/// Retry-After instead of joining the queue.
|
||||
///
|
||||
/// The trade: raising it lets more same-repo pushes wait their turn (fewer 503s for a
|
||||
/// hot repo, more memory pinned by waiters); lowering it sheds sooner. Only pushes to
|
||||
/// the SAME repo count, and only ones parked right now, so the cap can never deny a
|
||||
/// push to a different repo. The holder is deliberately not counted: a holder whose
|
||||
/// cleanup never ran would otherwise pin a slot forever and wedge the repo, which is
|
||||
/// the failure the `steal_after` reclaim exists to survive.
|
||||
///
|
||||
/// Default: 8, a quarter of the default `max_concurrent_git_pushes` (32). Raising the
|
||||
/// push pool does not raise this; set it explicitly. Must be between 1 and 1_048_576.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_REPO_LEASE_MAX_WAITERS",
|
||||
default_value_t = 8,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub repo_lease_max_waiters: usize,
|
||||
|
||||
/// Max concurrent post-push pin loops (`ipfs_pin` and `pinata`
|
||||
/// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding
|
||||
/// pin-task COUNT to one per repo, but each pin loop holds a full per-push
|
||||
/// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it,
|
||||
/// so N distinct repos could hold N such lists at once. This caps how many run
|
||||
/// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits) and never drops,
|
||||
/// since a dropped pin would lose the object's replication copy.
|
||||
///
|
||||
/// It does not cap the memory itself: the local IPFS path builds its list before
|
||||
/// taking a permit, so tasks parked on this pool still hold theirs, and how many
|
||||
/// park is capped only per repo. Lowering this knob bounds concurrent pinning, not
|
||||
/// how much an actor pushing to many repos can retain.
|
||||
///
|
||||
/// Default: 8. Must be between 1 and 1_048_576.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS",
|
||||
default_value_t = 8,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub max_concurrent_pin_tasks: usize,
|
||||
|
||||
/// Maximum concurrent read operations (`upload-pack` and the upload-pack
|
||||
/// `info/refs` advertisement) a single caller may hold at once, so one caller
|
||||
/// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are
|
||||
/// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a
|
||||
/// caller off this cap, so an authenticated client cannot mint DIDs to escape it.
|
||||
/// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`.
|
||||
/// Left unset (the default), a node behind an edge/NAT keys all callers on the
|
||||
/// edge IP, so this cap collapses to a single global cap rather than per-client.
|
||||
/// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a
|
||||
/// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a
|
||||
/// caller sheds a clean 503 + Retry-After.
|
||||
///
|
||||
/// Default: 16. Must be between 1 and 1_048_576.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER",
|
||||
default_value_t = 16,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub max_concurrent_reads_per_caller: usize,
|
||||
|
||||
/// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their
|
||||
/// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs
|
||||
/// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history
|
||||
/// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It
|
||||
/// draws from THIS pool, not any served-git pool: a distinct public cost center
|
||||
/// on a distinct surface, so sharing a git pool would let anonymous /ipfs
|
||||
/// traffic shed authenticated git ops (the auth-boundary trap). A permit is
|
||||
/// held for the whole request (across the repo loop) so it reflects real
|
||||
/// blocking-thread occupancy, not merely the tokio wait. Beyond this the request
|
||||
/// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling
|
||||
/// keeps the value under tokio's `Semaphore` permit limit so an oversized value
|
||||
/// is a clean CLI error rather than a boot-time panic. Default: 32.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS",
|
||||
default_value_t = 32,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub max_concurrent_ipfs_walks: usize,
|
||||
|
||||
/// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at
|
||||
/// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174).
|
||||
/// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`),
|
||||
/// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no
|
||||
/// admission step, so keying on the DID would let one host mint disposable DIDs to
|
||||
/// multiply its budget. A request with no resolvable key (no trusted header, no
|
||||
/// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a
|
||||
/// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_IPFS_WALK_PER_SOURCE",
|
||||
default_value_t = 4,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub ipfs_walk_per_source: usize,
|
||||
|
||||
/// Upper bound on the number of EXPENSIVE visibility walks
|
||||
/// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a
|
||||
/// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a
|
||||
/// path-scoped repo costs a walk, so the cap counts exactly those candidates
|
||||
/// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead
|
||||
/// (counting them here would starve a plain public copy past the cap out of
|
||||
/// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict
|
||||
/// and the scan continues; if the request then finds the object nowhere it
|
||||
/// sheds a retryable 503 + Retry-After rather than misreport existing content
|
||||
/// absent with a 404. The handler still short-circuits the moment it serves.
|
||||
/// Must be between 1 and 1_048_576. Default: 64.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_IPFS_MAX_REPOS_WALKED",
|
||||
default_value_t = 64,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub ipfs_max_repos_walked: usize,
|
||||
|
||||
/// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT —
|
||||
/// pass the repo-level visibility gate into the acquire + `cat-file` probe.
|
||||
/// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a
|
||||
/// full repo-archive download from object storage, so the worst-case
|
||||
/// object-store fetch count for one request equals this ceiling) plus a git
|
||||
/// probe subprocess. On exhaustion the scan STOPS — unlike
|
||||
/// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is
|
||||
/// no cheaper way to keep scanning — and the request sheds a retryable 503 +
|
||||
/// Retry-After rather than a false 404. Must be between 1 and 1_048_576.
|
||||
/// Default: 1024.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_IPFS_MAX_REPO_VISITS",
|
||||
default_value_t = 1024,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_048_576)
|
||||
)]
|
||||
pub ipfs_max_repo_visits: usize,
|
||||
|
||||
/// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's
|
||||
/// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how
|
||||
/// MANY requests hold walk slots; this bounds how LONG one admitted request
|
||||
/// may keep its slot. Without it, each repo iteration draws a fresh
|
||||
/// `git_acquire_timeout_secs` and each expensive walk a fresh
|
||||
/// `git_service_timeout_secs`, so one request scanning many repos could hold
|
||||
/// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe,
|
||||
/// visibility walk, content read) starts only while budget remains, and the
|
||||
/// acquire wait and walk deadline are clamped to `min(their own timeout,
|
||||
/// remaining budget)`; a stage is never started with zero remaining. On
|
||||
/// exhaustion the scan stops without a verdict and the request sheds a
|
||||
/// retryable 503 + Retry-After rather than a false 404. The clamps bound
|
||||
/// only the acquire and walk stages (overshoot there is the walk watchdog's
|
||||
/// SIGTERM grace + SIGKILL settle); the `object_type` /
|
||||
/// `read_object_content` probe subprocesses are budget-checked before they
|
||||
/// start AND each run under their own deadline (the lesser of
|
||||
/// `git_service_timeout_secs` and the remaining budget), reaped by
|
||||
/// process-group teardown, so a hung `cat-file` cannot hold the request's walk
|
||||
/// slot past it. Still unbounded: the probe's `object_store_readable` check is a
|
||||
/// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can
|
||||
/// hold the slot past the deadline.
|
||||
/// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is
|
||||
/// representability, NOT a policy view of a sane budget: `get_by_cid` derives the
|
||||
/// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition
|
||||
/// is an explicit overflow check rather than a debug-only one, so a value near the top
|
||||
/// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of
|
||||
/// setting a very long budget. The ceiling sits well below where that starts, about a
|
||||
/// factor of 5.85 (see the constant's own note), so it is a conservative margin rather than the
|
||||
/// exact overflow point; rejecting at parse time keeps the unrepresentable values out
|
||||
/// of every reachable configuration. Setting it very large is still the way to
|
||||
/// effectively disable the budget, and the documented sentinels (`999999999`,
|
||||
/// `1000000000`) are well inside the range.
|
||||
/// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length
|
||||
/// walk still fits.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS",
|
||||
default_value_t = 600,
|
||||
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
|
||||
)]
|
||||
pub ipfs_request_budget_secs: u64,
|
||||
|
||||
/// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The
|
||||
/// route is publicly reachable (`optional_signature`) and each request can drive
|
||||
/// a full-history git walk, so it carries a per-IP flood brake in addition to the
|
||||
/// concurrency cap above (a rate limit bounds request *rate*, the semaphore
|
||||
/// bounds concurrent slow holds — different axes). Keyed on the resolved client
|
||||
/// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600.
|
||||
#[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)]
|
||||
pub ipfs_rate_limit: usize,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -250,6 +555,36 @@ impl Config {
|
||||
}
|
||||
PathBuf::from(&self.key_path)
|
||||
}
|
||||
|
||||
/// DB connections reserved for everything other than held write-locks: auth
|
||||
/// lookups, visibility-rule reads, the post-receive tail's own DB writes, and
|
||||
/// admin tooling. A write pins one pooled connection for its whole duration, so
|
||||
/// the pool must clear the concurrent-write cap by at least this margin.
|
||||
pub const DB_POOL_APP_HEADROOM: u32 = 8;
|
||||
|
||||
/// Cross-field boot validation. Single-field ranges are enforced by clap; this
|
||||
/// catches combinations that ship a denial-of-service under otherwise-valid
|
||||
/// values. Call once at startup and fail fast on `Err`.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
// A write pins one pooled connection for its whole duration (the
|
||||
// connection-affine advisory lock in repo_store::acquire_write), and
|
||||
// concurrent writes are capped at max_concurrent_git_pushes. If the pool
|
||||
// does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes
|
||||
// drains every connection and every other DB path 503s. (#174 F1)
|
||||
let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64);
|
||||
if (self.db_max_connections as u64) < floor {
|
||||
return Err(format!(
|
||||
"GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \
|
||||
+ {} headroom = {}: each concurrent write pins one pooled connection for its whole \
|
||||
duration, so a smaller pool lets a burst of slow pushes starve every other DB path.",
|
||||
self.db_max_connections,
|
||||
self.max_concurrent_git_pushes,
|
||||
Self::DB_POOL_APP_HEADROOM,
|
||||
floor
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -272,4 +607,370 @@ mod tests {
|
||||
Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 (RED-before/GREEN-after): the upper bound is what keeps every duration
|
||||
/// derived from this knob in range — the lease steal bound's `* 2 + 60` on the write
|
||||
/// path, and the `Instant::now() + Duration::from_secs(..)` deadlines in
|
||||
/// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in
|
||||
/// release builds too. Checked at parse time so no reachable configuration can carry a
|
||||
/// value those sites cannot represent.
|
||||
///
|
||||
/// The pre-existing "set it very large to disable the bound" settings are asserted
|
||||
/// alongside the rejections on purpose. The bound exists to exclude unrepresentable
|
||||
/// values, not to impose a view of a reasonable timeout, so a node that has been
|
||||
/// running on ~31 years must not start failing at boot on upgrade.
|
||||
#[test]
|
||||
fn git_service_timeout_rejects_values_no_derived_duration_can_represent() {
|
||||
let parse = |secs: u64| {
|
||||
Config::try_parse_from([
|
||||
"gitlawb-node",
|
||||
"--git-service-timeout-secs",
|
||||
&secs.to_string(),
|
||||
])
|
||||
};
|
||||
|
||||
// Large "disable the bound" values that predate the ceiling still parse.
|
||||
for disable in [1_000_000_000, 999_999_999] {
|
||||
assert_eq!(
|
||||
parse(disable)
|
||||
.unwrap_or_else(|e| panic!("{disable} was a working setting: {e}"))
|
||||
.git_service_timeout_secs,
|
||||
disable
|
||||
);
|
||||
}
|
||||
|
||||
// At the ceiling: accepted, and every derived duration still fits.
|
||||
let at_max =
|
||||
parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse");
|
||||
assert_eq!(
|
||||
at_max.git_service_timeout_secs,
|
||||
GIT_SERVICE_TIMEOUT_SECS_MAX
|
||||
);
|
||||
assert!(at_max
|
||||
.git_service_timeout_secs
|
||||
.checked_mul(2)
|
||||
.and_then(|v| v.checked_add(60))
|
||||
.is_some());
|
||||
assert!(std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(
|
||||
at_max.git_service_timeout_secs
|
||||
))
|
||||
.is_some());
|
||||
|
||||
// Past the ceiling, and the top of the u64 range clap used to accept — the value
|
||||
// that panics `Instant::now() + Duration::from_secs(..)` outright.
|
||||
for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] {
|
||||
assert!(
|
||||
parse(over).is_err(),
|
||||
"{over} is past the representable ceiling and must be rejected at parse time"
|
||||
);
|
||||
}
|
||||
assert!(std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(u64::MAX))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks,
|
||||
8
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"])
|
||||
.max_concurrent_pin_tasks,
|
||||
2
|
||||
);
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err()
|
||||
);
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops,
|
||||
128
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"])
|
||||
.max_concurrent_git_ops,
|
||||
8
|
||||
);
|
||||
// 0 permits would shed every served-git request with a 503; clap must reject it.
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err());
|
||||
// Above the ceiling would panic tokio's Semaphore::new at boot (permits >
|
||||
// usize::MAX >> 3); clap must reject it as a clean CLI error instead.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"])
|
||||
.is_err()
|
||||
);
|
||||
// The ceiling itself is accepted.
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"])
|
||||
.max_concurrent_git_ops,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes,
|
||||
32
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"])
|
||||
.max_concurrent_git_pushes,
|
||||
8
|
||||
);
|
||||
// 0 permits would shed every push with a 503; clap must reject it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err()
|
||||
);
|
||||
// Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"])
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"])
|
||||
.max_concurrent_git_pushes,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks,
|
||||
32
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"])
|
||||
.max_concurrent_ipfs_walks,
|
||||
4
|
||||
);
|
||||
// 0 permits would shed every /ipfs walk with a 503; clap must reject it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err()
|
||||
);
|
||||
// Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"])
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"])
|
||||
.max_concurrent_ipfs_walks,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"])
|
||||
.ipfs_walk_per_source,
|
||||
2
|
||||
);
|
||||
// 0 would shed every /ipfs walk from a keyed source; clap must reject it.
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err());
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked,
|
||||
64
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"])
|
||||
.ipfs_max_repos_walked,
|
||||
8
|
||||
);
|
||||
// 0 would walk no repos (serve nothing); clap must reject it.
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err());
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits,
|
||||
1024
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"])
|
||||
.ipfs_max_repo_visits,
|
||||
8
|
||||
);
|
||||
// 0 would visit no repos (serve nothing); clap must reject it.
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err());
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs,
|
||||
600
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"])
|
||||
.ipfs_request_budget_secs,
|
||||
30
|
||||
);
|
||||
// 0 would expire every /ipfs request at its first stage (unconditional
|
||||
// 503); clap must reject it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived
|
||||
/// from this knob in range. `get_by_cid` builds the request budget as
|
||||
/// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an
|
||||
/// explicit overflow check rather than a debug-only one, so an oversized value aborts
|
||||
/// every `/ipfs/{cid}` request in a release build instead of setting a very long budget.
|
||||
/// The route is anon-reachable, so the failure is operator-triggered but publicly felt.
|
||||
/// Checked at parse time so no reachable configuration can carry a value the deadline
|
||||
/// cannot represent.
|
||||
///
|
||||
/// The large "disable the bound" settings are asserted alongside the rejections on
|
||||
/// purpose, the same way the `git_service_timeout_secs` sibling does it. The ceiling
|
||||
/// exists to exclude unrepresentable values, not to impose a view of a reasonable
|
||||
/// budget, so a node already running on such a value must not start failing at boot on
|
||||
/// upgrade. A test that only checked the boundary would pass with a far tighter cap.
|
||||
#[test]
|
||||
fn ipfs_request_budget_rejects_values_no_derived_duration_can_represent() {
|
||||
let parse = |secs: u64| {
|
||||
Config::try_parse_from([
|
||||
"gitlawb-node",
|
||||
"--ipfs-request-budget-secs",
|
||||
&secs.to_string(),
|
||||
])
|
||||
};
|
||||
|
||||
// Large "disable the bound" values that predate the ceiling still parse.
|
||||
for disable in [1_000_000_000, 999_999_999] {
|
||||
assert_eq!(
|
||||
parse(disable)
|
||||
.unwrap_or_else(|e| panic!("{disable} was a working setting: {e}"))
|
||||
.ipfs_request_budget_secs,
|
||||
disable
|
||||
);
|
||||
}
|
||||
|
||||
// At the ceiling: accepted, and the derived deadline still fits. This knob feeds
|
||||
// only the `Instant` addition (no multiply derivation like the lease steal bound),
|
||||
// so there is no `checked_mul` clause to carry over from the sibling test.
|
||||
let at_max =
|
||||
parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse");
|
||||
assert_eq!(
|
||||
at_max.ipfs_request_budget_secs,
|
||||
GIT_SERVICE_TIMEOUT_SECS_MAX
|
||||
);
|
||||
assert!(std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(
|
||||
at_max.ipfs_request_budget_secs
|
||||
))
|
||||
.is_some());
|
||||
|
||||
// Past the ceiling, and the top of the u64 range clap used to accept. Only the
|
||||
// latter actually panics `Instant::now() + Duration::from_secs(..)`; the ceiling
|
||||
// sits well below that, which is the conservative margin the constant documents.
|
||||
for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] {
|
||||
assert!(
|
||||
parse(over).is_err(),
|
||||
"{over} is past the representable ceiling and must be rejected at parse time"
|
||||
);
|
||||
}
|
||||
assert!(std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(u64::MAX))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller,
|
||||
16
|
||||
);
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"])
|
||||
.max_concurrent_reads_per_caller,
|
||||
4
|
||||
);
|
||||
// 0 would shed every read from a keyed caller; clap must reject it.
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"])
|
||||
.is_err()
|
||||
);
|
||||
assert!(Config::try_parse_from([
|
||||
"gitlawb-node",
|
||||
"--max-concurrent-reads-per-caller",
|
||||
"1048577"
|
||||
])
|
||||
.is_err());
|
||||
assert_eq!(
|
||||
Config::parse_from([
|
||||
"gitlawb-node",
|
||||
"--max-concurrent-reads-per-caller",
|
||||
"1048576"
|
||||
])
|
||||
.max_concurrent_reads_per_caller,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 F1: a connection-affine write lock pins a pooled connection per
|
||||
/// concurrent write, so the pool must clear `max_concurrent_git_pushes` by
|
||||
/// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path.
|
||||
/// `validate()` must reject an under-sized pool at boot.
|
||||
#[test]
|
||||
fn db_pool_must_clear_the_git_push_cap() {
|
||||
// Shipped defaults validate (48 >= 32 + 8).
|
||||
Config::parse_from(["gitlawb-node"])
|
||||
.validate()
|
||||
.expect("default config must validate");
|
||||
|
||||
// An under-sized pool relative to the push cap is rejected (20 < 32 + 8).
|
||||
let under = Config::parse_from([
|
||||
"gitlawb-node",
|
||||
"--db-max-connections",
|
||||
"20",
|
||||
"--max-concurrent-git-pushes",
|
||||
"32",
|
||||
]);
|
||||
assert!(
|
||||
under.validate().is_err(),
|
||||
"db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected"
|
||||
);
|
||||
|
||||
// Exactly at the floor validates (40 == 32 + 8).
|
||||
let at_floor = Config::parse_from([
|
||||
"gitlawb-node",
|
||||
"--db-max-connections",
|
||||
"40",
|
||||
"--max-concurrent-git-pushes",
|
||||
"32",
|
||||
]);
|
||||
assert!(
|
||||
at_floor.validate().is_ok(),
|
||||
"db_max_connections at the floor (pushes + headroom) must validate"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -170,7 +170,7 @@ pub async fn encrypt_and_pin(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope).await {
|
||||
let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await {
|
||||
Ok(c) if !c.is_empty() => c,
|
||||
Ok(_) => {
|
||||
tracing::warn!(oid = %oid, "pin_git_object returned no cid; skipping");
|
||||
|
||||
@@ -47,6 +47,9 @@ pub enum AppError {
|
||||
#[error("git service timed out: {0}")]
|
||||
Timeout(String),
|
||||
|
||||
#[error("server overloaded: {0}")]
|
||||
Overloaded(String),
|
||||
|
||||
#[error("database error: {0}")]
|
||||
Db(#[from] sqlx::Error),
|
||||
|
||||
@@ -147,6 +150,12 @@ impl IntoResponse for AppError {
|
||||
DB_UNAVAILABLE_CODE,
|
||||
DB_UNAVAILABLE_MESSAGE.into(),
|
||||
),
|
||||
// 503 with a Retry-After (attached after this match — the shared tail
|
||||
// can't carry per-variant headers). This is the single place Overloaded
|
||||
// becomes a response, so it can never ship a 503 without the retry hint.
|
||||
AppError::Overloaded(msg) => {
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone())
|
||||
}
|
||||
AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()),
|
||||
AppError::Internal(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -160,7 +169,17 @@ impl IntoResponse for AppError {
|
||||
"message": message,
|
||||
}));
|
||||
|
||||
(status, body).into_response()
|
||||
let mut resp = (status, body).into_response();
|
||||
// Overloaded advertises when to retry. It rides the shared tail above for
|
||||
// its body/status, so the header is attached here rather than in a bespoke
|
||||
// early return — keeping the variant handled in exactly one place.
|
||||
if matches!(self, AppError::Overloaded(_)) {
|
||||
resp.headers_mut().insert(
|
||||
axum::http::header::RETRY_AFTER,
|
||||
axum::http::HeaderValue::from_static("1"),
|
||||
);
|
||||
}
|
||||
resp
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,4 +201,14 @@ mod tests {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overloaded_maps_to_503_with_retry_after() {
|
||||
let resp = AppError::Overloaded("x".into()).into_response();
|
||||
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
resp.headers().get("retry-after").unwrap().to_str().unwrap(),
|
||||
"1"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -29,9 +29,9 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use anyhow::Result;
|
||||
|
||||
/// Env var that forces the push path to always full-scan, bypassing the delta
|
||||
/// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch
|
||||
@@ -64,11 +64,21 @@ pub enum PinCandidates {
|
||||
/// Fail-closed: any condition where the introduced set cannot be safely
|
||||
/// determined returns [`PinCandidates::FullScanRequired`] rather than a partial
|
||||
/// set, so the caller full-scans instead of silently under-pinning.
|
||||
pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates {
|
||||
// KTD7 kill-switch: force the (already-tested) full-scan fallback. The env
|
||||
// read is split out from the pure logic so the resolver stays unit-testable
|
||||
// without touching process-global state.
|
||||
resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan())
|
||||
pub fn resolve_push_delta(
|
||||
repo_path: &Path,
|
||||
new_tips: &[&str],
|
||||
old_tips: &[&str],
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> PinCandidates {
|
||||
resolve_push_delta_inner(
|
||||
repo_path,
|
||||
new_tips,
|
||||
old_tips,
|
||||
force_full_scan(),
|
||||
git_bin,
|
||||
deadline,
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether the force-full-scan kill-switch env var is set.
|
||||
@@ -83,6 +93,8 @@ fn resolve_push_delta_inner(
|
||||
new_tips: &[&str],
|
||||
old_tips: &[&str],
|
||||
force_full_scan: bool,
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> PinCandidates {
|
||||
if force_full_scan {
|
||||
tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan");
|
||||
@@ -102,7 +114,7 @@ fn resolve_push_delta_inner(
|
||||
// commit). Bare `cat-file -t` returns `tag` for an annotated tag, and
|
||||
// `for-each-ref %(*objecttype)` peels only one level — neither is correct.
|
||||
for tip in new_tips {
|
||||
match peeled_object_type(repo_path, tip) {
|
||||
match peeled_object_type(repo_path, tip, git_bin, deadline) {
|
||||
Some(t) if t == "commit" => {}
|
||||
other => {
|
||||
tracing::debug!(
|
||||
@@ -115,7 +127,7 @@ fn resolve_push_delta_inner(
|
||||
}
|
||||
}
|
||||
|
||||
match rev_list_delta(repo_path, new_tips, old_tips) {
|
||||
match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) {
|
||||
Ok(oids) => PinCandidates::Delta(oids),
|
||||
Err(e) => {
|
||||
tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan");
|
||||
@@ -126,41 +138,43 @@ fn resolve_push_delta_inner(
|
||||
|
||||
/// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`,
|
||||
/// `blob`), or `None` if the object is missing/unpeelable or git errored.
|
||||
fn peeled_object_type(repo_path: &Path, sha: &str) -> Option<String> {
|
||||
let output = Command::new("git")
|
||||
.args(["cat-file", "-t", &format!("{sha}^{{}}")])
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.ok()?;
|
||||
if !output.status.success() {
|
||||
return None;
|
||||
}
|
||||
Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
|
||||
fn peeled_object_type(
|
||||
repo_path: &Path,
|
||||
sha: &str,
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> Option<String> {
|
||||
let peel = format!("{sha}^{{}}");
|
||||
let out = crate::git::visibility_pack::run_bounded_git(
|
||||
git_bin,
|
||||
&["cat-file", "-t", &peel],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)
|
||||
.ok()?;
|
||||
Some(String::from_utf8_lossy(&out).trim().to_string())
|
||||
}
|
||||
|
||||
/// Run `git rev-list --objects --no-object-names <new> --not <old>` and return
|
||||
/// the bare OID set. Decides on `status.success()` *before* parsing stdout, so
|
||||
/// a walk that prints a valid prefix then errors mid-walk is discarded.
|
||||
fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result<Vec<String>> {
|
||||
fn rev_list_delta(
|
||||
repo_path: &Path,
|
||||
new_tips: &[&str],
|
||||
old_tips: &[&str],
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<String>> {
|
||||
let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"];
|
||||
args.extend_from_slice(new_tips);
|
||||
if !old_tips.is_empty() {
|
||||
args.push("--not");
|
||||
args.extend_from_slice(old_tips);
|
||||
}
|
||||
|
||||
let output = Command::new("git")
|
||||
.args(&args)
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("failed to run git rev-list for push delta")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
bail!("git rev-list failed: {stderr}");
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let out =
|
||||
crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?;
|
||||
let stdout = String::from_utf8_lossy(&out);
|
||||
Ok(stdout
|
||||
.lines()
|
||||
.map(|l| l.trim().to_string())
|
||||
@@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res
|
||||
/// reconciliation sweep relies on. It returns *all* objects (including
|
||||
/// unreachable/dangling ones), which is what the sweep needs to catch
|
||||
/// stragglers — do not swap it for a reachability walk.
|
||||
pub fn list_all_objects(repo_path: &Path) -> Result<Vec<String>> {
|
||||
let output = Command::new("git")
|
||||
.args([
|
||||
pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<Vec<String>> {
|
||||
let out = crate::git::visibility_pack::run_bounded_git(
|
||||
git_bin,
|
||||
&[
|
||||
"cat-file",
|
||||
"--batch-all-objects",
|
||||
"--batch-check=%(objectname)",
|
||||
])
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("failed to run git cat-file")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
bail!("git cat-file failed: {stderr}");
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
let stdout = String::from_utf8_lossy(&out);
|
||||
Ok(stdout
|
||||
.lines()
|
||||
.map(|l| l.trim().to_string())
|
||||
@@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result<Vec<String>> {
|
||||
/// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed
|
||||
/// filter needs to tell blobs (content, withholdable) from commits/trees
|
||||
/// (structural, never withheld) without typing the candidate list itself.
|
||||
pub fn list_all_objects_with_type(repo_path: &Path) -> Result<Vec<(String, String)>> {
|
||||
let output = Command::new("git")
|
||||
.args([
|
||||
pub fn list_all_objects_with_type(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<(String, String)>> {
|
||||
let out = crate::git::visibility_pack::run_bounded_git(
|
||||
git_bin,
|
||||
&[
|
||||
"cat-file",
|
||||
"--batch-all-objects",
|
||||
"--batch-check=%(objectname) %(objecttype)",
|
||||
])
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("failed to run git cat-file")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
bail!("git cat-file failed: {stderr}");
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
let stdout = String::from_utf8_lossy(&out);
|
||||
Ok(stdout
|
||||
.lines()
|
||||
.filter_map(|l| {
|
||||
@@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result<Vec<(String, Strin
|
||||
/// fail-closed pin filter drops any candidate blob absent from the reachable,
|
||||
/// visibility-allowed set; a dangling private blob is in this set but not the
|
||||
/// allowed set, so it never replicates (#99).
|
||||
pub fn all_blob_oids(repo_path: &Path) -> Result<HashSet<String>> {
|
||||
Ok(list_all_objects_with_type(repo_path)?
|
||||
pub fn all_blob_oids(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
deadline: Instant,
|
||||
) -> Result<HashSet<String>> {
|
||||
Ok(list_all_objects_with_type(repo_path, git_bin, deadline)?
|
||||
.into_iter()
|
||||
.filter(|(_, ty)| ty == "blob")
|
||||
.map(|(oid, _)| oid)
|
||||
@@ -268,22 +282,68 @@ pub struct PinCandidateSet {
|
||||
/// it can never leak because the withheld/fail-closed filter still runs on
|
||||
/// whatever set is returned. `full_scan` rides on the returned set so the caller
|
||||
/// knows when the dangling-inclusive filter is required.
|
||||
///
|
||||
/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`,
|
||||
/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta
|
||||
/// rev-list, and the full-scan fallback — run under ONE permit held for the
|
||||
/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git
|
||||
/// and never parks.
|
||||
///
|
||||
/// `force_full_scan` forces the full-scan fallback regardless of the tips — the
|
||||
/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with
|
||||
/// the env kill-switch (either forces), and it disqualifies the empty-tips fast
|
||||
/// path: a forced-scan call with no tips must still enumerate the repo, never
|
||||
/// silently resolve to an empty delta (which would pin nothing and lose the
|
||||
/// drained pushes' work — the F5 bug shape).
|
||||
pub async fn resolve_candidates_for_push(
|
||||
scan_sem: std::sync::Arc<tokio::sync::Semaphore>,
|
||||
repo_path: PathBuf,
|
||||
new_tips: Vec<String>,
|
||||
old_tips: Vec<String>,
|
||||
git_bin: String,
|
||||
timeout: Duration,
|
||||
force_full_scan: bool,
|
||||
) -> PinCandidateSet {
|
||||
let force = force_full_scan || self::force_full_scan();
|
||||
// No-git fast path: a deletion-only push resolves to an empty delta without
|
||||
// spawning any git child, so it must not park on the scan pool. A forced
|
||||
// full scan (caller flag or kill-switch) disqualifies it — the scan spawns git.
|
||||
if new_tips.is_empty() && !force {
|
||||
tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta");
|
||||
return PinCandidateSet {
|
||||
candidates: Vec::new(),
|
||||
full_scan: false,
|
||||
};
|
||||
}
|
||||
// Scan admission (#174 F4): DEFER, never shed — a dropped scan would
|
||||
// silently under-pin this push. The permit moves into the blocking closure
|
||||
// so a started scan always completes holding it. Residuals at
|
||||
// `acquire_scan_permit`.
|
||||
let permit =
|
||||
crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _permit = permit;
|
||||
// ONE shared deadline for the whole scan, per jatmn ("the same deadline").
|
||||
let deadline = Instant::now() + timeout;
|
||||
let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect();
|
||||
let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect();
|
||||
match resolve_push_delta(&repo_path, &new_refs, &old_refs) {
|
||||
// `force` already folds in the env kill-switch, so the forced arm skips the
|
||||
// delta machinery outright; the unforced arm goes through the normal
|
||||
// resolver (whose own env read is false here by construction).
|
||||
let resolved = if force {
|
||||
tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)");
|
||||
PinCandidates::FullScanRequired
|
||||
} else {
|
||||
resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline)
|
||||
};
|
||||
match resolved {
|
||||
PinCandidates::Delta(objs) => {
|
||||
tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta");
|
||||
PinCandidateSet { candidates: objs, full_scan: false }
|
||||
}
|
||||
PinCandidates::FullScanRequired => {
|
||||
tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback");
|
||||
match list_all_objects(&repo_path) {
|
||||
match list_all_objects(&repo_path, &git_bin, deadline) {
|
||||
Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true },
|
||||
Err(e) => {
|
||||
tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)");
|
||||
@@ -304,8 +364,13 @@ pub async fn resolve_candidates_for_push(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashSet;
|
||||
use std::process::Command;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn td() -> std::time::Instant {
|
||||
std::time::Instant::now() + std::time::Duration::from_secs(600)
|
||||
}
|
||||
|
||||
/// Minimal git helper for building test repos.
|
||||
struct Repo {
|
||||
_td: TempDir,
|
||||
@@ -363,9 +428,10 @@ mod tests {
|
||||
let repo = Repo::new();
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
let c2 = repo.commit_file("b.txt", "two\n");
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1]))
|
||||
.into_iter()
|
||||
.collect();
|
||||
let got: HashSet<String> =
|
||||
delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td()))
|
||||
.into_iter()
|
||||
.collect();
|
||||
// The new blob b.txt and commit c2 are in the delta; the old blob a.txt
|
||||
// and commit c1 are not.
|
||||
let new_blob = repo.rev("HEAD:b.txt");
|
||||
@@ -383,7 +449,7 @@ mod tests {
|
||||
// genuinely new objects, never fewer.
|
||||
let repo = Repo::new();
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c1], &[]))
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td()))
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(got.contains(&c1));
|
||||
@@ -399,9 +465,15 @@ mod tests {
|
||||
// Rewrite history: reset to base, commit a different file.
|
||||
repo.git(&["reset", "-q", "--hard", &base]);
|
||||
let new_tip = repo.commit_file("c.txt", "three\n");
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip]))
|
||||
.into_iter()
|
||||
.collect();
|
||||
let got: HashSet<String> = delta(resolve_push_delta(
|
||||
&repo.path,
|
||||
&[&new_tip],
|
||||
&[&old_tip],
|
||||
"git",
|
||||
td(),
|
||||
))
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(got.contains(&new_tip), "new tip in delta");
|
||||
assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt"))));
|
||||
// No error; force-push computes new-minus-old cleanly.
|
||||
@@ -413,7 +485,7 @@ mod tests {
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
// All updates were deletions => new_tips empty after the caller strips zeros.
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[], &[ZERO]),
|
||||
resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()),
|
||||
PinCandidates::Delta(Vec::new())
|
||||
);
|
||||
}
|
||||
@@ -424,7 +496,7 @@ mod tests {
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
let blob = repo.rev("HEAD:a.txt");
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[&blob], &[]),
|
||||
resolve_push_delta(&repo.path, &[&blob], &[], "git", td()),
|
||||
PinCandidates::FullScanRequired,
|
||||
"a blob tip must force full scan (rev-list would exit 0 and walk it)"
|
||||
);
|
||||
@@ -436,7 +508,7 @@ mod tests {
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
let tree = repo.rev("HEAD^{tree}");
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[&tree], &[]),
|
||||
resolve_push_delta(&repo.path, &[&tree], &[], "git", td()),
|
||||
PinCandidates::FullScanRequired
|
||||
);
|
||||
}
|
||||
@@ -449,7 +521,7 @@ mod tests {
|
||||
repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]);
|
||||
let tag = repo.rev("treetag");
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[&tag], &[]),
|
||||
resolve_push_delta(&repo.path, &[&tag], &[], "git", td()),
|
||||
PinCandidates::FullScanRequired,
|
||||
"annotated tag peeling to a tree must force full scan"
|
||||
);
|
||||
@@ -465,7 +537,7 @@ mod tests {
|
||||
repo.git(&["tag", "-a", "t2", "-m", "x", &t1]);
|
||||
let t2 = repo.rev("t2");
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[&t2], &[]),
|
||||
resolve_push_delta(&repo.path, &[&t2], &[], "git", td()),
|
||||
PinCandidates::FullScanRequired
|
||||
);
|
||||
}
|
||||
@@ -480,7 +552,7 @@ mod tests {
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
repo.git(&["tag", "-a", "rel", "-m", "release", &c1]);
|
||||
let tag = repo.rev("rel");
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&tag], &[]))
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td()))
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(
|
||||
@@ -501,7 +573,7 @@ mod tests {
|
||||
let t1 = repo.rev("t1");
|
||||
repo.git(&["tag", "-a", "t2", "-m", "x", &t1]);
|
||||
let t2 = repo.rev("t2");
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&t2], &[]))
|
||||
let got: HashSet<String> = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td()))
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(
|
||||
@@ -516,9 +588,16 @@ mod tests {
|
||||
let repo = Repo::new();
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
let c2 = repo.commit_file("b.txt", "two\n");
|
||||
let set =
|
||||
resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()])
|
||||
.await;
|
||||
let set = resolve_candidates_for_push(
|
||||
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
repo.path.clone(),
|
||||
vec![c2.clone()],
|
||||
vec![c1.clone()],
|
||||
"git".to_string(),
|
||||
std::time::Duration::from_secs(600),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(!set.full_scan, "happy-path delta is not a full scan");
|
||||
let got: HashSet<String> = set.candidates.into_iter().collect();
|
||||
let new_blob = repo.rev("HEAD:b.txt");
|
||||
@@ -536,20 +615,206 @@ mod tests {
|
||||
let repo = Repo::new();
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
let blob = repo.rev("HEAD:a.txt");
|
||||
let all: HashSet<String> = list_all_objects(&repo.path).unwrap().into_iter().collect();
|
||||
let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await;
|
||||
let all: HashSet<String> = list_all_objects(&repo.path, "git", td())
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let set = resolve_candidates_for_push(
|
||||
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
repo.path.clone(),
|
||||
vec![blob],
|
||||
vec![],
|
||||
"git".to_string(),
|
||||
std::time::Duration::from_secs(600),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(set.full_scan, "non-commit tip is signalled as a full scan");
|
||||
let got: HashSet<String> = set.candidates.into_iter().collect();
|
||||
assert_eq!(got, all, "non-commit tip falls back to full repo scan");
|
||||
}
|
||||
|
||||
/// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip
|
||||
/// cat-file type probe + delta rev-list, and the full-scan fallback) run under a
|
||||
/// scan-admission permit: with a zero-permit pool the call parks and spawns no
|
||||
/// git; once a permit is available the SAME call runs (defer, not shed). On
|
||||
/// ungated code the git runs regardless of the pool (RED).
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn resolve_candidates_defers_when_scan_pool_exhausted() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let marker = dir.path().join("git.ran");
|
||||
// Fake git records ANY invocation; cat-file reports a commit tip so the
|
||||
// delta stage proceeds, rev-list yields an empty delta.
|
||||
let fake = dir.path().join("fakegit");
|
||||
std::fs::write(
|
||||
&fake,
|
||||
format!(
|
||||
"#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n",
|
||||
marker.display()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(&fake, perm).unwrap();
|
||||
let git_bin = fake.to_str().unwrap().to_string();
|
||||
let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string();
|
||||
|
||||
let sem: Arc<Semaphore> = Arc::new(Semaphore::new(0));
|
||||
let blocked = tokio::time::timeout(
|
||||
Duration::from_millis(500),
|
||||
resolve_candidates_for_push(
|
||||
sem.clone(),
|
||||
dir.path().to_path_buf(),
|
||||
vec![tip.clone()],
|
||||
vec![],
|
||||
git_bin.clone(),
|
||||
Duration::from_secs(5),
|
||||
false,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
blocked.is_err(),
|
||||
"the pin-candidate scan must defer (park on admission) when the pool is exhausted"
|
||||
);
|
||||
assert!(
|
||||
!marker.exists(),
|
||||
"the scan's git must not spawn while its admission permit is unavailable (F4)"
|
||||
);
|
||||
|
||||
// Release admission: the SAME scan now runs (defer, not shed).
|
||||
sem.add_permits(1);
|
||||
let set = resolve_candidates_for_push(
|
||||
sem,
|
||||
dir.path().to_path_buf(),
|
||||
vec![tip],
|
||||
vec![],
|
||||
git_bin,
|
||||
Duration::from_secs(5),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
marker.exists(),
|
||||
"once admission is available the deferred scan runs its git"
|
||||
);
|
||||
assert!(!set.full_scan, "commit tip + empty rev-list is a delta");
|
||||
assert!(set.candidates.is_empty());
|
||||
}
|
||||
|
||||
/// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch
|
||||
/// off) computes its empty delta without spawning ANY git child, so it must
|
||||
/// complete without acquiring from the scan pool — even at zero permits. The
|
||||
/// per-tip cat-file probe means every push with a non-empty new tip DOES spawn
|
||||
/// git, so this is the only genuinely git-free stage.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn resolve_candidates_no_git_fast_path_skips_admission() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let marker = dir.path().join("git.ran");
|
||||
let fake = dir.path().join("fakegit");
|
||||
std::fs::write(
|
||||
&fake,
|
||||
format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(&fake, perm).unwrap();
|
||||
|
||||
let set = tokio::time::timeout(
|
||||
Duration::from_millis(500),
|
||||
resolve_candidates_for_push(
|
||||
Arc::new(Semaphore::new(0)),
|
||||
dir.path().to_path_buf(),
|
||||
vec![],
|
||||
vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()],
|
||||
fake.to_str().unwrap().to_string(),
|
||||
Duration::from_secs(5),
|
||||
false,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("the no-new-tips fast path must not park on the scan pool");
|
||||
assert_eq!(
|
||||
set,
|
||||
PinCandidateSet {
|
||||
candidates: Vec::new(),
|
||||
full_scan: false
|
||||
}
|
||||
);
|
||||
assert!(!marker.exists(), "the fast path must spawn no git at all");
|
||||
}
|
||||
|
||||
/// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit
|
||||
/// `force_full_scan` flag, and a flagged call with NO tips must still enumerate
|
||||
/// the whole repo (full_scan=true, non-empty candidates). The RED arm of the
|
||||
/// encoding question: if the marker were encoded as a plain empty-tips call
|
||||
/// (flag off — the fast path), the drain would resolve to an empty delta and
|
||||
/// pin nothing, silently losing the coalesced pushes' work.
|
||||
#[tokio::test]
|
||||
async fn forced_full_scan_with_no_tips_enumerates_the_repo() {
|
||||
let repo = Repo::new();
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
let all: HashSet<String> = list_all_objects(&repo.path, "git", td())
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(!all.is_empty(), "fixture repo has objects");
|
||||
|
||||
let set = resolve_candidates_for_push(
|
||||
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
repo.path.clone(),
|
||||
vec![],
|
||||
vec![],
|
||||
"git".to_string(),
|
||||
std::time::Duration::from_secs(600),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert!(set.full_scan, "the forced call is signalled as a full scan");
|
||||
let got: HashSet<String> = set.candidates.into_iter().collect();
|
||||
assert_eq!(
|
||||
got, all,
|
||||
"a forced full scan with no tips enumerates the repo — never an empty delta"
|
||||
);
|
||||
|
||||
// The discriminator: the SAME empty-tips call without the flag is the
|
||||
// deletion-only fast path (empty delta, pin nothing). The two must differ,
|
||||
// or the marker encoding has collapsed into the silent-loss shape.
|
||||
let unforced = resolve_candidates_for_push(
|
||||
std::sync::Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
repo.path.clone(),
|
||||
vec![],
|
||||
vec![],
|
||||
"git".to_string(),
|
||||
std::time::Duration::from_secs(600),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(!unforced.full_scan);
|
||||
assert!(unforced.candidates.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_oid_tip_forces_full_scan() {
|
||||
let repo = Repo::new();
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[bogus], &[]),
|
||||
resolve_push_delta(&repo.path, &[bogus], &[], "git", td()),
|
||||
PinCandidates::FullScanRequired,
|
||||
"a missing/corrupt tip OID must force full scan"
|
||||
);
|
||||
@@ -564,7 +829,7 @@ mod tests {
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
let c2 = repo.commit_file("b.txt", "two\n");
|
||||
let old_tree = repo.rev(&format!("{c1}^{{tree}}"));
|
||||
let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]);
|
||||
let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td());
|
||||
match result {
|
||||
PinCandidates::FullScanRequired => {} // safe: caller full-scans
|
||||
PinCandidates::Delta(objs) => {
|
||||
@@ -583,10 +848,15 @@ mod tests {
|
||||
// branch2 from base advances independently
|
||||
repo.git(&["checkout", "-q", "-b", "branch2", &base]);
|
||||
let b2 = repo.commit_file("c.txt", "three\n");
|
||||
let got: HashSet<String> =
|
||||
delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base]))
|
||||
.into_iter()
|
||||
.collect();
|
||||
let got: HashSet<String> = delta(resolve_push_delta(
|
||||
&repo.path,
|
||||
&[&b1, &b2],
|
||||
&[&base, &base],
|
||||
"git",
|
||||
td(),
|
||||
))
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert!(got.contains(&b1), "branch1 new commit");
|
||||
assert!(got.contains(&b2), "branch2 new commit");
|
||||
}
|
||||
@@ -595,7 +865,7 @@ mod tests {
|
||||
fn empty_repo_no_tips_yields_empty_delta() {
|
||||
let repo = Repo::new();
|
||||
assert_eq!(
|
||||
resolve_push_delta(&repo.path, &[], &[]),
|
||||
resolve_push_delta(&repo.path, &[], &[], "git", td()),
|
||||
PinCandidates::Delta(Vec::new())
|
||||
);
|
||||
}
|
||||
@@ -609,12 +879,12 @@ mod tests {
|
||||
let repo = Repo::new();
|
||||
let c1 = repo.commit_file("a.txt", "one\n");
|
||||
assert_eq!(
|
||||
resolve_push_delta_inner(&repo.path, &[&c1], &[], true),
|
||||
resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()),
|
||||
PinCandidates::FullScanRequired
|
||||
);
|
||||
// And with the flag off, the same push yields a Delta.
|
||||
assert!(matches!(
|
||||
resolve_push_delta_inner(&repo.path, &[&c1], &[], false),
|
||||
resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()),
|
||||
PinCandidates::Delta(_)
|
||||
));
|
||||
}
|
||||
@@ -624,7 +894,7 @@ mod tests {
|
||||
let repo = Repo::new();
|
||||
repo.commit_file("a.txt", "one\n");
|
||||
repo.commit_file("b.txt", "two\n");
|
||||
let all = list_all_objects(&repo.path).unwrap();
|
||||
let all = list_all_objects(&repo.path, "git", td()).unwrap();
|
||||
// 2 commits + 2 trees + 2 blobs = 6 objects.
|
||||
assert_eq!(all.len(), 6, "got: {all:?}");
|
||||
}
|
||||
@@ -642,7 +912,7 @@ mod tests {
|
||||
std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap();
|
||||
let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]);
|
||||
|
||||
let blobs = all_blob_oids(&repo.path).unwrap();
|
||||
let blobs = all_blob_oids(&repo.path, "git", td()).unwrap();
|
||||
assert!(blobs.contains(&reachable_blob), "reachable blob present");
|
||||
assert!(
|
||||
blobs.contains(&dangling),
|
||||
@@ -652,7 +922,7 @@ mod tests {
|
||||
assert!(!blobs.contains(&tree), "tree is not a blob");
|
||||
|
||||
// The typed lister tags each object; spot-check the dangling blob's type.
|
||||
let typed = list_all_objects_with_type(&repo.path).unwrap();
|
||||
let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap();
|
||||
assert!(
|
||||
typed
|
||||
.iter()
|
||||
@@ -660,4 +930,96 @@ mod tests {
|
||||
"dangling object is typed as a blob"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::time::{Duration, Instant};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
// Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge).
|
||||
let fake = dir.path().join("fakegit");
|
||||
std::fs::write(
|
||||
&fake,
|
||||
"#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n",
|
||||
)
|
||||
.unwrap();
|
||||
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(&fake, perm).unwrap();
|
||||
let git_bin = fake.to_str().unwrap().to_string();
|
||||
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let path = dir.path().to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(list_all_objects(
|
||||
&path,
|
||||
&git_bin,
|
||||
Instant::now() + Duration::from_millis(150),
|
||||
));
|
||||
});
|
||||
let res = rx.recv_timeout(Duration::from_secs(10)).expect(
|
||||
"list_all_objects must return within the watchdog budget, not block on a hung git",
|
||||
);
|
||||
assert!(
|
||||
res.is_err(),
|
||||
"a hung git must make list_all_objects error out, not hang"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file),
|
||||
/// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are
|
||||
/// the remaining candidate-discovery execs (the common push path). Each must
|
||||
/// return within the watchdog budget on a hung git rather than block and pin the
|
||||
/// write permit. Revert any one to a bare `Command::output()` and its arm blocks
|
||||
/// past the recv budget (RED).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn delta_path_exec_fns_time_out_on_a_hung_git() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::time::{Duration, Instant};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
// Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken
|
||||
// test can't leak a permanent orphan or wedge the suite).
|
||||
let fake = dir.path().join("fakegit");
|
||||
std::fs::write(
|
||||
&fake,
|
||||
"#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n",
|
||||
)
|
||||
.unwrap();
|
||||
let mut perm = std::fs::metadata(&fake).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(&fake, perm).unwrap();
|
||||
let git_bin = fake.to_str().unwrap().to_string();
|
||||
let path = dir.path().to_path_buf();
|
||||
|
||||
// Run `f` on a thread and require it to RETURN (not block) within 10s.
|
||||
fn returns_within<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> T {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(f());
|
||||
});
|
||||
rx.recv_timeout(Duration::from_secs(10)).expect(
|
||||
"a delta-path exec fn must return within the watchdog budget, not block on a hung git",
|
||||
)
|
||||
}
|
||||
let dl = || Instant::now() + Duration::from_millis(150);
|
||||
let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
|
||||
|
||||
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
|
||||
assert!(
|
||||
returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(),
|
||||
"peeled_object_type must time out to None on a hung cat-file"
|
||||
);
|
||||
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
|
||||
assert!(
|
||||
returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(),
|
||||
"rev_list_delta must error out on a hung rev-list"
|
||||
);
|
||||
let (p, g, d) = (path.clone(), git_bin.clone(), dl());
|
||||
assert!(
|
||||
returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(),
|
||||
"list_all_objects_with_type must error out on a hung cat-file"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,8 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::PgPool;
|
||||
use sqlx::pool::PoolConnection;
|
||||
use sqlx::{PgPool, Postgres};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -30,6 +31,12 @@ pub struct RepoStore {
|
||||
/// Tracks repos already confirmed to exist in Tigris — avoids redundant
|
||||
/// HEAD checks and background uploads for repos we've already migrated.
|
||||
migrated: Arc<Mutex<HashSet<String>>>,
|
||||
/// Test-only seam: armed here, copied into every `RepoWriteGuard` this store
|
||||
/// hands out, so a test that only holds the `AppState` (not the guard) can
|
||||
/// still park `release` at its pre-unlock point. See
|
||||
/// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests.
|
||||
#[cfg(test)]
|
||||
pre_unlock_gate: Option<Arc<tokio::sync::Notify>>,
|
||||
}
|
||||
|
||||
impl RepoStore {
|
||||
@@ -40,15 +47,27 @@ impl RepoStore {
|
||||
tigris: None,
|
||||
pool,
|
||||
migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
pre_unlock_gate: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only: every guard from this store parks in `release` right before the
|
||||
/// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future
|
||||
/// while it is parked reproduces a client disconnect inside `release`.
|
||||
#[cfg(test)]
|
||||
pub fn with_pre_unlock_gate(mut self, gate: Arc<tokio::sync::Notify>) -> Self {
|
||||
self.pre_unlock_gate = Some(gate);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn new(repos_dir: PathBuf, tigris: Option<TigrisClient>, pool: PgPool) -> Self {
|
||||
Self {
|
||||
repos_dir,
|
||||
tigris,
|
||||
pool,
|
||||
migrated: Arc::new(Mutex::new(HashSet::new())),
|
||||
#[cfg(test)]
|
||||
pre_unlock_gate: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,13 +176,44 @@ impl RepoStore {
|
||||
let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?;
|
||||
let lock_key = advisory_lock_key(&owner_slug, repo_name);
|
||||
|
||||
// Acquire Postgres advisory lock with retry using pg_try_advisory_lock
|
||||
// to avoid blocking indefinitely on stale locks from crashed connections.
|
||||
// Pin a dedicated pooled connection and build the guard holding it BEFORE
|
||||
// issuing the lock query. Session-level pg advisory locks are
|
||||
// connection-affine (they can only be released on the session that took
|
||||
// them), so the guard must own the locking connection; and building the
|
||||
// guard first means any cancellation after the lock is taken — a
|
||||
// `tokio::time::timeout` firing during the Tigris download below — drops a
|
||||
// guard that CAN release, closing the leak the outer timeout otherwise
|
||||
// opened (#174 F1).
|
||||
let conn = self
|
||||
.pool
|
||||
.acquire()
|
||||
.await
|
||||
.context("acquiring db connection for the write advisory lock")?;
|
||||
let mut guard = RepoWriteGuard {
|
||||
owner_slug: owner_slug.clone(),
|
||||
repo_name: repo_name.to_string(),
|
||||
local_path: local_path.clone(),
|
||||
lock_key,
|
||||
conn: Some(conn),
|
||||
locked: false,
|
||||
released: false,
|
||||
tigris: self.tigris.clone(),
|
||||
#[cfg(test)]
|
||||
test_pre_unlock_gate: self.pre_unlock_gate.clone(),
|
||||
};
|
||||
|
||||
// Acquire the advisory lock with retry, through the guard's OWN connection,
|
||||
// so the matching unlock (in release, or the Drop backstop) runs on the same
|
||||
// session — pg_advisory_unlock on a different pooled connection is a no-op.
|
||||
let mut acquired = false;
|
||||
for attempt in 0..60 {
|
||||
let c = guard
|
||||
.conn
|
||||
.as_deref_mut()
|
||||
.expect("write guard holds its connection during acquisition");
|
||||
let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(lock_key)
|
||||
.fetch_one(&self.pool)
|
||||
.fetch_one(&mut *c)
|
||||
.await
|
||||
.context("trying advisory lock")?;
|
||||
if row.0 {
|
||||
@@ -177,9 +227,11 @@ impl RepoStore {
|
||||
if !acquired {
|
||||
anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}");
|
||||
}
|
||||
guard.locked = true;
|
||||
|
||||
// Always download the latest from Tigris before writing.
|
||||
// Local disk may be stale if another machine pushed since our last access.
|
||||
// Always download the latest from Tigris before writing. Local disk may be
|
||||
// stale if another machine pushed since our last access. The guard already
|
||||
// owns the lock + its connection, so a cancellation here drops through Drop.
|
||||
if let Some(ref tigris) = self.tigris {
|
||||
if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) {
|
||||
debug!(repo = %repo_name, "write acquire: downloading latest from tigris");
|
||||
@@ -197,14 +249,7 @@ impl RepoStore {
|
||||
}
|
||||
}
|
||||
|
||||
Ok(RepoWriteGuard {
|
||||
owner_slug,
|
||||
repo_name: repo_name.to_string(),
|
||||
local_path,
|
||||
lock_key,
|
||||
pool: self.pool.clone(),
|
||||
tigris: self.tigris.clone(),
|
||||
})
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
/// Initialize a new bare repo on local disk and upload to Tigris.
|
||||
@@ -479,8 +524,62 @@ pub struct RepoWriteGuard {
|
||||
repo_name: String,
|
||||
pub local_path: PathBuf,
|
||||
lock_key: i64,
|
||||
pool: PgPool,
|
||||
/// The pooled connection that took the advisory lock. Session-level pg
|
||||
/// advisory locks are connection-affine, so the guard pins that connection
|
||||
/// for its whole lifetime and unlocks on it (in `release`, or the `Drop`
|
||||
/// backstop). `None` only after the connection has been taken, either to run
|
||||
/// the detached unlock in `Drop` or to be closed when `release`'s unlock
|
||||
/// errored (#174 F3b).
|
||||
conn: Option<PoolConnection<Postgres>>,
|
||||
/// Set once the advisory lock has actually been taken. A guard dropped
|
||||
/// before the lock is held (or after `release`) performs no unlock.
|
||||
locked: bool,
|
||||
/// Set once `release` has run its unlock, making the `Drop` backstop inert.
|
||||
released: bool,
|
||||
tigris: Option<TigrisClient>,
|
||||
/// Test-only seam: when set, `release` parks on this gate at the exact point
|
||||
/// it is about to await `pg_advisory_unlock` (connection still owned, not yet
|
||||
/// released). Dropping the `release` future while it is parked reproduces a
|
||||
/// mid-unlock cancellation, so a test can assert the `Drop` backstop still
|
||||
/// frees the session lock. Never set outside tests.
|
||||
#[cfg(test)]
|
||||
test_pre_unlock_gate: Option<Arc<tokio::sync::Notify>>,
|
||||
}
|
||||
|
||||
/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`.
|
||||
/// Long enough that a healthy socket always finishes well inside it, short enough that
|
||||
/// a blackholed one does not pin admission resources for a TCP timeout.
|
||||
const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Await `close` under a deadline (#174 F3c).
|
||||
///
|
||||
/// `release` awaits this INLINE while the global write permit, the per-source permit
|
||||
/// and the write lease are all still held, and sqlx puts no deadline on `close()`:
|
||||
/// it writes Terminate and then tears the socket down. The branch that reaches here is
|
||||
/// by definition a connection whose last statement errored, and a blackholed TCP path
|
||||
/// to Postgres (a cloud failover that drops packets without an RST) is a plausible
|
||||
/// cause, so an unbounded await here parks every later push to the repo behind three
|
||||
/// pinned admission resources until the steal bound.
|
||||
///
|
||||
/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns.
|
||||
/// Dropping it closes the socket, and closing the socket is what actually ends the
|
||||
/// session and makes Postgres release the lock, so the deadline costs nothing the
|
||||
/// graceful path was buying.
|
||||
async fn close_conn_bounded(
|
||||
repo_name: &str,
|
||||
close: impl std::future::Future<Output = Result<(), sqlx::Error>>,
|
||||
) {
|
||||
match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
warn!(repo = %repo_name, err = %e,
|
||||
"closing the write-lock connection failed, the session teardown still frees the lock server-side");
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(),
|
||||
"closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RepoWriteGuard {
|
||||
@@ -494,7 +593,7 @@ impl RepoWriteGuard {
|
||||
/// half-applied or otherwise inconsistent repo would propagate corruption to
|
||||
/// Tigris (and to every node that later downloads it). The lock is always
|
||||
/// released regardless, to avoid stale locks blocking future writes.
|
||||
pub async fn release(self, success: bool) {
|
||||
pub async fn release(mut self, success: bool) {
|
||||
// Upload to Tigris only on success.
|
||||
if success {
|
||||
if let Some(ref tigris) = self.tigris {
|
||||
@@ -509,11 +608,124 @@ impl RepoWriteGuard {
|
||||
warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo");
|
||||
}
|
||||
|
||||
// Release advisory lock
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(self.lock_key)
|
||||
.execute(&self.pool)
|
||||
.await;
|
||||
// Release the advisory lock on the SAME connection that took it (session
|
||||
// advisory locks are connection-affine). Unlock through the connection
|
||||
// while it is STILL owned by `self` — do not `take()` it first. If this
|
||||
// future is cancelled during the unlock await, `self` is dropped with
|
||||
// `conn == Some(..)` and `released == false`, so the `Drop` backstop still
|
||||
// runs the detached unlock. `released` is set only AFTER the await
|
||||
// resolves, so a cancellation cannot make the backstop inert (#174 F4).
|
||||
if self.locked {
|
||||
#[cfg(test)]
|
||||
let pre_unlock_gate = self.test_pre_unlock_gate.clone();
|
||||
let unlock = if let Some(conn) = self.conn.as_deref_mut() {
|
||||
// Test-only: park right before the unlock await so a test can drop
|
||||
// this future mid-unlock (connection owned, not yet released).
|
||||
#[cfg(test)]
|
||||
if let Some(gate) = pre_unlock_gate {
|
||||
gate.notified().await;
|
||||
}
|
||||
Some(
|
||||
sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(self.lock_key)
|
||||
.execute(&mut *conn)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// An unlock that ERRORS is a different failure from a cancellation: the
|
||||
// await resolved, so `Drop` is about to be made inert by `released`
|
||||
// below, but the session is still alive and still holds the lock
|
||||
// (statement timeout, admin cancel, aborted transaction). Returning that
|
||||
// `PoolConnection` to the pool would hand the next caller a connection
|
||||
// holding a lock nobody tracks (#174 F3b). Connection disposal is the
|
||||
// single mechanism here, and it is why we do not instead try to keep the
|
||||
// `Drop` backstop armed: disposal needs `conn.take()`, and `Drop`
|
||||
// early-returns on `conn == None`. Ending the session is what frees the
|
||||
// lock, so `released = true` still holds.
|
||||
if let Some(Err(e)) = unlock {
|
||||
warn!(repo = %self.repo_name, err = %e,
|
||||
"advisory unlock failed, closing the connection so the session ends and postgres drops the lock");
|
||||
if let Some(conn) = self.conn.take() {
|
||||
// `close()` over `detach()`: both consume the `PoolConnection` by
|
||||
// value in sqlx 0.8.6, but we are in an async fn, so `close()`
|
||||
// sends Terminate and waits for the socket to go down before
|
||||
// `release` returns. `detach()` would only end the session
|
||||
// whenever the returned `PgConnection` is dropped and its
|
||||
// background close completes. If this future is cancelled during
|
||||
// `close()`, the connection is dropped mid-close, which still
|
||||
// tears the session down. That last point is also why the await is
|
||||
// safe to bound: see `close_conn_bounded`, which gives it the
|
||||
// deadline sqlx does not.
|
||||
close_conn_bounded(&self.repo_name, conn.close()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.released = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RepoWriteGuard {
|
||||
/// Cancellation-safe backstop: if the guard is dropped while still holding the
|
||||
/// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a
|
||||
/// handler future was dropped before `release`), unlock on the pinned
|
||||
/// connection. This is NOT the backstop for an unlock that ran and returned an
|
||||
/// error: that case is closed inside `release` by disposing of the connection,
|
||||
/// because `Drop` early-returns on `conn == None` and the two mechanisms cannot
|
||||
/// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the
|
||||
/// same session (connection-affine). An off-runtime drop has nothing to spawn onto,
|
||||
/// so it disposes of the connection instead. On runtime
|
||||
/// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock
|
||||
/// may not run — but shutdown tears down the pool, and closing the connection
|
||||
/// releases the session-level advisory lock server-side, so this too is bounded.
|
||||
fn drop(&mut self) {
|
||||
if self.released || !self.locked {
|
||||
return;
|
||||
}
|
||||
let Some(mut conn) = self.conn.take() else {
|
||||
return;
|
||||
};
|
||||
let lock_key = self.lock_key;
|
||||
let repo_name = self.repo_name.clone();
|
||||
match tokio::runtime::Handle::try_current() {
|
||||
Ok(handle) => {
|
||||
handle.spawn(async move {
|
||||
let unlock = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(lock_key)
|
||||
.execute(&mut *conn)
|
||||
.await;
|
||||
// Same failure as `release`'s (#174 F3b), one level down: the await
|
||||
// RESOLVED with an error, so the session is alive and still holds
|
||||
// the lock. Letting this async block end here would drop `conn` and
|
||||
// RETURN it to the pool, handing the next caller a connection
|
||||
// holding a lock nobody tracks. Close it instead, which both keeps
|
||||
// it out of the pool and ends the session that holds the lock.
|
||||
if let Err(e) = unlock {
|
||||
warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock");
|
||||
close_conn_bounded(&repo_name, conn.close()).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
// No runtime to spawn the unlock onto, and the connection is already
|
||||
// out of the guard, so there is no path that unlocks on this session.
|
||||
// Returning it to the pool would hand the next caller a connection
|
||||
// still holding the lock. `PoolConnection`'s own drop also spawns its
|
||||
// return-to-pool task, which panics with no runtime. `detach` gives up
|
||||
// the pool slot and yields a plain `PgConnection`; dropping that closes
|
||||
// the socket, which ends the session and is what frees the lock
|
||||
// server-side. `Drop` cannot await, so this is the whole disposal:
|
||||
// `close_conn_bounded` is not available here.
|
||||
drop(conn.detach());
|
||||
warn!(
|
||||
repo = %repo_name,
|
||||
"RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \
|
||||
possible, so the pinned connection is disposed of instead: ending \
|
||||
the session is what releases the advisory lock"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -932,4 +1144,616 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ──
|
||||
|
||||
/// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the
|
||||
/// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when
|
||||
/// it fires during the Tigris await — must still release the session advisory
|
||||
/// lock. A checker connection is held OUT of the pool first, so `acquire_write`
|
||||
/// is forced onto a distinct session; the checker (a different session) then
|
||||
/// probes the lock, so advisory-lock re-entrancy cannot mask a leak.
|
||||
///
|
||||
/// Load-bearing: RED today (no `Drop` releases the lock → held by
|
||||
/// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns
|
||||
/// false). GREEN after the connection-affine `Drop` backstop.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
|
||||
let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA";
|
||||
let name = "leaktest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
// Distinct session for the probe: hold it out of the pool BEFORE acquiring,
|
||||
// so acquire_write cannot use it and a re-entrant probe cannot falsely read free.
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
|
||||
let guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
// The cancellation shape: drop without release().
|
||||
drop(guard);
|
||||
// Let the detached unlock task run.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
||||
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
free,
|
||||
"advisory lock must be released when the guard is dropped without release()"
|
||||
);
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// F1 (latent non-affine release): `release()` must unlock on the SAME
|
||||
/// session that locked, so the lock is freed regardless of which pooled
|
||||
/// connection would service a fresh query. Observed from a distinct session.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
|
||||
let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB";
|
||||
let name = "affinetest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
let guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
guard.release(false).await;
|
||||
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
free,
|
||||
"release() must free the advisory lock via connection-affine unlock"
|
||||
);
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
// ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ──────────
|
||||
|
||||
/// F4 (P1): a cancellation DURING the unlock await must still free the session
|
||||
/// advisory lock. `release` unlocks through the connection while `self` still
|
||||
/// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some`
|
||||
/// + `locked && !released` and runs its detached-unlock backstop. A test-only
|
||||
/// gate parks `release` at the exact pre-unlock point; dropping the future
|
||||
/// there reproduces the cancellation.
|
||||
///
|
||||
/// Load-bearing: RED on the original ordering (`self.conn.take()` before the
|
||||
/// await → at cancellation `self.conn == None` → `Drop` skips → the local
|
||||
/// connection returns to the pool with the session lock still held → the
|
||||
/// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
|
||||
let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC";
|
||||
let name = "canceltest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
// Distinct session for the probe, held out of the pool before acquiring.
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
|
||||
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
// Arm the pre-unlock gate; it is never notified, so `release` parks on it
|
||||
// with the connection still owned and `released` still false.
|
||||
let gate = Arc::new(tokio::sync::Notify::new());
|
||||
guard.test_pre_unlock_gate = Some(gate);
|
||||
|
||||
// Box the future so we can drop it ourselves (tokio::pin! keeps it alive to
|
||||
// end of scope, which would defer the guard's Drop past the assertions).
|
||||
let mut fut = Box::pin(guard.release(false));
|
||||
let parked =
|
||||
tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await;
|
||||
assert!(
|
||||
parked.is_err(),
|
||||
"release should park on the pre-unlock gate, not complete"
|
||||
);
|
||||
// Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop.
|
||||
drop(fut);
|
||||
|
||||
// Let the Drop backstop's detached unlock task run.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
||||
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
free,
|
||||
"advisory lock must be freed when release is cancelled mid-unlock — the \
|
||||
connection must stay owned by the guard so Drop's backstop can run"
|
||||
);
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// F4: the ordinary success path still frees the lock, and a second write
|
||||
/// acquire on the same repo returns promptly (no stale-lock retry loop).
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
|
||||
let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD";
|
||||
let name = "reltruetest";
|
||||
|
||||
let guard = store
|
||||
.acquire_write(owner, name)
|
||||
.await
|
||||
.expect("first acquire");
|
||||
guard.release(true).await;
|
||||
|
||||
let again = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
store.acquire_write(owner, name),
|
||||
)
|
||||
.await
|
||||
.expect("second acquire_write must not hit the ~60s stale-lock retry loop")
|
||||
.expect("second acquire");
|
||||
again.release(true).await;
|
||||
}
|
||||
|
||||
// ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─
|
||||
|
||||
/// Put the guard's pinned connection into a failed-transaction state, so the
|
||||
/// next statement on it errors while the SESSION stays alive and keeps holding
|
||||
/// the session-level advisory lock (those survive a transaction abort; only
|
||||
/// `pg_advisory_xact_lock` would not). This is the smallest injection that
|
||||
/// reproduces F3b's shape: `pg_advisory_unlock` returning `Err` on a live,
|
||||
/// still-locking session. No production seam is needed because the tests live
|
||||
/// in this module and can reach `conn` directly.
|
||||
async fn poison_guard_connection(guard: &mut RepoWriteGuard) {
|
||||
let conn = guard
|
||||
.conn
|
||||
.as_deref_mut()
|
||||
.expect("guard holds its connection before release");
|
||||
sqlx::query("BEGIN")
|
||||
.execute(&mut *conn)
|
||||
.await
|
||||
.expect("open a transaction on the guard connection");
|
||||
let poisoned = sqlx::query("SELECT 1 / 0").execute(&mut *conn).await;
|
||||
assert!(
|
||||
poisoned.is_err(),
|
||||
"the poison statement must fail so the transaction is aborted"
|
||||
);
|
||||
}
|
||||
|
||||
/// How long the polls below may wait for an asynchronous server-side effect.
|
||||
/// Postgres frees a session advisory lock when the backend exits, which happens
|
||||
/// asynchronously to our socket close, so these are polled to a generous deadline
|
||||
/// rather than slept for a fixed interval: a constant sleep is a flake under load,
|
||||
/// and one that is long enough to be safe is dead time on every run.
|
||||
const RELEASE_POLL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
|
||||
/// Poll `cond` until it holds, failing at the deadline so a regression fails the
|
||||
/// test rather than hanging the suite.
|
||||
async fn wait_until(mut cond: impl FnMut() -> bool, what: &str) {
|
||||
let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE;
|
||||
while !cond() {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"timed out waiting for {what}"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll the advisory lock from `checker` until it is free. `pg_try_advisory_lock`
|
||||
/// ACQUIRES on success, so a true result both answers the question and leaves the
|
||||
/// checker session holding the lock; the caller unlocks it.
|
||||
async fn wait_until_lock_free(checker: &mut sqlx::PgConnection, key: i64, what: &str) {
|
||||
let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE;
|
||||
loop {
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
if free {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"timed out waiting for {what}"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A second pool over the same test database with the idle reaper DISABLED.
|
||||
///
|
||||
/// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to
|
||||
/// it is closed by the reaper about a second later, which ends the session and
|
||||
/// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside
|
||||
/// that window by luck; polling to a deadline long enough to be flake-proof would
|
||||
/// land outside it and go green whether or not `release` disposed of the
|
||||
/// connection, so the tests below would stop testing anything (measured: with the
|
||||
/// reaper in play, the disposal shows up at ~2s even with the fix reverted). With
|
||||
/// no reaper, `release` is the only thing that can end that session, so the poll
|
||||
/// measures exactly the property these two tests exist for.
|
||||
async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool {
|
||||
sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.idle_timeout(None)
|
||||
.max_lifetime(None)
|
||||
.connect_with(pool.connect_options().as_ref().clone())
|
||||
.await
|
||||
.expect("a second pool over the test database")
|
||||
}
|
||||
|
||||
/// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED.
|
||||
/// `release` awaits it inline while the global write permit, the per-source permit
|
||||
/// and the write lease are all still held, and sqlx's `close()` carries no deadline
|
||||
/// of its own, so a blackholed socket would park every later push to that repo
|
||||
/// behind three pinned admission resources.
|
||||
///
|
||||
/// What this covers: the deadline itself. A close that never resolves still lets
|
||||
/// `close_conn_bounded` return, which is the property `release` depends on. What it
|
||||
/// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is
|
||||
/// what stalls in production. Making a real `PgConnection::close` hang needs a
|
||||
/// blackholed TCP path to Postgres, and the flip has to land after the unlock
|
||||
/// statement round-trips but before the Terminate write, which is not a seam this
|
||||
/// module exposes. A never-resolving future is the faithful stand-in for that
|
||||
/// close, and the F3b tests above already cover that `release` really routes its
|
||||
/// close through here.
|
||||
///
|
||||
/// Time is paused, so nothing here depends on wall clock: the runtime auto-advances
|
||||
/// to the next timer, and the assertion is on which timer fired, not on elapsed
|
||||
/// time. The outer bound is what turns a removed deadline into a failure rather
|
||||
/// than a hung suite.
|
||||
///
|
||||
/// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the
|
||||
/// inner future never resolves, so the outer bound fires and this fails.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unlock_error_connection_close_is_bounded() {
|
||||
let hanging = std::future::pending::<Result<(), sqlx::Error>>();
|
||||
let outcome = tokio::time::timeout(
|
||||
UNLOCK_ERROR_CLOSE_TIMEOUT * 4,
|
||||
close_conn_bounded("boundedclosetest", hanging),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
outcome.is_ok(),
|
||||
"a connection close that never completes must not hold the write lease and \
|
||||
both admission permits open-endedly: close_conn_bounded must give up and \
|
||||
drop the connection"
|
||||
);
|
||||
}
|
||||
|
||||
/// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive
|
||||
/// (statement timeout, admin cancel, aborted transaction), the lock must not
|
||||
/// survive `release`. The old code discarded the error with `let _ =` and set
|
||||
/// `released = true` anyway, so `Drop` early-returned and the `PoolConnection`
|
||||
/// went back to the pool still holding the session lock.
|
||||
///
|
||||
/// Observed from a SEPARATE connection held out of the pool before acquiring:
|
||||
/// session advisory locks are re-entrant and counted, so probing from the
|
||||
/// holding session (or via a fresh `acquire_write` that may be handed the same
|
||||
/// connection) would report free whether or not the fix is present.
|
||||
///
|
||||
/// Load-bearing: RED before the fix (lock still held → `pg_try_advisory_lock`
|
||||
/// returns false), GREEN after (the errored connection is closed, so the
|
||||
/// session ends and Postgres drops the lock).
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store_pool = pool_without_idle_reaper(&pool).await;
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
|
||||
let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF";
|
||||
let name = "unlockerrtest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
// Distinct session for the probe, held out of the pool before acquiring.
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
|
||||
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
poison_guard_connection(&mut guard).await;
|
||||
|
||||
// Sanity: the poisoned session is still alive and still holds the lock, so
|
||||
// the assertion below measures the release path and not a dead session.
|
||||
let (free_before,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!free_before,
|
||||
"the poisoned session must still hold the lock before release"
|
||||
);
|
||||
|
||||
guard.release(false).await;
|
||||
|
||||
// Postgres drops the lock when the disposed session's backend exits, which is
|
||||
// asynchronous to our socket close: poll for it rather than sleeping a
|
||||
// constant. The deadline is what keeps this load-bearing: a release that
|
||||
// leaves the lock held never satisfies the probe and fails here.
|
||||
wait_until_lock_free(
|
||||
&mut checker,
|
||||
key,
|
||||
"an errored pg_advisory_unlock must not leave the lock held: release must \
|
||||
dispose of the connection so the session ends",
|
||||
)
|
||||
.await;
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// F3b: the connection that saw the unlock error must leave the pool entirely,
|
||||
/// rather than being handed to the next caller while still holding the lock.
|
||||
/// `PgPool::size()` counts the connections the pool owns, so closing the
|
||||
/// errored one is observable as a drop in that count.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_with_failing_unlock_does_not_return_the_connection(
|
||||
pool: sqlx::PgPool,
|
||||
) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store_pool = pool_without_idle_reaper(&pool).await;
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
|
||||
let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG";
|
||||
let name = "unlockerrpooltest";
|
||||
|
||||
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
poison_guard_connection(&mut guard).await;
|
||||
let size_before = store_pool.size();
|
||||
assert!(size_before > 0, "the pool owns the guard's connection");
|
||||
|
||||
guard.release(false).await;
|
||||
|
||||
// The pool's size drops when the closed connection's slot is given up, which
|
||||
// is not synchronous with `release` returning: poll rather than sleep.
|
||||
wait_until(
|
||||
|| store_pool.size() == size_before - 1,
|
||||
"the connection that saw the unlock error to be closed rather than returned \
|
||||
to the pool still holding the session lock",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// F3b regression guard on the success path: a normal unlock keeps the
|
||||
/// connection in the pool and marks the guard released, so the disposal branch
|
||||
/// is confined to the error case.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_success_keeps_the_connection_and_frees_the_lock(
|
||||
pool: sqlx::PgPool,
|
||||
) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone());
|
||||
let owner = "did:key:z6MkUnlockOkProofHHHHHHHHHHHHHHHHHHHHHHHH";
|
||||
let name = "unlockoktest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
let guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
let size_before = pool.size();
|
||||
|
||||
guard.release(false).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
||||
|
||||
assert_eq!(
|
||||
pool.size(),
|
||||
size_before,
|
||||
"a successful unlock must leave the connection in the pool"
|
||||
);
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(free, "the success path must still free the lock");
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
// ── the Drop backstop disposes its connection too (#174 U8) ─────────────
|
||||
|
||||
/// U8 (P2): the `Drop` backstop carries the same hazard F3b closed in `release`.
|
||||
/// When the detached `pg_advisory_unlock` ERRORS on a live session, the async
|
||||
/// block ends and drops the moved `PoolConnection`, which RETURNS it to the pool
|
||||
/// while that session may still hold the lock, handing the next caller a
|
||||
/// connection holding a lock nobody tracks. The errored connection must be closed
|
||||
/// instead: that keeps it out of the pool and ends the session, which is what
|
||||
/// frees the lock server-side.
|
||||
///
|
||||
/// Run against a pool with the idle reaper disabled, for the reason spelled out on
|
||||
/// `pool_without_idle_reaper`: with the reaper in play the session dies on its own
|
||||
/// about a second later and the assertion stops measuring the disposal.
|
||||
///
|
||||
/// Load-bearing: RED before the fix (the connection goes back to the pool, so the
|
||||
/// size never drops and this times out), GREEN after.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_drop_with_failing_unlock_does_not_return_the_connection(
|
||||
pool: sqlx::PgPool,
|
||||
) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store_pool = pool_without_idle_reaper(&pool).await;
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
|
||||
let owner = "did:key:z6MkDropUnlockErrProofIIIIIIIIIIIIIIIIIIII";
|
||||
let name = "dropunlockerrtest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
// Distinct session for the probe, held out of the store's pool entirely.
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
|
||||
let mut guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
poison_guard_connection(&mut guard).await;
|
||||
let size_before = store_pool.size();
|
||||
assert!(size_before > 0, "the pool owns the guard's connection");
|
||||
|
||||
// The backstop shape: dropped without release(), with an unlock that errors.
|
||||
drop(guard);
|
||||
|
||||
wait_until(
|
||||
|| store_pool.size() == size_before - 1,
|
||||
"the connection whose detached unlock errored to be closed rather than \
|
||||
returned to the pool still holding the session lock",
|
||||
)
|
||||
.await;
|
||||
wait_until_lock_free(
|
||||
&mut checker,
|
||||
key,
|
||||
"an errored detached unlock must not leave the lock held: Drop must dispose \
|
||||
of the connection so the session ends",
|
||||
)
|
||||
.await;
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// U8 regression guard on the success path: a detached unlock that SUCCEEDS must
|
||||
/// still return the connection to the pool. Without this, "close the connection on
|
||||
/// Drop" could be widened to "always close" and the test above would not notice.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store_pool = pool_without_idle_reaper(&pool).await;
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
|
||||
let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ";
|
||||
let name = "dropunlockoktest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
let guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
let size_before = store_pool.size();
|
||||
assert!(size_before > 0, "the pool owns the guard's connection");
|
||||
|
||||
drop(guard);
|
||||
|
||||
// The connection goes back only once the detached unlock task has finished.
|
||||
wait_until(
|
||||
|| store_pool.num_idle() > 0,
|
||||
"the detached unlock to finish and hand the connection back",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
store_pool.size(),
|
||||
size_before,
|
||||
"a successful detached unlock must leave the connection in the pool"
|
||||
);
|
||||
wait_until_lock_free(
|
||||
&mut checker,
|
||||
key,
|
||||
"the Drop backstop's successful unlock to free the lock",
|
||||
)
|
||||
.await;
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the
|
||||
/// unlock onto, and the connection has already been taken out of the guard, so the
|
||||
/// old code dropped it with no unlock attempted at all, back to the pool, session
|
||||
/// lock still held. Dropping a `PoolConnection` off a runtime is worse than that:
|
||||
/// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the
|
||||
/// old arm also panicked in a destructor.
|
||||
///
|
||||
/// Reached by dropping the guard on a plain `std::thread`, where
|
||||
/// `Handle::try_current()` fails.
|
||||
///
|
||||
/// Load-bearing: RED before the fix (the join sees the "requires a Tokio context"
|
||||
/// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the
|
||||
/// pool slot, so nothing is spawned and dropping the detached connection closes
|
||||
/// the socket, which ends the session and frees the lock).
|
||||
#[sqlx::test]
|
||||
async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let store_pool = pool_without_idle_reaper(&pool).await;
|
||||
let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone());
|
||||
let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK";
|
||||
let name = "dropoffruntimetest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
let mut checker = pool.acquire().await.expect("checker connection");
|
||||
let guard = store.acquire_write(owner, name).await.expect("acquire");
|
||||
let size_before = store_pool.size();
|
||||
assert!(size_before > 0, "the pool owns the guard's connection");
|
||||
|
||||
let dropped = std::thread::spawn(move || drop(guard)).join();
|
||||
assert!(
|
||||
dropped.is_ok(),
|
||||
"dropping a write guard off a Tokio runtime must not panic"
|
||||
);
|
||||
|
||||
wait_until(
|
||||
|| store_pool.size() == size_before - 1,
|
||||
"the connection of a guard dropped off a runtime to be disposed of rather \
|
||||
than returned to the pool with no unlock attempted",
|
||||
)
|
||||
.await;
|
||||
wait_until_lock_free(
|
||||
&mut checker,
|
||||
key,
|
||||
"a guard dropped off a runtime to end its session so postgres drops the lock",
|
||||
)
|
||||
.await;
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// F4: releasing a guard that never took the lock (`locked == false`, the state
|
||||
/// acquire_write leaves after a failed acquisition) must not unlock or panic.
|
||||
#[sqlx::test]
|
||||
async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE";
|
||||
let name = "notlockedtest";
|
||||
let slug = owner.replace([':', '/'], "_");
|
||||
let key = advisory_lock_key(&slug, name);
|
||||
|
||||
let guard = RepoWriteGuard {
|
||||
owner_slug: slug,
|
||||
repo_name: name.to_string(),
|
||||
local_path: dir.path().to_path_buf(),
|
||||
lock_key: key,
|
||||
conn: Some(pool.acquire().await.expect("conn")),
|
||||
locked: false,
|
||||
released: false,
|
||||
tigris: None,
|
||||
test_pre_unlock_gate: None,
|
||||
};
|
||||
// Must complete without panic and issue no unlock.
|
||||
guard.release(false).await;
|
||||
|
||||
let mut checker = pool.acquire().await.expect("checker");
|
||||
let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(key)
|
||||
.fetch_one(&mut *checker)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(free, "release on an unlocked guard must not touch the key");
|
||||
let _ = sqlx::query("SELECT pg_advisory_unlock($1)")
|
||||
.bind(key)
|
||||
.execute(&mut *checker)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -31,6 +31,25 @@ impl TigrisClient {
|
||||
})
|
||||
}
|
||||
|
||||
/// Test-only constructor with an explicit S3 endpoint, region, and static
|
||||
/// credentials — no env-var reads, so parallel tests cannot race each other's
|
||||
/// `AWS_*` environment the way the env-based `new` would. Lets a test point
|
||||
/// the client at a non-routable endpoint to exercise acquire-stall paths.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self {
|
||||
let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test");
|
||||
let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
|
||||
.endpoint_url(endpoint_url)
|
||||
.region(aws_config::Region::new("auto"))
|
||||
.credentials_provider(creds)
|
||||
.load()
|
||||
.await;
|
||||
Self {
|
||||
s3: S3Client::new(&config),
|
||||
bucket: bucket.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst`
|
||||
fn repo_key(owner_slug: &str, repo_name: &str) -> String {
|
||||
format!("repos/v1/{owner_slug}/{repo_name}.tar.zst")
|
||||
|
||||
@@ -4,11 +4,321 @@
|
||||
//! content is held back.
|
||||
|
||||
use crate::db::VisibilityRule;
|
||||
use crate::git::store;
|
||||
use crate::visibility::{visibility_check, Decision};
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::process::Stdio;
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3).
|
||||
/// The walk is fast for a real repo; this bound exists to reap a hung or
|
||||
/// pathologically slow git child so it cannot pin a served-git permit (the read
|
||||
/// permit on the upload-pack serve path, the write permit on the receive-pack
|
||||
/// post-push replication path) past the deadline. Every caller funnels through
|
||||
/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers
|
||||
/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this
|
||||
/// fixed budget only backs the `git_bin`-less test wrappers.
|
||||
#[cfg(test)]
|
||||
const WALK_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
|
||||
/// How long the process-group watchdog waits after SIGTERM before escalating to
|
||||
/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only
|
||||
/// paid on a timeout (already the exceptional path).
|
||||
#[cfg(unix)]
|
||||
const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Run one git child under a shared `deadline` with process-group teardown,
|
||||
/// BLOCKING, and return its stdout. The child runs in its own process group; a
|
||||
/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs,
|
||||
/// the whole group if the deadline passes before the child is reaped, so a hung or
|
||||
/// slow git can pin neither a served-git permit nor a blocking thread past the
|
||||
/// deadline (jatmn's "retain admission until they are reaped"). This is the
|
||||
/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the
|
||||
/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot
|
||||
/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so
|
||||
/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can
|
||||
/// drive the teardown in tests without mutating the process-global PATH;
|
||||
/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children).
|
||||
/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT
|
||||
/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child
|
||||
/// waitable, so the caller's later `child.wait()` still collects the status and the
|
||||
/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)`
|
||||
/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually
|
||||
/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a).
|
||||
#[cfg(unix)]
|
||||
fn child_terminated_without_reaping(pid: i32) -> bool {
|
||||
// SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory;
|
||||
// WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking.
|
||||
let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe {
|
||||
libc::waitid(
|
||||
libc::P_PID,
|
||||
pid as libc::id_t,
|
||||
&mut info,
|
||||
libc::WEXITED | libc::WNOHANG | libc::WNOWAIT,
|
||||
)
|
||||
};
|
||||
// rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero
|
||||
// si_pid means the child has entered a waitable, exited state. EINTR/other errors
|
||||
// (rc != 0) are treated as "not yet terminated" and the caller re-polls.
|
||||
rc == 0 && unsafe { info.si_pid() } != 0
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn run_bounded_git_raw(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::sync::mpsc::RecvTimeoutError;
|
||||
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let mut child = std::process::Command::new(git_bin)
|
||||
.args(args)
|
||||
.current_dir(repo_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.process_group(0)
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn git {label}"))?;
|
||||
// With process_group(0) the child leads its own group, so pgid == its pid.
|
||||
let pgid = child.id() as i32;
|
||||
|
||||
// Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace
|
||||
// for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL
|
||||
// SIGKILL of the group. It never stands down on leader-reap alone: a group member
|
||||
// that ignores SIGTERM while the leader exits cleanly would otherwise escape the
|
||||
// SIGKILL and keep running past the deadline (finding 3, #174). The main thread
|
||||
// defers reaping the leader until this thread returns (see below), so the leader's
|
||||
// pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have
|
||||
// been recycled — which is why this no longer needs the old `reaped` short-circuit.
|
||||
// Kept off the main thread because the main thread's stdout drain is exactly what
|
||||
// blocks until a hung child is torn down.
|
||||
let (done_tx, done_rx) = mpsc::channel::<()>();
|
||||
let watchdog = std::thread::spawn(move || -> bool {
|
||||
let wait = deadline.saturating_duration_since(Instant::now());
|
||||
match done_rx.recv_timeout(wait) {
|
||||
Ok(()) | Err(RecvTimeoutError::Disconnected) => false,
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
// SAFETY: kill(2) takes only integers and borrows no Rust memory;
|
||||
// ESRCH on an already-gone group is ignored.
|
||||
unsafe { libc::kill(-pgid, libc::SIGTERM) };
|
||||
// Fixed grace: because the main thread defers the leader's reap, a
|
||||
// fully-exited group still shows a zombie leader here, so polling for
|
||||
// ESRCH cannot detect early completion — just wait the grace, then
|
||||
// SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op;
|
||||
// on a SIGTERM-ignoring member it is what actually kills it.
|
||||
std::thread::sleep(WATCHDOG_TERM_GRACE);
|
||||
unsafe { libc::kill(-pgid, libc::SIGKILL) };
|
||||
// Brief settle so the SIGKILL is delivered before the main thread
|
||||
// reaps the leader and frees the pgid. A wedged (D-state) member
|
||||
// survives even SIGKILL — the documented residual, as in smart_http.
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
if unsafe { libc::kill(-pgid, 0) } == 0 {
|
||||
tracing::warn!(
|
||||
pgid,
|
||||
"withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Feed stdin on a writer thread and drain stderr on a reader thread so the main
|
||||
// thread can drain stdout concurrently; writing all of stdin (or draining one
|
||||
// pipe) before the others can deadlock once a pipe buffer fills.
|
||||
let mut stdin = child.stdin.take();
|
||||
let input = stdin_bytes.to_vec();
|
||||
let writer = std::thread::spawn(move || {
|
||||
if let Some(mut s) = stdin.take() {
|
||||
let _ = s.write_all(&input);
|
||||
}
|
||||
});
|
||||
let mut stderr = child.stderr.take().context("git stderr was not piped")?;
|
||||
let err_reader = std::thread::spawn(move || {
|
||||
let mut err = Vec::new();
|
||||
let _ = stderr.read_to_end(&mut err);
|
||||
err
|
||||
});
|
||||
let mut stdout = child.stdout.take().context("git stdout was not piped")?;
|
||||
let mut out = Vec::new();
|
||||
// Blocking drain, unblocked by the child closing stdout on exit. The watchdog's
|
||||
// SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible
|
||||
// (D-state) I/O survives even SIGKILL, so this drain and the wait below can block
|
||||
// until the kernel returns, pinning the walk thread and its permit. That residual
|
||||
// is unreachable in userspace (no signal reaps a D-state process) and matches the
|
||||
// async `reap_group_on_timeout`, which likewise only warns and gives up there.
|
||||
let read_result = stdout.read_to_end(&mut out);
|
||||
// The drain has returned, but that only means all stdout write ends are closed —
|
||||
// NOT that the child has exited. A group member, or the leader itself, can close
|
||||
// stdout and keep running; standing the watchdog down on the drain alone (as the
|
||||
// old code did) would then let `child.wait()` block forever on that live child,
|
||||
// past the deadline, pinning the walk thread and its permit (finding P1-a, #174).
|
||||
// So stand the watchdog down only once the child has ACTUALLY terminated, detected
|
||||
// WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its
|
||||
// pgid un-recycled until the watchdog finishes and we join it below. Past the
|
||||
// deadline the watchdog owns the teardown, so we stop polling and let it run the
|
||||
// full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every
|
||||
// `kill(-pgid)` firing while the pid is still unreaped and guarantees a
|
||||
// stdout-closing-then-hanging member has been SIGKILLed rather than left running.
|
||||
loop {
|
||||
if child_terminated_without_reaping(pgid) {
|
||||
let _ = done_tx.send(());
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(2));
|
||||
}
|
||||
let killed = watchdog.join().unwrap_or(false);
|
||||
let status = child.wait().context("git wait failed")?;
|
||||
let err = err_reader.join().unwrap_or_default();
|
||||
let _ = writer.join();
|
||||
read_result.context("failed to read git stdout")?;
|
||||
// The watchdog runs off a wall clock that can race a child finishing right at the
|
||||
// deadline. A child that exited on its own (success) is not a timeout even if the
|
||||
// watchdog fired late; only a child that did not exit successfully is a genuine
|
||||
// timeout, which keeps a walk completing at its budget from a spurious 504.
|
||||
if killed && !status.success() {
|
||||
return Err(crate::git::smart_http::GitServiceTimeout.into());
|
||||
}
|
||||
Ok((status, out, err))
|
||||
}
|
||||
|
||||
/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin
|
||||
/// wrapper the walk callers use. Probes that must distinguish exit classes —
|
||||
/// `git cat-file` absence vs an object-store access failure — call
|
||||
/// [`run_bounded_git_raw`] and classify the status/stderr themselves.
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn run_bounded_git(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<u8>> {
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets
|
||||
/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only),
|
||||
/// so this bounds a single child on its own: threads feed stdin and drain stderr
|
||||
/// while the main thread drains stdout, and a watchdog thread kills the child at the
|
||||
/// deadline (which closes stdout and unblocks the drain). The child is shared with
|
||||
/// the watchdog behind a mutex that the main thread does NOT hold while draining, so
|
||||
/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct
|
||||
/// child, not a descendant group — which is why the hardened, group-aware path above
|
||||
/// is gated to Unix, the only target the served node actually runs on (the Windows
|
||||
/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with
|
||||
/// the Unix version's signature and result semantics so every caller compiles on all
|
||||
/// targets (#174).
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) fn run_bounded_git_raw(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::mpsc::RecvTimeoutError;
|
||||
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let mut child = std::process::Command::new(git_bin)
|
||||
.args(args)
|
||||
.current_dir(repo_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn git {label}"))?;
|
||||
|
||||
let mut stdin = child.stdin.take();
|
||||
let input = stdin_bytes.to_vec();
|
||||
let writer = std::thread::spawn(move || {
|
||||
if let Some(mut s) = stdin.take() {
|
||||
let _ = s.write_all(&input);
|
||||
}
|
||||
});
|
||||
let mut stderr = child.stderr.take().context("git stderr was not piped")?;
|
||||
let err_reader = std::thread::spawn(move || {
|
||||
let mut err = Vec::new();
|
||||
let _ = stderr.read_to_end(&mut err);
|
||||
err
|
||||
});
|
||||
let mut stdout = child.stdout.take().context("git stdout was not piped")?;
|
||||
|
||||
// Share the child with the watchdog. The main thread drains stdout WITHOUT
|
||||
// holding this lock, so the watchdog can always acquire it to kill on timeout;
|
||||
// killing closes stdout and unblocks the drain below.
|
||||
let child = std::sync::Arc::new(std::sync::Mutex::new(child));
|
||||
let (done_tx, done_rx) = mpsc::channel::<()>();
|
||||
let watchdog = {
|
||||
let child = child.clone();
|
||||
std::thread::spawn(move || -> bool {
|
||||
let wait = deadline.saturating_duration_since(Instant::now());
|
||||
match done_rx.recv_timeout(wait) {
|
||||
Ok(()) | Err(RecvTimeoutError::Disconnected) => false,
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Ok(mut c) = child.lock() {
|
||||
let _ = c.kill();
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
let read_result = stdout.read_to_end(&mut out);
|
||||
// The drain has returned (child exited or was killed), so taking the lock here
|
||||
// cannot deadlock against the watchdog.
|
||||
let status = child
|
||||
.lock()
|
||||
.expect("git child mutex poisoned")
|
||||
.wait()
|
||||
.context("git wait failed")?;
|
||||
let _ = done_tx.send(());
|
||||
let killed = watchdog.join().unwrap_or(false);
|
||||
let err = err_reader.join().unwrap_or_default();
|
||||
let _ = writer.join();
|
||||
read_result.context("failed to read git stdout")?;
|
||||
if killed && !status.success() {
|
||||
return Err(crate::git::smart_http::GitServiceTimeout.into());
|
||||
}
|
||||
Ok((status, out, err))
|
||||
}
|
||||
|
||||
/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics.
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) fn run_bounded_git(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<u8>> {
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Fail closed unless every ref ultimately resolves to a commit (a ref pointing
|
||||
/// directly at a blob or tree, or an annotated tag — even a nested one — of such
|
||||
@@ -21,19 +331,15 @@ use std::path::Path;
|
||||
/// Full peeling is why this is not `for-each-ref %(*objecttype)`, which
|
||||
/// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a-
|
||||
/// commit as a non-commit.
|
||||
fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
|
||||
let refs = std::process::Command::new("git")
|
||||
.args(["for-each-ref", "--format=%(refname)"])
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("git for-each-ref failed")?;
|
||||
if !refs.status.success() {
|
||||
anyhow::bail!(
|
||||
"git for-each-ref failed: {}",
|
||||
String::from_utf8_lossy(&refs.stderr)
|
||||
);
|
||||
}
|
||||
let refs_stdout = String::from_utf8_lossy(&refs.stdout);
|
||||
fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> {
|
||||
let refs_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["for-each-ref", "--format=%(refname)"],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
let refs_stdout = String::from_utf8_lossy(&refs_out);
|
||||
let refnames: Vec<&str> = refs_stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
@@ -43,59 +349,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Peel every ref in one `git cat-file --batch-check` pass: one
|
||||
// `<refname>^{}` query per line, one output line per input line, in order.
|
||||
// The stdin write runs on a separate thread so this thread can drain stdout
|
||||
// concurrently. cat-file echoes the full query on a `<query> missing` line,
|
||||
// so output scales with refname length (not a fixed size per ref); writing
|
||||
// all of stdin before reading any stdout would deadlock both pipes once the
|
||||
// child's stdout buffer fills. Dropping `stdin` at the end of the closure
|
||||
// sends EOF.
|
||||
// Peel every ref in one `git cat-file --batch-check` pass: one `<refname>^{}`
|
||||
// query per line, one output line per input line, in order. cat-file echoes the
|
||||
// full query on a `<query> missing` line, so output scales with refname length;
|
||||
// run_bounded_git drains stdout concurrently with the stdin write, so the pipe
|
||||
// cannot deadlock, and the whole peel is bounded by the shared walk deadline.
|
||||
let queries = refnames
|
||||
.iter()
|
||||
.map(|r| format!("{r}^{{}}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
use std::io::Write;
|
||||
let mut child = std::process::Command::new("git")
|
||||
.args(["cat-file", "--batch-check=%(objecttype)"])
|
||||
.current_dir(repo_path)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.context("failed to spawn git cat-file")?;
|
||||
// Feed stdin on a writer thread so this thread can drain stdout via
|
||||
// wait_with_output concurrently; a None handle (the pipe vanished) becomes a
|
||||
// broken-pipe write error. wait_with_output reaps the child unconditionally
|
||||
// before any error is surfaced, so no path drops it unwaited (#53), and the
|
||||
// writer is joined only after the drain so the join cannot deadlock.
|
||||
let writer = child
|
||||
.stdin
|
||||
.take()
|
||||
.map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes())));
|
||||
let peel_result = child.wait_with_output();
|
||||
let write_result = match writer {
|
||||
Some(handle) => handle
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?,
|
||||
None => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::BrokenPipe,
|
||||
"git cat-file stdin unavailable",
|
||||
)),
|
||||
};
|
||||
// Surface a write error only if the process didn't already fail with a
|
||||
// clearer status.
|
||||
let peel = peel_result.context("git cat-file failed")?;
|
||||
if !peel.status.success() {
|
||||
anyhow::bail!(
|
||||
"git cat-file --batch-check failed: {}",
|
||||
String::from_utf8_lossy(&peel.stderr)
|
||||
);
|
||||
}
|
||||
write_result.context("failed to write to git cat-file stdin")?;
|
||||
let peel_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["cat-file", "--batch-check=%(objecttype)"],
|
||||
repo_path,
|
||||
queries.as_bytes(),
|
||||
deadline,
|
||||
)?;
|
||||
|
||||
let peel_stdout = String::from_utf8_lossy(&peel.stdout);
|
||||
let peel_stdout = String::from_utf8_lossy(&peel_out);
|
||||
let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect();
|
||||
// A short read means at least one ref went unclassified — fail closed.
|
||||
if types.len() != refnames.len() {
|
||||
@@ -147,47 +419,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> {
|
||||
/// Fails closed: if commit enumeration or any tree walk fails, returns an error so
|
||||
/// the caller aborts the serve/pin rather than producing a partial (under-withheld)
|
||||
/// set.
|
||||
fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
|
||||
assert_all_refs_are_commits(repo_path)?;
|
||||
fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result<Vec<(String, String)>> {
|
||||
// One deadline spans the whole walk (the ref check, the HEAD probe, rev-list,
|
||||
// and every per-commit ls-tree), so a slow or hung walk is bounded as a unit
|
||||
// rather than granting each git child a fresh timeout.
|
||||
let deadline = Instant::now() + timeout;
|
||||
assert_all_refs_are_commits(repo_path, git_bin, deadline)?;
|
||||
|
||||
// Enumerate every reachable commit, not just ref tips. `--all` walks all refs;
|
||||
// append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no
|
||||
// ref) is still classified. When HEAD does not resolve (unborn branch on an
|
||||
// empty repo) `--all` alone yields nothing, which is correct — no objects exist.
|
||||
let head = store::head_commit(repo_path).context("resolve HEAD failed")?;
|
||||
// empty repo) `--all` alone yields nothing, which is correct: no objects exist.
|
||||
// The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means
|
||||
// HEAD resolves), replacing the previously unbounded `store::head_commit` child.
|
||||
let head_resolves = run_bounded_git(
|
||||
git_bin,
|
||||
&["rev-parse", "--verify", "HEAD"],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)
|
||||
.is_ok();
|
||||
let mut rev_args = vec!["rev-list", "--all"];
|
||||
if head.is_some() {
|
||||
if head_resolves {
|
||||
rev_args.push("HEAD");
|
||||
}
|
||||
let commits = std::process::Command::new("git")
|
||||
.args(&rev_args)
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("git rev-list --all failed")?;
|
||||
if !commits.status.success() {
|
||||
anyhow::bail!(
|
||||
"git rev-list --all failed: {}",
|
||||
String::from_utf8_lossy(&commits.stderr)
|
||||
);
|
||||
}
|
||||
let commits_stdout = String::from_utf8_lossy(&commits.stdout);
|
||||
let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?;
|
||||
let commits_stdout = String::from_utf8_lossy(&commits_out);
|
||||
let mut out: HashSet<(String, String)> = HashSet::new();
|
||||
for commit in commits_stdout.lines() {
|
||||
let commit = commit.trim();
|
||||
if commit.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let listing = std::process::Command::new("git")
|
||||
.args(["ls-tree", "-rz", commit])
|
||||
.current_dir(repo_path)
|
||||
.output()
|
||||
.context("git ls-tree -rz failed")?;
|
||||
if !listing.status.success() {
|
||||
anyhow::bail!(
|
||||
"git ls-tree -rz {commit} failed: {}",
|
||||
String::from_utf8_lossy(&listing.stderr)
|
||||
);
|
||||
}
|
||||
let listing_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["ls-tree", "-rz", commit],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
// `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r`
|
||||
// C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes
|
||||
// "secret/caf\303\251.txt"), and that quoted literal would not match a
|
||||
@@ -198,7 +469,7 @@ fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
|
||||
// path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string
|
||||
// would no longer match its deny rule — the same under-withholding class, one
|
||||
// layer down. Fail closed instead so the caller aborts rather than leaks.
|
||||
let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else {
|
||||
let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else {
|
||||
anyhow::bail!(
|
||||
"git ls-tree -rz {commit} returned a non-UTF-8 path; \
|
||||
refusing to produce a partial (under-withheld) set"
|
||||
@@ -229,6 +500,7 @@ fn blob_paths(repo_path: &Path) -> Result<Vec<(String, String)>> {
|
||||
///
|
||||
/// The whole-repo "/" gate is handled by the caller before this function runs:
|
||||
/// if "/" denies, the caller gets a 404 and never reaches the filtered serve.
|
||||
#[cfg(test)]
|
||||
pub fn withheld_blob_oids(
|
||||
repo_path: &Path,
|
||||
rules: &[VisibilityRule],
|
||||
@@ -236,7 +508,33 @@ pub fn withheld_blob_oids(
|
||||
owner_did: &str,
|
||||
caller: Option<&str>,
|
||||
) -> Result<HashSet<String>> {
|
||||
let pairs = blob_paths(repo_path)?;
|
||||
withheld_blob_oids_bounded(
|
||||
repo_path,
|
||||
"git",
|
||||
WALK_TIMEOUT,
|
||||
rules,
|
||||
is_public,
|
||||
owner_did,
|
||||
caller,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served
|
||||
/// handlers call this with the operator-configured git binary and
|
||||
/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same
|
||||
/// budget as the other served-git ops and a fake `git` can drive its teardown in
|
||||
/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the
|
||||
/// classification tests that run against real git.
|
||||
pub fn withheld_blob_oids_bounded(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
timeout: Duration,
|
||||
rules: &[VisibilityRule],
|
||||
is_public: bool,
|
||||
owner_did: &str,
|
||||
caller: Option<&str>,
|
||||
) -> Result<HashSet<String>> {
|
||||
let pairs = blob_paths(repo_path, git_bin, timeout)?;
|
||||
Ok(withheld_from_pairs(
|
||||
&pairs, rules, is_public, owner_did, caller,
|
||||
))
|
||||
@@ -304,6 +602,7 @@ pub fn replicable_objects(all: Vec<String>, withheld: &HashSet<String>) -> Vec<S
|
||||
/// at an allowed path is included even when also denied elsewhere (its content
|
||||
/// is public elsewhere). A dangling blob is absent from the reachable walk, so
|
||||
/// it is never in this set and the fail-closed filter drops it (#99).
|
||||
#[cfg(test)]
|
||||
pub fn replicable_blob_set(
|
||||
repo_path: &Path,
|
||||
rules: &[VisibilityRule],
|
||||
@@ -313,6 +612,21 @@ pub fn replicable_blob_set(
|
||||
allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None)
|
||||
}
|
||||
|
||||
/// [`replicable_blob_set`] with an injectable `git_bin` and walk `timeout`, for the
|
||||
/// fail-closed full-scan pin path on the receive-pack side.
|
||||
pub fn replicable_blob_set_bounded(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
timeout: Duration,
|
||||
rules: &[VisibilityRule],
|
||||
is_public: bool,
|
||||
owner_did: &str,
|
||||
) -> Result<HashSet<String>> {
|
||||
allowed_blob_set_for_caller_bounded(
|
||||
repo_path, git_bin, timeout, rules, is_public, owner_did, None,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The
|
||||
/// caller-aware generalization of `replicable_blob_set` (which is the anonymous
|
||||
/// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against
|
||||
@@ -325,6 +639,7 @@ pub fn replicable_blob_set(
|
||||
/// elsewhere (its content is readable to this caller elsewhere). Trees and
|
||||
/// commits are NOT included here; the caller decides per object type whether
|
||||
/// the allow-set applies (it does not for trees/commits — KTD3).
|
||||
#[cfg(test)]
|
||||
pub fn allowed_blob_set_for_caller(
|
||||
repo_path: &Path,
|
||||
rules: &[VisibilityRule],
|
||||
@@ -332,7 +647,29 @@ pub fn allowed_blob_set_for_caller(
|
||||
owner_did: &str,
|
||||
caller: Option<&str>,
|
||||
) -> Result<HashSet<String>> {
|
||||
let pairs = blob_paths(repo_path)?;
|
||||
allowed_blob_set_for_caller_bounded(
|
||||
repo_path,
|
||||
"git",
|
||||
WALK_TIMEOUT,
|
||||
rules,
|
||||
is_public,
|
||||
owner_did,
|
||||
caller,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`,
|
||||
/// for the `GET /ipfs/{cid}` gate.
|
||||
pub fn allowed_blob_set_for_caller_bounded(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
timeout: Duration,
|
||||
rules: &[VisibilityRule],
|
||||
is_public: bool,
|
||||
owner_did: &str,
|
||||
caller: Option<&str>,
|
||||
) -> Result<HashSet<String>> {
|
||||
let pairs = blob_paths(repo_path, git_bin, timeout)?;
|
||||
let mut allowed = HashSet::new();
|
||||
for (oid, path) in &pairs {
|
||||
if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow {
|
||||
@@ -365,14 +702,28 @@ pub fn replicable_objects_fail_closed(
|
||||
/// owner plus any reader DID that `visibility_check` Allows at some path the
|
||||
/// blob appears at. Least-privilege: a reader of one private subtree is not a
|
||||
/// recipient of a blob that only lives in another.
|
||||
#[cfg(test)]
|
||||
pub fn withheld_blob_recipients(
|
||||
repo_path: &Path,
|
||||
rules: &[VisibilityRule],
|
||||
is_public: bool,
|
||||
owner_did: &str,
|
||||
) -> Result<HashMap<String, BTreeSet<String>>> {
|
||||
withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did)
|
||||
}
|
||||
|
||||
/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for
|
||||
/// the receive-pack encrypt-then-pin path.
|
||||
pub fn withheld_blob_recipients_bounded(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
timeout: Duration,
|
||||
rules: &[VisibilityRule],
|
||||
is_public: bool,
|
||||
owner_did: &str,
|
||||
) -> Result<HashMap<String, BTreeSet<String>>> {
|
||||
// One history walk feeds both the withheld set and the recipient mapping.
|
||||
let pairs = blob_paths(repo_path)?;
|
||||
let pairs = blob_paths(repo_path, git_bin, timeout)?;
|
||||
let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None);
|
||||
if withheld.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
@@ -402,6 +753,300 @@ pub fn withheld_blob_recipients(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Write an executable fake `git` shell script into `dir` and return its path,
|
||||
/// so a test can drive the walk's process-group teardown without a real git and
|
||||
/// without mutating the process-global PATH (the crate's only injection seam).
|
||||
#[cfg(unix)]
|
||||
fn write_fake_git(dir: &Path, body: &str) -> String {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let p = dir.join("fakegit");
|
||||
std::fs::write(&p, body).unwrap();
|
||||
let mut perm = std::fs::metadata(&p).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(&p, perm).unwrap();
|
||||
p.to_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
/// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so
|
||||
/// a hung git child cannot pin the caller's permit past the deadline. A fake git
|
||||
/// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout`
|
||||
/// within the watchdog budget (not block for the child's lifetime), and the
|
||||
/// child's process group must be reaped (its recorded leader PID gone). Every
|
||||
/// caller (upload-pack serve, receive-pack replication) funnels through
|
||||
/// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize
|
||||
/// the watchdog SIGTERM and this hangs past the recv budget (RED).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn blob_paths_times_out_and_reaps_a_hung_walk() {
|
||||
use std::time::Duration;
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Fast on every stage except rev-list, which records its own (group-leader)
|
||||
// PID and then hangs. `sleep 30` bounds the worst case if the watchdog is
|
||||
// ever broken, so a regression cannot wedge the suite for 300s.
|
||||
let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n";
|
||||
let git_bin = write_fake_git(tmp.path(), body);
|
||||
|
||||
// Run the walk on a thread with a short budget; the recv_timeout succeeding
|
||||
// is itself proof the walk did not block on the hung child.
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let path = tmp.path().to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200)));
|
||||
});
|
||||
let result = rx.recv_timeout(Duration::from_secs(10)).expect(
|
||||
"blob_paths must return within the watchdog budget, not hang on a stuck git child",
|
||||
);
|
||||
let err = result.expect_err("a hung rev-list must abort the walk with an error");
|
||||
assert!(
|
||||
err.downcast_ref::<crate::git::smart_http::GitServiceTimeout>()
|
||||
.is_some(),
|
||||
// `{err:#}` prints the whole anyhow chain. Plain `{err}` shows only the top
|
||||
// context ("failed to spawn git for-each-ref") and drops the underlying io
|
||||
// error, which left a real beta-lane CI failure undiagnosable.
|
||||
"a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err:#}"
|
||||
);
|
||||
|
||||
// The recorded process-group leader must be gone: the watchdog reaps the
|
||||
// whole group before blob_paths returns, so no orphaned git lingers.
|
||||
let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid"))
|
||||
.expect("the fake git must have recorded its rev-list PID")
|
||||
.trim()
|
||||
.parse()
|
||||
.expect("recorded PID must parse");
|
||||
let mut gone = false;
|
||||
for _ in 0..200 {
|
||||
// SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means
|
||||
// the process is gone. Borrows no Rust memory.
|
||||
if unsafe { libc::kill(pid, 0) } != 0 {
|
||||
gone = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(
|
||||
gone,
|
||||
"the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is
|
||||
/// never reported as a timeout even when the watchdog fires, so a walk finishing
|
||||
/// right at its deadline is not a spurious 504. The fake only exits when signalled
|
||||
/// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always
|
||||
/// reaches its kill path (killed == true) yet the child's status is success.
|
||||
/// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() {
|
||||
use std::time::{Duration, Instant};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n";
|
||||
let git_bin = write_fake_git(tmp.path(), body);
|
||||
let out = run_bounded_git(
|
||||
&git_bin,
|
||||
&["rev-list"],
|
||||
tmp.path(),
|
||||
b"",
|
||||
Instant::now() + Duration::from_millis(100),
|
||||
);
|
||||
assert!(
|
||||
out.is_ok(),
|
||||
"a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped
|
||||
/// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its
|
||||
/// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still
|
||||
/// return (via SIGKILL at the grace step) with a timeout error and the group must
|
||||
/// be gone. (A truly uninterruptible D-state child, which no signal can reap, is
|
||||
/// the documented residual this teardown, like the async twin, cannot cover.)
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() {
|
||||
use std::time::{Duration, Instant};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n";
|
||||
let git_bin = write_fake_git(tmp.path(), body);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let path = tmp.path().to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(run_bounded_git(
|
||||
&git_bin,
|
||||
&["rev-list"],
|
||||
&path,
|
||||
b"",
|
||||
Instant::now() + Duration::from_millis(100),
|
||||
));
|
||||
});
|
||||
let out = rx
|
||||
.recv_timeout(Duration::from_secs(10))
|
||||
.expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child");
|
||||
assert!(
|
||||
out.is_err(),
|
||||
"a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}"
|
||||
);
|
||||
let pid: i32 = std::fs::read_to_string(tmp.path().join("pid"))
|
||||
.unwrap()
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let mut gone = false;
|
||||
for _ in 0..300 {
|
||||
if unsafe { libc::kill(pid, 0) } != 0 {
|
||||
gone = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(
|
||||
gone,
|
||||
"the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must
|
||||
/// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The
|
||||
/// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its
|
||||
/// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores
|
||||
/// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the
|
||||
/// group, the leader exits, its stdout closes, the main drain unblocks, and the
|
||||
/// leader is reaped — the exact window a `reaped`-gated watchdog stands down in,
|
||||
/// before escalating to SIGKILL. The descendant must be dead when run_bounded_git
|
||||
/// returns; a teardown that stands down on leader-reap leaves it running (RED).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() {
|
||||
use std::time::{Duration, Instant};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Both loops are bounded (~30s) so a broken teardown cannot leak a permanent
|
||||
// orphan or wedge the suite; the assertion fires well before then.
|
||||
let body = "#!/bin/sh\n\
|
||||
case \"$1\" in\n\
|
||||
rev-list)\n\
|
||||
sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\
|
||||
trap 'exit 0' TERM\n\
|
||||
i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\
|
||||
*) : ;;\n\
|
||||
esac\n";
|
||||
let git_bin = write_fake_git(tmp.path(), body);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let path = tmp.path().to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(run_bounded_git(
|
||||
&git_bin,
|
||||
&["rev-list"],
|
||||
&path,
|
||||
b"",
|
||||
Instant::now() + Duration::from_millis(100),
|
||||
));
|
||||
});
|
||||
let _ = rx
|
||||
.recv_timeout(Duration::from_secs(10))
|
||||
.expect("run_bounded_git must return within the watchdog budget");
|
||||
|
||||
// Wait for the descendant to record its OWN pid, then assert it is gone.
|
||||
let desc_pid_path = tmp.path().join("desc.pid");
|
||||
let mut desc: Option<i32> = None;
|
||||
for _ in 0..200 {
|
||||
if let Some(p) = std::fs::read_to_string(&desc_pid_path)
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<i32>().ok())
|
||||
{
|
||||
desc = Some(p);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
let desc = desc.expect("the fake leader must have spawned and recorded a descendant");
|
||||
let mut gone = false;
|
||||
for _ in 0..300 {
|
||||
if unsafe { libc::kill(desc, 0) } != 0 {
|
||||
gone = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
// Kill it regardless so a RED run leaks no orphan.
|
||||
unsafe { libc::kill(desc, libc::SIGKILL) };
|
||||
assert!(
|
||||
gone,
|
||||
"a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own
|
||||
/// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code
|
||||
/// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down
|
||||
/// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()`
|
||||
/// then blocks on the still-alive leader — pinning the walk thread and its read/
|
||||
/// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS.
|
||||
/// This is distinct from the descendant case above: there the leader sleeps until
|
||||
/// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of
|
||||
/// the deadline. The fix keeps the watchdog armed until the child is actually
|
||||
/// reaped, so the deadline SIGTERM still fires and the call returns within budget.
|
||||
/// A pre-fix build blocks on `child.wait()` past the recv budget (RED).
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() {
|
||||
use std::time::{Duration, Instant};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs
|
||||
// immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst
|
||||
// case so a RED run cannot wedge the suite; the recv budget fires first.
|
||||
//
|
||||
// The deadline below is deliberately far larger than the work it bounds: the
|
||||
// watchdog tears the whole group down at the deadline, so a deadline tight
|
||||
// enough to race `/bin/sh` reaching its first statement kills the leader
|
||||
// BEFORE it records its pid, and the `leader.pid` read below then fails with
|
||||
// NotFound. That is a harness race, not a product failure, and it is exactly
|
||||
// what a loaded CI runner reproduced (test (beta), 2026-08-04). Sizing it at
|
||||
// 2s keeps the property under test intact, since the pre-fix build blocks on
|
||||
// `child.wait()` for the child's full `sleep 30` and still trips the 10s recv
|
||||
// budget below (RED), while leaving a 20x margin over the ~100ms that a
|
||||
// healthy start actually needs.
|
||||
let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n";
|
||||
let git_bin = write_fake_git(tmp.path(), body);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let path = tmp.path().to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(run_bounded_git(
|
||||
&git_bin,
|
||||
&["rev-list"],
|
||||
&path,
|
||||
b"",
|
||||
Instant::now() + Duration::from_secs(2),
|
||||
));
|
||||
});
|
||||
let out = rx.recv_timeout(Duration::from_secs(10)).expect(
|
||||
"run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()",
|
||||
);
|
||||
assert!(
|
||||
out.is_err(),
|
||||
"a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}"
|
||||
);
|
||||
let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid"))
|
||||
.unwrap()
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let mut gone = false;
|
||||
for _ in 0..300 {
|
||||
if unsafe { libc::kill(pid, 0) } != 0 {
|
||||
gone = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
// Kill it regardless so a RED run leaks no orphan.
|
||||
unsafe { libc::kill(pid, libc::SIGKILL) };
|
||||
assert!(
|
||||
gone,
|
||||
"the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running"
|
||||
);
|
||||
}
|
||||
|
||||
use crate::db::VisibilityMode;
|
||||
use chrono::Utc;
|
||||
use std::process::Command;
|
||||
@@ -735,7 +1380,12 @@ mod tests {
|
||||
String::from_utf8_lossy(&out.stdout).trim().to_string()
|
||||
};
|
||||
|
||||
let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap();
|
||||
let all_blobs = crate::git::push_delta::all_blob_oids(
|
||||
&work,
|
||||
"git",
|
||||
std::time::Instant::now() + std::time::Duration::from_secs(600),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
all_blobs.contains(&dangling_oid),
|
||||
"precondition: the dangling blob is in the all-objects universe"
|
||||
|
||||
+1035
-17
File diff suppressed because it is too large.
Load diff
+142
-10
@@ -74,6 +74,13 @@ async fn main() -> Result<()> {
|
||||
// bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS.
|
||||
bootstrap::merge_seeds(&mut config);
|
||||
|
||||
// Fail fast on config combinations that are individually in-range but jointly
|
||||
// unsafe — notably a DB pool too small for the concurrent-write cap, which
|
||||
// would let a push burst starve every other DB path (#174 F1).
|
||||
config
|
||||
.validate()
|
||||
.map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?;
|
||||
|
||||
if !config.public_read {
|
||||
warn!(
|
||||
"GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha"
|
||||
@@ -378,7 +385,81 @@ async fn main() -> Result<()> {
|
||||
sync_trigger_rate_limiter,
|
||||
peer_write_rate_limiter,
|
||||
shutdown_tx: shutdown_tx.clone(),
|
||||
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)),
|
||||
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(
|
||||
config.max_concurrent_git_pushes,
|
||||
)),
|
||||
// Anon receive-pack advertisements get their OWN pool, same size as the
|
||||
// write pool but disjoint, so filling it (which takes many source IPs, each
|
||||
// capped by git_push_advert_per_caller) never occupies a permit the
|
||||
// authenticated POST needs (#174).
|
||||
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(
|
||||
config.max_concurrent_git_pushes,
|
||||
)),
|
||||
// Bounds concurrent detached post-push encryption walks, sized from the push
|
||||
// pool (no separate knob — Q1): completed pushes cannot outnumber active
|
||||
// encryption walks past this (#174 P1-e).
|
||||
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(
|
||||
config.max_concurrent_git_pushes,
|
||||
)),
|
||||
// Bounds how many post-push pin loops run concurrently across all repos (#174 F6),
|
||||
// independent of the per-repo encrypt-task coalescing below. Not a bound on the
|
||||
// MB-scale object-id lists themselves: parked tasks still hold theirs (see the
|
||||
// field doc on AppState::pin_semaphore).
|
||||
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)),
|
||||
// Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher
|
||||
// cannot grow the outstanding parked-waiter set past one task per repo (#174
|
||||
// P2-2). No knob: it is a natural cap (one entry per distinct repo), not a
|
||||
// sized pool.
|
||||
encrypt_inflight: crate::state::EncryptInflight::new(),
|
||||
// Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg
|
||||
// advisory lock so a disconnected push's still-reaping git group can't be raced
|
||||
// by a second same-node push. The map is naturally capped (one entry per contended
|
||||
// repo, freed when unreferenced); the sized knob is how many pushes may PARK on
|
||||
// one repo, since each parked push holds a fully buffered pack.
|
||||
repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters),
|
||||
git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
config.max_concurrent_reads_per_caller,
|
||||
),
|
||||
// Per-source cap on the receive-pack advertisement, sized to an eighth of the
|
||||
// write pool (min 1): one resolved client key (rate_limit::client_key) can hold
|
||||
// at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore,
|
||||
// disjoint from the write pool), so saturating that pool takes ~8 distinct keys
|
||||
// (#174). That bounds an IPv4 or single-address caller; a caller controlling many
|
||||
// addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since
|
||||
// client_key uses the full IP with no prefix folding. Narrowing the keying is a
|
||||
// deferred design call, not something these caps claim to solve. Sized off the
|
||||
// write pool only because the advert pool is created at the same size; an advert
|
||||
// flood cannot touch a write permit.
|
||||
git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
rate_limit::per_source_push_cap(config.max_concurrent_git_pushes),
|
||||
),
|
||||
// Per-source cap on the authenticated receive-pack POST, sized like the advert
|
||||
// cap: one resolved client key can hold at most this many write-pool slots, so
|
||||
// monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as
|
||||
// above: keys are full IPs, so a caller with many addresses has many caps.
|
||||
git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
rate_limit::per_source_push_cap(config.max_concurrent_git_pushes),
|
||||
),
|
||||
// Bounds concurrent /ipfs visibility walks — a distinct public cost center, so
|
||||
// its own pool + per-source sub-cap + per-IP rate limiter, never a git pool
|
||||
// (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15).
|
||||
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(
|
||||
config.max_concurrent_ipfs_walks,
|
||||
)),
|
||||
git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
config.ipfs_walk_per_source,
|
||||
),
|
||||
ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded(
|
||||
config.ipfs_rate_limit,
|
||||
std::time::Duration::from_secs(3600),
|
||||
200_000,
|
||||
),
|
||||
git_bin: "git".to_string(),
|
||||
};
|
||||
if config.ipfs_rate_limit == 0 {
|
||||
tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled");
|
||||
}
|
||||
|
||||
// Periodic peer-count poll for the metrics gauge. If p2p is disabled
|
||||
// we still set the gauge to 0 so dashboards don't show "no data".
|
||||
@@ -408,22 +489,14 @@ async fn main() -> Result<()> {
|
||||
|
||||
// Periodic cleanup of expired rate limit entries + consumed-proof ledger
|
||||
{
|
||||
let rl = state.rate_limiter.clone();
|
||||
let create_ip_rl = state.create_ip_rate_limiter.clone();
|
||||
let push_rl = state.push_rate_limiter.clone();
|
||||
let sync_trigger_rl = state.sync_trigger_rate_limiter.clone();
|
||||
let peer_write_rl = state.peer_write_rate_limiter.clone();
|
||||
let sweep_state = state.clone();
|
||||
let db = state.db.clone();
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(300)) => {
|
||||
rl.cleanup().await;
|
||||
create_ip_rl.cleanup().await;
|
||||
push_rl.cleanup().await;
|
||||
sync_trigger_rl.cleanup().await;
|
||||
peer_write_rl.cleanup().await;
|
||||
sweep_rate_limiters(&sweep_state).await;
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
@@ -959,6 +1032,65 @@ async fn gossip_task(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod rate_limiter_sweep_tests {
|
||||
use crate::rate_limit::RateLimiter;
|
||||
use std::time::Duration;
|
||||
|
||||
// Every per-key limiter the router mounts must be swept by the periodic
|
||||
// task, the `/ipfs` one included: a limiter left out keeps expired keys
|
||||
// until its map fills and the inline capacity sweep fires. Fails on the
|
||||
// pre-fix sweeper, which skipped `ipfs_rate_limiter`.
|
||||
#[tokio::test]
|
||||
async fn sweep_evicts_expired_keys_from_every_limiter() {
|
||||
let window = Duration::from_millis(30);
|
||||
let mut state = crate::test_support::test_state_lazy();
|
||||
state.rate_limiter = RateLimiter::new(10, window);
|
||||
state.create_ip_rate_limiter = RateLimiter::new(10, window);
|
||||
state.push_rate_limiter = RateLimiter::new(10, window);
|
||||
state.sync_trigger_rate_limiter = RateLimiter::new(10, window);
|
||||
state.peer_write_rate_limiter = RateLimiter::new(10, window);
|
||||
state.ipfs_rate_limiter = RateLimiter::new(10, window);
|
||||
|
||||
let limiters = |s: &crate::state::AppState| {
|
||||
[
|
||||
s.rate_limiter.clone(),
|
||||
s.create_ip_rate_limiter.clone(),
|
||||
s.push_rate_limiter.clone(),
|
||||
s.sync_trigger_rate_limiter.clone(),
|
||||
s.peer_write_rate_limiter.clone(),
|
||||
s.ipfs_rate_limiter.clone(),
|
||||
]
|
||||
};
|
||||
for l in limiters(&state) {
|
||||
assert!(l.check("1.2.3.4").await);
|
||||
assert_eq!(l.tracked_keys().await, 1);
|
||||
}
|
||||
|
||||
tokio::time::sleep(window * 3).await;
|
||||
super::sweep_rate_limiters(&state).await;
|
||||
|
||||
for (i, l) in limiters(&state).into_iter().enumerate() {
|
||||
assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Evict expired entries from every per-key rate limiter on the state.
|
||||
///
|
||||
/// Named and driven off `AppState` so the periodic sweeper stays in step with
|
||||
/// the limiters the router actually mounts: adding a limiter field and
|
||||
/// forgetting it here leaves its keys pinned until the map hits `max_keys` and
|
||||
/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way).
|
||||
async fn sweep_rate_limiters(state: &AppState) {
|
||||
state.rate_limiter.cleanup().await;
|
||||
state.create_ip_rate_limiter.cleanup().await;
|
||||
state.push_rate_limiter.cleanup().await;
|
||||
state.sync_trigger_rate_limiter.cleanup().await;
|
||||
state.peer_write_rate_limiter.cleanup().await;
|
||||
state.ipfs_rate_limiter.cleanup().await;
|
||||
}
|
||||
|
||||
/// Build the shared node HTTP client used for every outbound fan-out (sync
|
||||
/// trigger, profile/repo fetches, gossip announce + peer pings).
|
||||
///
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! no-op, so nodes without Pinata backing work fine.
|
||||
|
||||
use anyhow::Result;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Pin a single git object's raw bytes on Pinata (v3 API).
|
||||
///
|
||||
@@ -72,25 +73,77 @@ pub async fn pin_object(
|
||||
/// `object_list` is the already-withheld-filtered OID set to pin: the caller
|
||||
/// applies `visibility_pack::replicable_objects` on the delta path or the
|
||||
/// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is
|
||||
/// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors
|
||||
/// this shape — change both in lockstep. Objects already recorded with a
|
||||
/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly
|
||||
/// pinned object.
|
||||
/// still needed to read each object's bytes, and `git_bin` names the binary those
|
||||
/// reads run: the production caller passes the literal `"git"`, and a test passes a
|
||||
/// fake so the loop's own bound can be driven with a git that never answers.
|
||||
/// Objects already recorded with a `pinata_cid` are skipped. Returns
|
||||
/// `(sha_hex, cid)` pairs for each newly pinned object.
|
||||
///
|
||||
/// # What `batch_budget` does and does not bound
|
||||
///
|
||||
/// The loop runs under a `pin_semaphore` permit and that pool defers rather than
|
||||
/// sheds, so the hold has to be bounded by something other than the pusher's object
|
||||
/// count. Two things here are:
|
||||
///
|
||||
/// - this loop's own wall-clock: the deadline is taken once at loop start and
|
||||
/// checked at the top of every iteration, so no object's work begins with less
|
||||
/// than the read floor left. It is a gate, not a hard ceiling, since a started
|
||||
/// iteration still runs to completion;
|
||||
/// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the
|
||||
/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_pinata_cid`
|
||||
/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL
|
||||
/// process-group teardown, so a hung `git cat-file` costs this batch its remaining
|
||||
/// budget plus one watchdog teardown instead of holding the permit for the child's
|
||||
/// whole lifetime and blocking a runtime worker while it does.
|
||||
///
|
||||
/// So the LOOP's hold is bounded by roughly `batch_budget`, plus one watchdog
|
||||
/// teardown and one upload (the shared client's whole-request timeout bounds the
|
||||
/// upload; `pin_object` takes no per-request override). The PERMIT's hold is NOT
|
||||
/// bounded by any of this: `api::repos` acquires the permit and then re-derives the
|
||||
/// object list with `pinata_object_list_for_refs` BEFORE this function is entered,
|
||||
/// and that walk carries no aggregate deadline. The DB round-trips
|
||||
/// (`has_pinata_cid`, `record_pinata_cid`) are untimed inside the budgeted region
|
||||
/// too.
|
||||
///
|
||||
/// The twin in `ipfs_pin.rs` shares the budget gate and the bounded read with this
|
||||
/// loop, so both are at parity again. Change them in lockstep: the skip-if-pinned
|
||||
/// check, the fault arms, the returned pairs, and the budget handling.
|
||||
// Eight arguments, one over clippy's threshold: the two the budget adds (`git_bin`,
|
||||
// `batch_budget`) are what put the read under test injection and under a deadline, and
|
||||
// grouping them into a struct would only move the same values behind a name the twin in
|
||||
// `ipfs_pin.rs` does not use. Same allow as the sibling call sites in `api::repos`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn pin_new_objects(
|
||||
client: &reqwest::Client,
|
||||
upload_url: &str,
|
||||
jwt: &str,
|
||||
repo_path: &std::path::Path,
|
||||
git_bin: &str,
|
||||
object_list: Vec<String>,
|
||||
db: &crate::db::Db,
|
||||
batch_budget: Duration,
|
||||
) -> Vec<(String, String)> {
|
||||
if jwt.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let deadline = Instant::now() + batch_budget;
|
||||
let total = object_list.len();
|
||||
let mut pinned = Vec::new();
|
||||
|
||||
for sha in object_list {
|
||||
for (attempted, sha) in object_list.into_iter().enumerate() {
|
||||
// Top of the iteration, before any of this object's work: an object is never
|
||||
// started with a remainder too small to cover a bounded read's teardown. The
|
||||
// gate is shared with the IPFS loop so the two cannot drift apart in how they
|
||||
// report a truncated batch. Consumed as a guard only: the read below runs against
|
||||
// the absolute batch deadline, and `pin_object` takes no per-request override, so
|
||||
// the remainder has no other consumer here.
|
||||
if crate::ipfs_pin::batch_budget_gate("Pinata", deadline, pinned.len(), total - attempted)
|
||||
.is_none()
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
match db.has_pinata_cid(&sha).await {
|
||||
Ok(true) => continue,
|
||||
Ok(false) => {}
|
||||
@@ -100,13 +153,73 @@ pub async fn pin_new_objects(
|
||||
}
|
||||
}
|
||||
|
||||
let data = match crate::git::store::read_object(repo_path, &sha) {
|
||||
Ok(Some((_kind, bytes))) => bytes,
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
// Read raw object content, bounded and reaped, under `spawn_blocking`: this is
|
||||
// synchronous blocking work (child spawn, pipe drain, watchdog join), so calling
|
||||
// it from the runtime task would block a worker thread for its whole duration.
|
||||
// Placement mirrors the `/ipfs` serve path and the IPFS pin loop.
|
||||
//
|
||||
// The read runs against the ABSOLUTE batch deadline, not against the remainder
|
||||
// measured at the top of the iteration: the `has_pinata_cid` round-trip above sits
|
||||
// between the two, so `Instant::now() + budget_left` would land past `deadline` by
|
||||
// however long the DB took, and under a saturated pool that is the dominant term.
|
||||
// A slow DB check must not push the read's own bound out.
|
||||
let read_deadline = deadline;
|
||||
let read_path = repo_path.to_path_buf();
|
||||
let read_sha = sha.clone();
|
||||
let read_git = git_bin.to_string();
|
||||
let read = tokio::task::spawn_blocking(move || {
|
||||
crate::git::store::read_object_bounded(&read_git, &read_path, &read_sha, read_deadline)
|
||||
})
|
||||
.await;
|
||||
let data = match read {
|
||||
Ok(Ok(Some((_kind, bytes)))) => bytes,
|
||||
// A verified absence, and the only outcome that is not a fault.
|
||||
Ok(Ok(None)) => continue,
|
||||
// A Transient fault does NOT by itself mean the store is gone. It also covers
|
||||
// a spawn or watchdog-timeout failure of the reaped child, an unaffordable
|
||||
// confirming re-probe, and, because readability is judged FOR one oid, a
|
||||
// single unreadable `objects/<xx>` fan-out, which is 1/256 of the store. So
|
||||
// re-check store-wide before deciding what the fault costs.
|
||||
Ok(Err(e @ crate::git::store::ProbeError::Transient(_))) => {
|
||||
if !crate::git::store::object_store_readable_store_wide(repo_path) {
|
||||
// Genuinely store-wide: every remaining object fails identically, and
|
||||
// continuing would spawn one doomed bounded child per object and spend
|
||||
// the batch budget reaping them.
|
||||
tracing::warn!(
|
||||
sha = %sha,
|
||||
err = %e,
|
||||
unattempted = total - attempted,
|
||||
"object store unreadable while pinning to Pinata; stopping the batch"
|
||||
);
|
||||
break;
|
||||
}
|
||||
// The store still reads store-wide, so the fault is object-scoped or
|
||||
// transient to this read. Breaking would forfeit a healthy store's
|
||||
// remaining objects permanently: the documented recovery re-derives the
|
||||
// same list and breaks at the same index.
|
||||
tracing::warn!(
|
||||
sha = %sha,
|
||||
err = %e,
|
||||
"transient fault reading git object for Pinata; the object store is \
|
||||
still readable store-wide, so this costs only this object"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// The store is readable and git still failed: a corrupt object, or a
|
||||
// repo-wide fault git reports immediately. Either way it is per-object work
|
||||
// that stays inside the budget, and breaking would forfeit a healthy store's
|
||||
// remaining objects over one bad one, permanently (a later full-scan push
|
||||
// re-offers the same object and breaks in the same place).
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(sha = %sha, err = %e, "failed to read git object for Pinata");
|
||||
continue;
|
||||
}
|
||||
// A panic in the read closure leaves no evidence that the failure is
|
||||
// object-scoped, so fail toward the conservative arm.
|
||||
Err(e) => {
|
||||
tracing::warn!(sha = %sha, err = %e, "bounded git read task failed; stopping the batch");
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
match pin_object(client, upload_url, jwt, &sha, &data).await {
|
||||
@@ -132,6 +245,626 @@ pub async fn pin_new_objects(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Write `n` loose blobs into a fresh bare repo and return their oids. The read
|
||||
/// path shells to `git cat-file`, so the objects must genuinely exist on disk: a
|
||||
/// fabricated oid would `continue` past the upload and the loop scenarios below
|
||||
/// would prove nothing. Copied from `ipfs_pin.rs`'s test mod rather than shared,
|
||||
/// since test mods are private.
|
||||
fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec<String> {
|
||||
crate::git::store::init_bare(repo_path).expect("init bare repo");
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let mut cmd = std::process::Command::new("git");
|
||||
cmd.args(["hash-object", "-w", "--stdin"])
|
||||
.current_dir(repo_path)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped());
|
||||
let mut child = cmd.spawn().expect("spawn git hash-object");
|
||||
{
|
||||
use std::io::Write;
|
||||
child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.expect("stdin")
|
||||
.write_all(format!("pinata loop object {i}\n").as_bytes())
|
||||
.expect("write stdin");
|
||||
}
|
||||
let out = child.wait_with_output().expect("hash-object output");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"git hash-object: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&out.stdout).trim().to_string()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A live Pinata-shaped endpoint that answers each upload with a v3 `data.cid`
|
||||
/// body after `delays[i]` for the i-th request it accepts (the last entry
|
||||
/// repeats), counting the requests it received.
|
||||
///
|
||||
/// Hand rolled rather than driven with `mockito` like the `pin_object` tests
|
||||
/// above: mockito has no per-response delay primitive, and the batch-budget test
|
||||
/// needs uploads that are slow enough to exhaust the budget partway. Drains the
|
||||
/// full request, headers plus the declared `Content-Length` body, before
|
||||
/// sleeping: answering early and closing would surface as a write failure on the
|
||||
/// client and turn a slow-but-healthy upload into a different failure shape.
|
||||
/// Same fixture shape as `ipfs_pin.rs`'s `delaying_endpoint`.
|
||||
async fn delaying_pinata_endpoint(
|
||||
delays: Vec<Duration>,
|
||||
requests: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||
) -> String {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let endpoint = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let mut seen = 0usize;
|
||||
while let Ok((mut sock, _)) = listener.accept().await {
|
||||
let delay = *delays
|
||||
.get(seen)
|
||||
.or_else(|| delays.last())
|
||||
.unwrap_or(&Duration::ZERO);
|
||||
seen += 1;
|
||||
requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
tokio::spawn(async move {
|
||||
let mut acc = Vec::new();
|
||||
let mut buf = [0u8; 4096];
|
||||
loop {
|
||||
let n = match sock.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
};
|
||||
acc.extend_from_slice(&buf[..n]);
|
||||
if let Some(hdr_end) =
|
||||
acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4)
|
||||
{
|
||||
let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase();
|
||||
let len: usize = headers
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix("content-length:"))
|
||||
.and_then(|v| v.trim().parse().ok())
|
||||
.unwrap_or(0);
|
||||
if acc.len() >= hdr_end + len {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
let body = br#"{"data":{"cid":"QmPinataBatchTestCid","name":"git.bin"}}"#;
|
||||
let head = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
let _ = sock.write_all(head.as_bytes()).await;
|
||||
let _ = sock.write_all(body).await;
|
||||
let _ = sock.flush().await;
|
||||
});
|
||||
}
|
||||
});
|
||||
endpoint
|
||||
}
|
||||
|
||||
/// A `tracing` sink a test can read back, so the truncation warn and its sink
|
||||
/// label can be asserted on rather than assumed. Installed with `set_default`,
|
||||
/// which is thread-local and scoped to the guard, so it cannot bleed into any
|
||||
/// other test in the binary.
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
impl CapturedLogs {
|
||||
fn text(&self) -> String {
|
||||
String::from_utf8_lossy(&self.0.lock().unwrap()).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::io::Write for CapturedLogs {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
|
||||
type Writer = CapturedLogs;
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_ansi(false)
|
||||
.finish();
|
||||
let guard = tracing::subscriber::set_default(subscriber);
|
||||
(logs, guard)
|
||||
}
|
||||
|
||||
/// Write an executable `/bin/sh` script. Copied per module rather than shared:
|
||||
/// `store.rs`, `visibility_pack.rs` and `ipfs_pin.rs` each keep their own, since
|
||||
/// their test mods are private and not reachable from here.
|
||||
#[cfg(unix)]
|
||||
fn write_script(path: &std::path::Path, body: &str) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::write(path, body).expect("write fake git");
|
||||
let mut perm = std::fs::metadata(path).unwrap().permissions();
|
||||
perm.set_mode(0o755);
|
||||
std::fs::set_permissions(path, perm).unwrap();
|
||||
}
|
||||
|
||||
/// The permit-hold bound on this branch. `pin_new_objects` runs under the same
|
||||
/// deferring `pin_semaphore` as the IPFS loop, so without a batch deadline the
|
||||
/// hold is O(N) with N chosen by the pusher. Five objects against an endpoint
|
||||
/// that takes 2s each, under a 5.5s budget, must stop partway: the batch is
|
||||
/// truncated and the remainder is left unattempted with exactly one warn naming
|
||||
/// how many, labelled for this sink.
|
||||
///
|
||||
/// The windows are deliberately loose. Four pins would need every upload to
|
||||
/// answer in under 1.4s, which the endpoint's own 2s sleep forbids, and one pin
|
||||
/// needs only the first upload to land inside 5.5s, so both bounds hold with
|
||||
/// more than a second of slack on a loaded box.
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) {
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("slow.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 5);
|
||||
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let endpoint = delaying_pinata_endpoint(
|
||||
vec![Duration::from_secs(2)],
|
||||
std::sync::Arc::clone(&requests),
|
||||
)
|
||||
.await;
|
||||
|
||||
let (logs, _guard) = capture_logs();
|
||||
let client = reqwest::Client::new();
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
"git",
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_millis(5500),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("wedge guard: a 5.5s budget cannot take 30s");
|
||||
|
||||
assert!(
|
||||
(1..=3).contains(&pinned.len()),
|
||||
"the batch must stop partway, not pin all five and not stall on the first: pinned {}",
|
||||
pinned.len()
|
||||
);
|
||||
assert_eq!(
|
||||
requests.load(std::sync::atomic::Ordering::SeqCst),
|
||||
pinned.len(),
|
||||
"no upload may be issued for an object the budget stopped short of"
|
||||
);
|
||||
let text = logs.text();
|
||||
let warns: Vec<&str> = text
|
||||
.lines()
|
||||
.filter(|l| l.contains("pin batch deadline reached"))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
warns.len(),
|
||||
1,
|
||||
"the deadline must be reported exactly once for the batch, not per object: {text}"
|
||||
);
|
||||
assert!(
|
||||
warns[0].contains("Pinata"),
|
||||
"the truncation warn must name this sink, not the twin's: {}",
|
||||
warns[0]
|
||||
);
|
||||
let unattempted: usize = warns[0]
|
||||
.split("unattempted=")
|
||||
.nth(1)
|
||||
.and_then(|s| {
|
||||
s.split(|c: char| !c.is_ascii_digit())
|
||||
.next()
|
||||
.and_then(|d| d.parse().ok())
|
||||
})
|
||||
.unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}"));
|
||||
assert!(
|
||||
unattempted >= 1 && unattempted + pinned.len() <= 5,
|
||||
"unattempted={unattempted} with {} pinned is not a partial batch of five",
|
||||
pinned.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 F3 on this branch: the git read runs while the `pin_semaphore` permit is
|
||||
/// held, so a wedged `git cat-file` used to hold that permit for as long as the
|
||||
/// child lived, with no deadline and no reaping, on a path a pusher drives. With
|
||||
/// the read bounded, a git that never answers costs the batch its budget plus one
|
||||
/// watchdog teardown and no more.
|
||||
///
|
||||
/// The fake traps SIGTERM and sleeps a BOUNDED 30s, following the fixture in
|
||||
/// `visibility_pack.rs`: with the deadline neutralized the read would otherwise
|
||||
/// leave the blocking closure and its child alive long after the test-level
|
||||
/// timeout fires, wedging the run instead of reporting a failure. The endpoint is
|
||||
/// never reached, since no object's bytes are ever produced.
|
||||
///
|
||||
/// The batch ends on the BUDGET, not on the fault arm: the repo is a healthy bare
|
||||
/// store, so the timeout's `Transient` verdict is object-scoped and the loop moves on,
|
||||
/// only to find the budget spent. Capturing that warn is not decoration. `tracing`
|
||||
/// caches a callsite's interest globally the first time it is hit, and a hit from a
|
||||
/// thread with no subscriber caches it as never-interested for the whole binary, which
|
||||
/// silently blinds the deadline tests running beside this one.
|
||||
#[cfg(unix)]
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) {
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("wedged.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 3);
|
||||
let fake = tmp.path().join("hanging-git");
|
||||
write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n");
|
||||
|
||||
let (logs, _guard) = capture_logs();
|
||||
let client = reqwest::Client::new();
|
||||
let started = std::time::Instant::now();
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(25),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
"http://127.0.0.1:9",
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
fake.to_str().unwrap(),
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_secs(2),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect(
|
||||
"a wedged git must not hold the pin permit past the batch budget: the read is \
|
||||
bounded and reaped, so this cannot reach the outer timeout",
|
||||
);
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
assert!(
|
||||
pinned.is_empty(),
|
||||
"a git that never answers cannot produce a pinned object: {pinned:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(20),
|
||||
"elapsed {elapsed:?} must stay inside the budget plus one watchdog teardown"
|
||||
);
|
||||
let text = logs.text();
|
||||
assert_eq!(
|
||||
text.lines()
|
||||
.filter(|l| l.contains("pin batch deadline reached"))
|
||||
.count(),
|
||||
1,
|
||||
"one wedged read must spend the whole budget and stop the batch there, exactly \
|
||||
once: {text}"
|
||||
);
|
||||
|
||||
// The child's process group must be gone once the call returns; a bounded read
|
||||
// that leaves the child running has only moved the hold somewhere else.
|
||||
let pid: i32 = std::fs::read_to_string(repo_path.join("pid"))
|
||||
.expect("the fake git must have recorded its pid, or it was never on the read path")
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let mut gone = false;
|
||||
for _ in 0..200 {
|
||||
// SAFETY: kill(2) with signal 0 only probes existence; ESRCH means gone.
|
||||
if unsafe { libc::kill(pid, 0) } != 0 {
|
||||
gone = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(
|
||||
gone,
|
||||
"the reaped fake git ({pid}) must not outlive the call"
|
||||
);
|
||||
}
|
||||
|
||||
/// A `git_bin` wrapper that records every invocation's arguments and then execs the
|
||||
/// real git, so a test can tell which objects the loop actually attempted. The returned
|
||||
/// pin list cannot: it is empty both when the loop broke after one object and when it
|
||||
/// continued past all of them. Copied from `ipfs_pin.rs`'s test mod, like the fixtures
|
||||
/// above, since test mods are private.
|
||||
#[cfg(unix)]
|
||||
fn counting_git(dir: &std::path::Path, log: &std::path::Path) -> String {
|
||||
let fake = dir.join("counting-git");
|
||||
write_script(
|
||||
&fake,
|
||||
&format!(
|
||||
"#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n",
|
||||
log.display()
|
||||
),
|
||||
);
|
||||
fake.to_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
/// How many objects the loop actually attempted, read off the invocation log.
|
||||
///
|
||||
/// Counts `--batch-check` invocations, not log lines and not oid occurrences: the type
|
||||
/// probe carries its oid on stdin rather than in argv, so an oid appears in the log only
|
||||
/// once an object has already got past its probe, and a healthy object costs two
|
||||
/// invocations to a faulting one's one.
|
||||
fn objects_attempted(log: &std::path::Path) -> usize {
|
||||
std::fs::read_to_string(log)
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.filter(|l| l.contains("--batch-check"))
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The store-wide fault arm, the twin of the IPFS loop's. When the object store cannot
|
||||
/// be read at all every remaining object fails identically, so continuing would spawn
|
||||
/// one doomed bounded child per object and burn the batch budget on reaping them.
|
||||
///
|
||||
/// The fixture looks wrong and is not: with the objects LOOSE and only `objects/pack`
|
||||
/// unreadable, git still resolves each object, but it prints an `error:` diagnostic
|
||||
/// that the probe routes to a fault before the present/missing parse, so the read
|
||||
/// reaches the fault classification and (the store being unreadable) returns
|
||||
/// `Transient`, which the store-wide re-check then confirms really is store-wide.
|
||||
#[cfg(unix)]
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_breaks_the_batch_on_an_unreadable_store(pool: sqlx::PgPool) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("unreadable.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 5);
|
||||
let log = tmp.path().join("calls.log");
|
||||
let git_bin = counting_git(tmp.path(), &log);
|
||||
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let endpoint =
|
||||
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let pack_dir = repo_path.join("objects").join("pack");
|
||||
let chmod = |mode: u32| {
|
||||
let mut perms = std::fs::metadata(&pack_dir).unwrap().permissions();
|
||||
perms.set_mode(mode);
|
||||
std::fs::set_permissions(&pack_dir, perms).unwrap();
|
||||
};
|
||||
chmod(0o000);
|
||||
// Root bypasses permission bits, so witness the exact operation the probe performs
|
||||
// and skip rather than falsely fail.
|
||||
let genuinely_unreadable = std::fs::read_dir(&pack_dir).is_err();
|
||||
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
&git_bin,
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_secs(60),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("an immediately-faulting store cannot take 30s");
|
||||
let attempted = objects_attempted(&log);
|
||||
chmod(0o755); // restore BEFORE any assertion that can panic, so TempDir cleans up
|
||||
|
||||
if genuinely_unreadable {
|
||||
assert!(
|
||||
pinned.is_empty(),
|
||||
"nothing can be pinned through a store that cannot be read: {pinned:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
attempted, 1,
|
||||
"a store-wide fault must break the batch after the first object, not spawn \
|
||||
one doomed bounded child per object: {attempted} of 5 objects were read"
|
||||
);
|
||||
assert_eq!(
|
||||
requests.load(std::sync::atomic::Ordering::SeqCst),
|
||||
0,
|
||||
"no upload may be issued for an object whose bytes were never read"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The must-not direction of the arm above, the twin of the IPFS loop's. One corrupt
|
||||
/// loose object among healthy ones is a `Deterministic` fault (the store is readable,
|
||||
/// git still fails), and the documented recovery path cannot repair it: a later
|
||||
/// full-scan push re-offers the same object and would break at the same place, so
|
||||
/// breaking here stops the repo replicating permanently.
|
||||
///
|
||||
/// Deliberately not the bad-config corruption, which is repo-wide: all five objects
|
||||
/// would fault and the test would pin the store-wide case rather than the object-scoped
|
||||
/// one this arm rests on.
|
||||
#[cfg(unix)]
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_continues_past_a_deterministic_fault(pool: sqlx::PgPool) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("corrupt.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 5);
|
||||
let log = tmp.path().join("calls.log");
|
||||
let git_bin = counting_git(tmp.path(), &log);
|
||||
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let endpoint =
|
||||
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Overwrite exactly one loose object with non-zlib garbage (0o444 by default).
|
||||
let victim = repo_path
|
||||
.join("objects")
|
||||
.join(&oids[0][0..2])
|
||||
.join(&oids[0][2..]);
|
||||
assert!(victim.is_file(), "fixture must leave the blob loose");
|
||||
let mut perms = std::fs::metadata(&victim).unwrap().permissions();
|
||||
perms.set_mode(0o644);
|
||||
std::fs::set_permissions(&victim, perms).unwrap();
|
||||
std::fs::write(&victim, b"garbage not a zlib stream").unwrap();
|
||||
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
&git_bin,
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_secs(60),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("an immediate endpoint and four healthy objects cannot take 30s");
|
||||
|
||||
assert_eq!(
|
||||
objects_attempted(&log),
|
||||
5,
|
||||
"an object-scoped fault must not stop the batch: every object must be read"
|
||||
);
|
||||
assert_eq!(
|
||||
pinned.len(),
|
||||
4,
|
||||
"one corrupt object must cost only itself: the other four must still pin"
|
||||
);
|
||||
assert_eq!(
|
||||
requests.load(std::sync::atomic::Ordering::SeqCst),
|
||||
4,
|
||||
"exactly the readable objects may reach the endpoint"
|
||||
);
|
||||
}
|
||||
|
||||
/// The normal direction, and the dedup branch driven both ways: on a healthy
|
||||
/// store with a healthy endpoint and a generous budget every object pins and the
|
||||
/// CID is recorded, and a second call over the same list uploads nothing because
|
||||
/// `has_pinata_cid` now answers true for all of them.
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_pins_every_object_then_skips_the_recorded_ones(pool: sqlx::PgPool) {
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("healthy.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 3);
|
||||
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let endpoint =
|
||||
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
"git",
|
||||
oids.clone(),
|
||||
&db,
|
||||
Duration::from_secs(60),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("an immediate endpoint and three healthy objects cannot take 30s");
|
||||
|
||||
assert_eq!(pinned.len(), 3, "every healthy object must pin: {pinned:?}");
|
||||
for (i, (sha, cid)) in pinned.iter().enumerate() {
|
||||
assert_eq!(sha, &oids[i], "the pairs must carry the objects' own oids");
|
||||
assert_eq!(cid, "QmPinataBatchTestCid");
|
||||
assert!(
|
||||
db.has_pinata_cid(sha).await.unwrap(),
|
||||
"a pinned object must be recorded so the next batch skips it"
|
||||
);
|
||||
}
|
||||
assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||
|
||||
let again = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"test-jwt",
|
||||
&repo_path,
|
||||
"git",
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_secs(60),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("a fully deduped batch cannot take 30s");
|
||||
assert!(again.is_empty(), "already-recorded objects must be skipped");
|
||||
assert_eq!(
|
||||
requests.load(std::sync::atomic::Ordering::SeqCst),
|
||||
3,
|
||||
"the skip must happen before the upload, not after it"
|
||||
);
|
||||
}
|
||||
|
||||
/// The no-op configuration still short-circuits with the budgeted signature: an
|
||||
/// empty JWT must return before any git child is spawned and before any request
|
||||
/// is issued. The `git_bin` here records every invocation, so "git was never
|
||||
/// touched" is observed rather than assumed.
|
||||
#[cfg(unix)]
|
||||
#[sqlx::test]
|
||||
async fn pin_new_objects_with_an_empty_jwt_touches_neither_git_nor_the_endpoint(
|
||||
pool: sqlx::PgPool,
|
||||
) {
|
||||
let db = crate::db::Db::for_testing(pool);
|
||||
db.run_migrations().await.expect("migrations");
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let repo_path = tmp.path().join("unconfigured.git");
|
||||
let oids = seed_loose_blobs(&repo_path, 2);
|
||||
let log = tmp.path().join("calls.log");
|
||||
let fake = tmp.path().join("counting-git");
|
||||
write_script(
|
||||
&fake,
|
||||
&format!(
|
||||
"#!/bin/sh\necho \"$*\" >> {}\nexec git \"$@\"\n",
|
||||
log.display()
|
||||
),
|
||||
);
|
||||
let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let endpoint =
|
||||
delaying_pinata_endpoint(vec![Duration::ZERO], std::sync::Arc::clone(&requests)).await;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let pinned = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
pin_new_objects(
|
||||
&client,
|
||||
&endpoint,
|
||||
"",
|
||||
&repo_path,
|
||||
fake.to_str().unwrap(),
|
||||
oids,
|
||||
&db,
|
||||
Duration::from_secs(60),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("an unconfigured sink returns immediately");
|
||||
|
||||
assert!(pinned.is_empty(), "an empty JWT pins nothing");
|
||||
assert!(
|
||||
!log.exists(),
|
||||
"no git child may be spawned when the sink is not configured"
|
||||
);
|
||||
assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pin_skipped_when_jwt_empty() {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
@@ -121,6 +121,13 @@ impl RateLimiter {
|
||||
true
|
||||
}
|
||||
|
||||
/// Number of keys currently tracked. Tests use it to observe what a sweep
|
||||
/// reclaimed; there is no production reader.
|
||||
#[cfg(test)]
|
||||
pub async fn tracked_keys(&self) -> usize {
|
||||
self.state.lock().await.len()
|
||||
}
|
||||
|
||||
pub async fn cleanup(&self) {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock().await;
|
||||
@@ -132,6 +139,110 @@ impl RateLimiter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-source concurrency cap derived from the write-pool size: one resolved client
|
||||
/// key (see [`client_key`]) may hold at most an eighth of the pool, so saturating it
|
||||
/// takes ~8 distinct keys. Real for an IPv4 or single-address caller; a caller with a
|
||||
/// routed IPv6 /64 has 2^64 keys, because `client_key` returns the full address with
|
||||
/// no prefix folding. Narrowing that keying is a deferred decision, so this cap is a
|
||||
/// bound per key, not a bound per operator.
|
||||
///
|
||||
/// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0
|
||||
/// would shed EVERY receive-pack advertisement and break all pushes. The floor is
|
||||
/// load-bearing at the minimum write-pool size (1), which integer-divides to 0.
|
||||
pub(crate) fn per_source_push_cap(max_concurrent_git_pushes: usize) -> usize {
|
||||
(max_concurrent_git_pushes / 8).max(1)
|
||||
}
|
||||
|
||||
/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which
|
||||
/// caps request RATE. Each caller key may hold at most `per_caller` in-flight
|
||||
/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None`
|
||||
/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or
|
||||
/// DID) monopolizing the served-git read pool (#174).
|
||||
///
|
||||
/// The key map is self-bounding: a key is removed the instant its in-flight count
|
||||
/// reaches zero, so it never holds more keys than there are concurrently-active
|
||||
/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert
|
||||
/// backstop guarantees a key farm can never grow the map even if that invariant
|
||||
/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15).
|
||||
///
|
||||
/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the
|
||||
/// permit's `Drop` must release synchronously; the critical section holds no await.
|
||||
#[derive(Clone)]
|
||||
pub struct PerCallerConcurrency {
|
||||
state: Arc<std::sync::Mutex<HashMap<String, usize>>>,
|
||||
per_caller: usize,
|
||||
max_keys: usize,
|
||||
}
|
||||
|
||||
/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements
|
||||
/// the caller's in-flight count and removes the key when it reaches zero.
|
||||
pub struct PerCallerPermit {
|
||||
state: Arc<std::sync::Mutex<HashMap<String, usize>>>,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl PerCallerConcurrency {
|
||||
pub fn new(per_caller: usize, max_keys: usize) -> Self {
|
||||
Self {
|
||||
state: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
per_caller: per_caller.max(1),
|
||||
max_keys: max_keys.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience constructor with the default key bound.
|
||||
pub fn with_default_max_keys(per_caller: usize) -> Self {
|
||||
Self::new(per_caller, DEFAULT_MAX_KEYS)
|
||||
}
|
||||
|
||||
/// `Some(permit)` when the caller is under its cap and the map has room;
|
||||
/// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is
|
||||
/// rejected without allocating.
|
||||
pub fn try_acquire(&self, key: &str) -> Option<PerCallerPermit> {
|
||||
// Recover from a poisoned lock rather than panicking: the critical section
|
||||
// is pure counter arithmetic and cannot itself panic, so a poisoned mutex
|
||||
// would only ever come from an unrelated abort, and a slightly-off count
|
||||
// self-heals as permits drop. A panic here would instead brick the limiter
|
||||
// for every caller (each subsequent lock re-panics).
|
||||
let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
match map.get_mut(key) {
|
||||
Some(count) => {
|
||||
if *count >= self.per_caller {
|
||||
return None;
|
||||
}
|
||||
*count += 1;
|
||||
}
|
||||
None => {
|
||||
if map.len() >= self.max_keys {
|
||||
return None;
|
||||
}
|
||||
map.insert(key.to_string(), 1);
|
||||
}
|
||||
}
|
||||
Some(PerCallerPermit {
|
||||
state: self.state.clone(),
|
||||
key: key.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn tracked_keys(&self) -> usize {
|
||||
self.state.lock().unwrap().len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PerCallerPermit {
|
||||
fn drop(&mut self) {
|
||||
let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(count) = map.get_mut(&self.key) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
map.remove(&self.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn rate_limit_by_did(request: Request, next: Next) -> Response {
|
||||
let limiter = request.extensions().get::<RateLimiter>().cloned();
|
||||
|
||||
@@ -288,6 +399,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() {
|
||||
let lim = PerCallerConcurrency::new(2, 100);
|
||||
let p1 = lim.try_acquire("did:key:zA").expect("first under cap");
|
||||
let p2 = lim.try_acquire("did:key:zA").expect("second under cap");
|
||||
assert!(
|
||||
lim.try_acquire("did:key:zA").is_none(),
|
||||
"a third in-flight op for the same caller sheds (over the per-caller cap)"
|
||||
);
|
||||
// A DIFFERENT caller is unaffected — the cap is per-caller, not global.
|
||||
let _other = lim
|
||||
.try_acquire("did:key:zB")
|
||||
.expect("a different caller has its own budget");
|
||||
drop(p1);
|
||||
assert!(
|
||||
lim.try_acquire("did:key:zA").is_some(),
|
||||
"freeing one in-flight slot lets the same caller back in"
|
||||
);
|
||||
drop(p2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() {
|
||||
// Self-bounding: acquire+drop many distinct keys — the map never grows
|
||||
// because a key is removed the instant its in-flight count hits zero.
|
||||
let lim = PerCallerConcurrency::new(4, 3);
|
||||
for i in 0..50 {
|
||||
let _p = lim.try_acquire(&format!("k{i}"));
|
||||
}
|
||||
assert_eq!(
|
||||
lim.tracked_keys(),
|
||||
0,
|
||||
"keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty"
|
||||
);
|
||||
// Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds
|
||||
// WITHOUT growing the map past the cap (INV-15 — a rejected request never
|
||||
// allocates an entry).
|
||||
let held: Vec<_> = (0..3)
|
||||
.map(|i| lim.try_acquire(&format!("h{i}")).unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lim.tracked_keys(),
|
||||
3,
|
||||
"three distinct callers held concurrently"
|
||||
);
|
||||
assert!(
|
||||
lim.try_acquire("h3").is_none(),
|
||||
"a new key at max_keys is rejected"
|
||||
);
|
||||
assert_eq!(
|
||||
lim.tracked_keys(),
|
||||
3,
|
||||
"the rejected new key did not allocate an entry (reject-before-insert)"
|
||||
);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn allows_within_limit() {
|
||||
let limiter = RateLimiter::new(3, Duration::from_secs(60));
|
||||
|
||||
@@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router {
|
||||
// identity and can apply per-repo visibility (#110); anonymous callers stay
|
||||
// anonymous and still read genuinely public content. `/api/v1/ipfs/pins`
|
||||
// stays unsigned — gating the pin index is tracked separately (#121).
|
||||
// `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each
|
||||
// request can drive a full-history git walk, so the per-IP rate limiter is the
|
||||
// outermost layer (rejects a flood before the walk-admission work), mirroring the
|
||||
// push/create routers. The extension MUST be attached or rate_limit_by_ip is a
|
||||
// silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before.
|
||||
let ipfs_limiter = rate_limit::IpRateLimiter {
|
||||
limiter: state.ipfs_rate_limiter.clone(),
|
||||
trust: state.push_limiter_trust,
|
||||
};
|
||||
let ipfs_routes = Router::new()
|
||||
.route("/ipfs/{cid}", get(ipfs::get_by_cid))
|
||||
.layer(middleware::from_fn(auth::optional_signature))
|
||||
.layer(middleware::from_fn(rate_limit::rate_limit_by_ip))
|
||||
.layer(axum::Extension(ipfs_limiter))
|
||||
.merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins)));
|
||||
|
||||
// ── Arweave permanent anchors ──────────────────────────────────────────
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -605,7 +605,7 @@ async fn replicate_encrypted_blobs(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope).await {
|
||||
match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope, None).await {
|
||||
Ok(cid) if !cid.is_empty() => {
|
||||
if cid != blob.cid {
|
||||
warn!(oid = %blob.oid, expected = %blob.cid, got = %cid, "replicated envelope CID mismatch; skipping record");
|
||||
|
||||
@@ -82,6 +82,26 @@ fn build_state(db: Arc<crate::db::Db>, pool: PgPool) -> AppState {
|
||||
sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)),
|
||||
peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
|
||||
shutdown_tx: tokio::sync::watch::channel(false).0,
|
||||
// Generous — no test drives the handler-level shed (git_permit is unit-tested).
|
||||
git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
encrypt_inflight: crate::state::EncryptInflight::new(),
|
||||
repo_write_leases: crate::state::RepoWriteLeases::new(8),
|
||||
git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16),
|
||||
git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
8,
|
||||
),
|
||||
git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8),
|
||||
// Generous — a test that drives the /ipfs walk shed overrides these directly.
|
||||
git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)),
|
||||
git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(
|
||||
16,
|
||||
),
|
||||
ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)),
|
||||
git_bin: "git".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +129,25 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body
|
||||
.expect("request builder")
|
||||
}
|
||||
|
||||
/// A local endpoint whose TCP accept succeeds instantly but that never writes an
|
||||
/// HTTP response, so any request against it stalls deterministically until the
|
||||
/// caller's own timeout. (A non-routable address hangs only if the network
|
||||
/// blackholes the SYN — a fast RST would end the stall early and make a timeout
|
||||
/// test pass for the wrong reason.) The accepted sockets are parked in the
|
||||
/// spawned task, which dies with the test's runtime, so the peer never sees a
|
||||
/// close mid-test.
|
||||
pub(crate) async fn silent_http_endpoint() -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let endpoint = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let mut held = Vec::new();
|
||||
while let Ok((sock, _)) = listener.accept().await {
|
||||
held.push(sock);
|
||||
}
|
||||
});
|
||||
endpoint
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -187,6 +226,207 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the
|
||||
/// DB. The held `git_permit` acquire now sits after the per-source cap, so the
|
||||
/// cheap early shed is carried by an explicit `available_permits() == 0` check at
|
||||
/// the top of the handler (the held permit remains the authoritative bound further
|
||||
/// down). That check is a permit-less snapshot: it spares a request's DB work once
|
||||
/// the pool is ALREADY saturated, which is the case this test drives, and it does
|
||||
/// not bound the DB window in general. DB-free here because an exhausted semaphore
|
||||
/// sheds before any DB/disk access, so a lazy state works. Remove the early-shed block
|
||||
/// from git_info_refs and this goes red (the request falls through to the DB and
|
||||
/// returns something other than 503).
|
||||
#[tokio::test]
|
||||
async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() {
|
||||
let mut state = test_state_lazy();
|
||||
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/{owner}/{repo}/info/refs",
|
||||
axum::routing::get(crate::api::repos::git_info_refs),
|
||||
)
|
||||
.with_state(state);
|
||||
let resp = router
|
||||
.oneshot(anon_get(
|
||||
"/alice/repo.git/info/refs?service=git-upload-pack",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"an exhausted git semaphore must shed info/refs with 503 before touching the DB"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get("retry-after")
|
||||
.and_then(|h| h.to_str().ok()),
|
||||
Some("1"),
|
||||
"the 503 shed must carry Retry-After"
|
||||
);
|
||||
}
|
||||
|
||||
/// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same
|
||||
/// explicit `available_permits() == 0` early check at the top, so an ALREADY
|
||||
/// exhausted semaphore must shed the request with a 503 before its DB/disk work.
|
||||
/// That is the case the permit-less snapshot does deliver; it is not an admission
|
||||
/// bound on the DB window. Anonymous-reachable, so no auth injection is needed.
|
||||
/// Remove the early-shed block from git_upload_pack and this goes red.
|
||||
#[tokio::test]
|
||||
async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() {
|
||||
let mut state = test_state_lazy();
|
||||
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/{owner}/{repo}/git-upload-pack",
|
||||
axum::routing::post(crate::api::repos::git_upload_pack),
|
||||
)
|
||||
.with_state(state);
|
||||
let req = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/alice/repo.git/git-upload-pack")
|
||||
.body(Body::from(&b"0000"[..]))
|
||||
.unwrap();
|
||||
let resp = router.oneshot(req).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get("retry-after")
|
||||
.and_then(|h| h.to_str().ok()),
|
||||
Some("1"),
|
||||
"the 503 shed must carry Retry-After"
|
||||
);
|
||||
}
|
||||
|
||||
/// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed
|
||||
/// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174),
|
||||
/// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk
|
||||
/// work (the case the permit-less snapshot delivers, not an admission bound on the
|
||||
/// DB window), while the write pool (reserved for authenticated POSTs) is left
|
||||
/// free here.
|
||||
/// Flip the pool selection back to the write pool, or remove the early-shed
|
||||
/// block, and this goes red.
|
||||
#[tokio::test]
|
||||
async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() {
|
||||
let mut state = test_state_lazy();
|
||||
state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/{owner}/{repo}/info/refs",
|
||||
axum::routing::get(crate::api::repos::git_info_refs),
|
||||
)
|
||||
.with_state(state);
|
||||
let resp = router
|
||||
.oneshot(anon_get(
|
||||
"/alice/repo.git/info/refs?service=git-receive-pack",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get("retry-after")
|
||||
.and_then(|h| h.to_str().ok()),
|
||||
Some("1"),
|
||||
"the 503 shed must carry Retry-After"
|
||||
);
|
||||
}
|
||||
|
||||
/// PR3 (#62) sibling for the push path: git-receive-pack requires an
|
||||
/// AuthenticatedDid extension (production: require_signature injects it), so the
|
||||
/// request carries one via signed_request_as — without it the Extension
|
||||
/// extractor 500s before the handler body reaches the shed. What sits at the top of
|
||||
/// the handler is a permit-less `available_permits() == 0` peek, NOT the permit
|
||||
/// itself: the authoritative held acquire is taken after the per-repo lease, so a
|
||||
/// lease-blocked waiter pins no write slot. An ALREADY exhausted pool is the case
|
||||
/// the peek delivers, so the request sheds 503 before its DB work here. Remove the
|
||||
/// early-shed block from git_receive_pack and this goes red.
|
||||
#[tokio::test]
|
||||
async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() {
|
||||
let mut state = test_state_lazy();
|
||||
state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/{owner}/{repo}/git-receive-pack",
|
||||
axum::routing::post(crate::api::repos::git_receive_pack),
|
||||
)
|
||||
.with_state(state);
|
||||
let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
let resp = router
|
||||
.oneshot(signed_request_as(
|
||||
owner,
|
||||
Method::POST,
|
||||
"/alice/repo.git/git-receive-pack",
|
||||
Body::from(&b"0000"[..]),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"an exhausted write pool must shed git-receive-pack with 503 before touching the DB"
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get("retry-after")
|
||||
.and_then(|h| h.to_str().ok()),
|
||||
Some("1"),
|
||||
"the 503 shed must carry Retry-After"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an
|
||||
/// authenticated push — the write pool is a separate budget. Read pool at zero,
|
||||
/// write pool with capacity: the push proceeds PAST admission (it then errors on
|
||||
/// the placeholder DB, but crucially it is not a 503). Route git-receive-pack
|
||||
/// back to the read pool and this goes red — that is the isolation proof.
|
||||
#[tokio::test]
|
||||
async fn git_receive_pack_not_shed_by_exhausted_read_pool() {
|
||||
let mut state = test_state_lazy();
|
||||
// Read pool exhausted as if a flood of anonymous clones held every slot.
|
||||
state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
// Write pool keeps its default capacity from test_state_lazy.
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
"/{owner}/{repo}/git-receive-pack",
|
||||
axum::routing::post(crate::api::repos::git_receive_pack),
|
||||
)
|
||||
.with_state(state);
|
||||
let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA";
|
||||
let resp = router
|
||||
.oneshot(signed_request_as(
|
||||
owner,
|
||||
Method::POST,
|
||||
"/alice/repo.git/git-receive-pack",
|
||||
Body::from(&b"0000"[..]),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_ne!(
|
||||
resp.status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)"
|
||||
);
|
||||
}
|
||||
|
||||
/// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner
|
||||
/// before any git work (so no on-disk repo is needed for the rejection).
|
||||
#[sqlx::test]
|
||||
@@ -3285,7 +3525,10 @@ mod tests {
|
||||
/// the handler skips the whole repo rather than serving. Asserts no leak of the
|
||||
/// withheld blob AND that even the *public* blob in that repo is withheld — the
|
||||
/// latter distinguishes fail-closed-skip from normal per-blob withholding and
|
||||
/// would serve 200 if the error arm wrongly proceeded.
|
||||
/// would serve 200 if the error arm wrongly proceeded. The skip carries no
|
||||
/// VERDICT (F2), so the response is the retryable truncation 503, not a 404
|
||||
/// claiming the object is absent — never-serve-unproven and never-404-unproven
|
||||
/// hold together.
|
||||
#[sqlx::test]
|
||||
async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) {
|
||||
use crate::db::VisibilityMode;
|
||||
@@ -3330,7 +3573,8 @@ mod tests {
|
||||
.await
|
||||
.expect("deny rule");
|
||||
|
||||
// Withheld secret CID under a walk error → 404, no leak.
|
||||
// Withheld secret CID under a walk error → the repo is skipped without a
|
||||
// verdict, so the scan is truncated (503), and nothing leaks.
|
||||
let (st, body) = cid_parts(
|
||||
cid_router(&state)
|
||||
.oneshot(cid_anon(&secret_cid))
|
||||
@@ -3340,17 +3584,17 @@ mod tests {
|
||||
.await;
|
||||
assert_eq!(
|
||||
st,
|
||||
StatusCode::NOT_FOUND,
|
||||
"walk error must not serve the withheld blob"
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"walk error must not serve the withheld blob — the unproven skip sheds 503"
|
||||
);
|
||||
assert!(
|
||||
!body.contains("TOP SECRET"),
|
||||
"walk-error 404 must not leak the secret"
|
||||
"walk-error 503 must not leak the secret"
|
||||
);
|
||||
|
||||
// The PUBLIC blob in the same repo is also 404: the walk error fails closed
|
||||
// by skipping the whole repo, not by serving. Without the fail-closed arm
|
||||
// this would serve 200, so this assertion is the load-bearing discriminator.
|
||||
// The PUBLIC blob in the same repo is also not served: the walk error fails
|
||||
// closed by skipping the whole repo. Without the fail-closed arm this would
|
||||
// serve 200, so this assertion is the load-bearing discriminator.
|
||||
let (st, _) = cid_parts(
|
||||
cid_router(&state)
|
||||
.oneshot(cid_anon(&public_cid))
|
||||
@@ -3360,8 +3604,9 @@ mod tests {
|
||||
.await;
|
||||
assert_eq!(
|
||||
st,
|
||||
StatusCode::NOT_FOUND,
|
||||
"walk error fails closed: repo skipped, even the public blob is not served"
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"walk error fails closed: repo skipped without a verdict, even the public \
|
||||
blob is not served and the scan sheds 503"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
//! #174 U6 — INV-22 completeness guard (rung-raising).
|
||||
//!
|
||||
//! INV-22: a permit held per op recovers only if every path that holds it is also
|
||||
//! duration-bounded and reaps the process group before releasing admission, and every
|
||||
//! detached git/blocking task carries its own admission. PR #174 fixed five paths
|
||||
//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together
|
||||
//! those form the five-revert matrix. This guard adds the missing piece: a source-scan
|
||||
//! tripwire that fails when a NEW site reintroduces the class, or when one of the five
|
||||
//! gates is removed.
|
||||
//!
|
||||
//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same
|
||||
//! file it lives in would match its own identifier literals and pass vacuously. Here
|
||||
//! the scanned `src/` files never contain this file's literals, so each check is
|
||||
//! load-bearing — reverting the named gate turns the assertion red.
|
||||
//!
|
||||
//! These are deliberately coarse structural checks, not a parser. They cannot prove a
|
||||
//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and
|
||||
//! not bypassed*, which is what stops the class from silently regressing.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
fn src(rel: &str) -> String {
|
||||
let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel);
|
||||
std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inv22_concurrency_gates_present_and_not_bypassed() {
|
||||
let repos = src("api/repos.rs");
|
||||
let smart_http = src("git/smart_http.rs");
|
||||
let vis = src("git/visibility_pack.rs");
|
||||
|
||||
// U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the
|
||||
// child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise
|
||||
// a child that closes stdout then hangs pins the permit past the deadline. The
|
||||
// probe is defined and called, so >= 2 occurrences; reverting the fix removes both.
|
||||
assert!(
|
||||
vis.matches("child_terminated_without_reaping").count() >= 2,
|
||||
"U1/P1-a gate missing: run_bounded_git must confirm child exit via \
|
||||
child_terminated_without_reaping before signalling the watchdog"
|
||||
);
|
||||
|
||||
// U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper
|
||||
// that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned
|
||||
// via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the
|
||||
// timeout path already has an async context and never calls it).
|
||||
assert!(
|
||||
smart_http.contains("Handle::try_current"),
|
||||
"U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \
|
||||
TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM"
|
||||
);
|
||||
|
||||
// U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the
|
||||
// global write permit. The acquire reads `state.git_write_per_caller`; comments name
|
||||
// the field without the `state.` prefix, so this targets the real acquire site.
|
||||
assert!(
|
||||
repos.contains("state.git_write_per_caller"),
|
||||
"U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \
|
||||
(state.git_write_per_caller) before the global write permit"
|
||||
);
|
||||
|
||||
// U5 / P1-e: the detached post-push encryption walk must run through the
|
||||
// admission-gated helper, which is wired to the shared encrypt pool.
|
||||
assert!(
|
||||
repos.contains("fn withheld_recipients_gated")
|
||||
&& repos.contains("state.git_encrypt_semaphore"),
|
||||
"U5/P1-e gate missing: the encryption walk must run through \
|
||||
withheld_recipients_gated, which acquires git_encrypt_semaphore"
|
||||
);
|
||||
|
||||
// P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere
|
||||
// but inside withheld_recipients_gated. A second call site (count > 1) is a new
|
||||
// detached git walk that skips the admission gate — exactly the class U5 closed.
|
||||
assert_eq!(
|
||||
repos.matches("withheld_blob_recipients_bounded").count(),
|
||||
1,
|
||||
"P1-e bypass: the bounded recipients walk must be invoked only inside \
|
||||
withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap"
|
||||
);
|
||||
|
||||
// U4 / P2-2: the detached post-push encryption task must be gated by the per-repo
|
||||
// coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is
|
||||
// bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked
|
||||
// waiters (the unbounded set U4 closed); the semaphore only caps active walks.
|
||||
assert!(
|
||||
repos.contains("state.encrypt_inflight.try_begin"),
|
||||
"U4/P2-2 gate missing: the detached post-push encryption spawn must consult \
|
||||
encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)"
|
||||
);
|
||||
|
||||
// F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own
|
||||
// pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and
|
||||
// the task must loop-drain them (`finish_or_take_pending`) before releasing it.
|
||||
// Removing the drain reverts to the silent loss: a coalesced push's pins and
|
||||
// recovery copies are absent until an unrelated later push. Scan only the
|
||||
// production half of the file — the u5 tests in its `mod tests` also name the
|
||||
// drain call, and matching them would make this check vacuous.
|
||||
let repos_production = repos
|
||||
.split("#[cfg(test)]")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
assert!(
|
||||
repos_production.contains("finish_or_take_pending"),
|
||||
"F5 gate missing: the post-push encryption task must loop-drain coalesced \
|
||||
pushes via finish_or_take_pending before releasing its repo key"
|
||||
);
|
||||
|
||||
// F4: every post-receive scan helper admits itself to the shared scan pool via
|
||||
// `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a
|
||||
// push burst cannot accumulate unbounded concurrent scans once the write permit
|
||||
// is released. Two halves, both load-bearing: the helper body must actually
|
||||
// acquire the pool (state.rs sits the helper at the end of the file, so the
|
||||
// definition tail contains no other `acquire_owned` to match vacuously), and
|
||||
// within each scan helper the first qualified call precedes the first
|
||||
// `spawn_blocking`. Severing a call site pushes the next occurrence past the
|
||||
// helper's own `spawn_blocking` (or off the end of the file), turning the
|
||||
// assertion red; comments name the helper without the `crate::state::` prefix,
|
||||
// so this targets the real call sites.
|
||||
let state_rs = src("state.rs");
|
||||
let helper_def = state_rs
|
||||
.find("fn acquire_scan_permit")
|
||||
.expect("F4 gate missing: state.rs no longer defines acquire_scan_permit");
|
||||
assert!(
|
||||
state_rs[helper_def..].contains("acquire_owned"),
|
||||
"F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned"
|
||||
);
|
||||
let push_delta = src("git/push_delta.rs");
|
||||
for (file_src, file, helper) in [
|
||||
(&repos, "api/repos.rs", "fn replication_withheld_set"),
|
||||
(&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"),
|
||||
(
|
||||
&push_delta,
|
||||
"git/push_delta.rs",
|
||||
"fn resolve_candidates_for_push",
|
||||
),
|
||||
] {
|
||||
let start = file_src
|
||||
.find(helper)
|
||||
.unwrap_or_else(|| panic!("{file}: `{helper}` not found"));
|
||||
let tail = &file_src[start..];
|
||||
let acquire = tail
|
||||
.find("crate::state::acquire_scan_permit(")
|
||||
.unwrap_or_else(|| {
|
||||
panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit")
|
||||
});
|
||||
let spawn = tail.find("spawn_blocking").unwrap_or_else(|| {
|
||||
panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard")
|
||||
});
|
||||
assert!(
|
||||
acquire < spawn,
|
||||
"F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \
|
||||
permit BEFORE dispatching the blocking git scan"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release`
|
||||
/// must await `pg_advisory_unlock` while `self` still owns the pooled connection,
|
||||
/// and must not mark itself `released` until that await resolves. Either shape,
|
||||
/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection
|
||||
/// early leaves `Drop` with `conn == None`, and setting `released = true` early
|
||||
/// leaves the `Drop` backstop inert — both strand the session lock on cancellation.
|
||||
///
|
||||
/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the
|
||||
/// connection and unlocks, so a whole-file scan would match it and read as a false
|
||||
/// pass. Reverting either ordering turns this red (proven load-bearing).
|
||||
#[test]
|
||||
fn f4_release_keeps_conn_owned_until_unlock_resolves() {
|
||||
let repo_store = src("git/repo_store.rs");
|
||||
|
||||
let rel_start = repo_store
|
||||
.find("pub async fn release(mut self")
|
||||
.expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release");
|
||||
let rel_end = repo_store[rel_start..]
|
||||
.find("impl Drop for RepoWriteGuard")
|
||||
.map(|off| rel_start + off)
|
||||
.expect("F4 gate: release fn / Drop impl markers moved — update this guard");
|
||||
let release_body = &repo_store[rel_start..rel_end];
|
||||
|
||||
let unlock = release_body
|
||||
.find("pg_advisory_unlock")
|
||||
.expect("F4 gate: release must still issue pg_advisory_unlock");
|
||||
let before_unlock = &release_body[..unlock];
|
||||
|
||||
// (a) the connection must still be owned by `self` at the unlock await.
|
||||
assert!(
|
||||
!before_unlock.contains("self.conn.take()"),
|
||||
"F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \
|
||||
pg_advisory_unlock. A cancellation during the unlock await then strands the \
|
||||
session advisory lock (Drop sees conn == None and skips its backstop). \
|
||||
Unlock through the still-owned connection instead."
|
||||
);
|
||||
// (b) `released` must not be set before the unlock await — the other
|
||||
// reintroduction shape a single-reorder check on (a) alone is blind to.
|
||||
assert!(
|
||||
!before_unlock.contains("released = true"),
|
||||
"F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \
|
||||
pg_advisory_unlock. A cancellation during the await then leaves the Drop \
|
||||
backstop inert (it early-returns on released). Set released only AFTER the \
|
||||
unlock await resolves."
|
||||
);
|
||||
}
|
||||
|
||||
/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires
|
||||
/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial
|
||||
/// metadata queries, and the per-repo loop's first budget gate runs only later. So
|
||||
/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the
|
||||
/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot
|
||||
/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs`
|
||||
/// (the `mod tests` half names the same calls in its own harness and would make the
|
||||
/// check vacuous) and asserts each query call is immediately preceded by a
|
||||
/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven
|
||||
/// load-bearing).
|
||||
#[test]
|
||||
fn f6_ipfs_metadata_queries_are_deadline_wrapped() {
|
||||
let ipfs = src("api/ipfs.rs");
|
||||
let production = ipfs
|
||||
.split("#[cfg(test)]")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] {
|
||||
assert_eq!(
|
||||
production.matches(call).count(),
|
||||
1,
|
||||
"F6 guard stale: `{call}` must appear exactly once in the production half \
|
||||
of api/ipfs.rs (the deadline-wrapped handler call) — update this guard"
|
||||
);
|
||||
let idx = production
|
||||
.find(call)
|
||||
.unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`"));
|
||||
// The wrapper opens a few lines above the call (match tokio::time::timeout( ...
|
||||
// remaining budget ..., <query> )); a 240-char lookback covers it without
|
||||
// reaching the previous statement.
|
||||
let window = &production[idx.saturating_sub(240)..idx];
|
||||
assert!(
|
||||
window.contains("tokio::time::timeout("),
|
||||
"F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \
|
||||
clamped to the remaining request budget. An unwrapped await pins the held \
|
||||
walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue
|
||||
/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the
|
||||
/// worker once a pin slot frees — it must NOT move the full per-push object list into
|
||||
/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes
|
||||
/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so
|
||||
/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is
|
||||
/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained
|
||||
/// data, not the task count.
|
||||
///
|
||||
/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the
|
||||
/// `mod tests` half names the same identifiers in its own harness and would make the
|
||||
/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form —
|
||||
/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red.
|
||||
/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata
|
||||
/// pin permit is acquired, so the object list is materialized only inside the pin-bounded
|
||||
/// section and a parked task holds O(ref tuples).
|
||||
#[test]
|
||||
fn f2_pinata_enqueues_refs_not_retained_object_lists() {
|
||||
let repos = src("api/repos.rs");
|
||||
let production = repos
|
||||
.split("#[cfg(test)]")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
// (a) the retained-list form must be gone from production.
|
||||
assert!(
|
||||
!production.contains("object_list_pinata"),
|
||||
"F2/KTD-3 regression: the Pinata task retains a full per-push object list \
|
||||
(`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \
|
||||
tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)."
|
||||
);
|
||||
|
||||
// (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the
|
||||
// Pinata pin-admission clone so the window is the Pinata task, not the sibling
|
||||
// IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives).
|
||||
let anchor = production
|
||||
.find("let pin_sem_pinata")
|
||||
.expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved");
|
||||
let tail = &production[anchor..];
|
||||
let acquire = tail
|
||||
.find(".acquire_owned()")
|
||||
.expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)");
|
||||
let rederive = tail.find("pinata_object_list_for_refs(").expect(
|
||||
"F2 gate missing: the Pinata task must re-derive its object set via \
|
||||
pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure",
|
||||
);
|
||||
assert!(
|
||||
acquire < rederive,
|
||||
"F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \
|
||||
permit is acquired, so a parked task never holds the MB-scale object list"
|
||||
);
|
||||
|
||||
// The re-derivation is driven by the push's ref tuples (the enqueued unit), not a
|
||||
// retained object list — tie "enqueue ref_updates" to the call explicitly.
|
||||
assert!(
|
||||
tail[rederive..].starts_with("pinata_object_list_for_refs(")
|
||||
&& tail[rederive..]
|
||||
.get(..400)
|
||||
.map(|w| w.contains("ref_updates_clone"))
|
||||
.unwrap_or(false),
|
||||
"F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \
|
||||
(ref_updates_clone), the small unit the parked task retains"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's
|
||||
/// git group is reaped): `git_receive_pack` must take the per-repo in-process write
|
||||
/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the
|
||||
/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is
|
||||
/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at
|
||||
/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the
|
||||
/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test
|
||||
/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this
|
||||
/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs`
|
||||
/// (the `mod tests` half names these identifiers in its own harness).
|
||||
#[test]
|
||||
fn f3_second_writer_leased_until_reap() {
|
||||
let repos = src("api/repos.rs");
|
||||
let smart_http = src("git/smart_http.rs");
|
||||
let repos_production = repos
|
||||
.split("#[cfg(test)]")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
// The lease is acquired, then carried by the AdmissionGuard (.with_lease), before
|
||||
// receive_pack runs the write. Severing any of the three turns this red.
|
||||
let lease_acquire = repos_production.find("repo_write_leases").expect(
|
||||
"F3 gate missing: git_receive_pack no longer takes the per-repo write lease \
|
||||
(state.repo_write_leases)",
|
||||
);
|
||||
let with_lease = repos_production.find(".with_lease(").expect(
|
||||
"F3 gate missing: the write-path AdmissionGuard no longer carries the lease \
|
||||
(.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \
|
||||
NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).",
|
||||
);
|
||||
let receive = repos_production
|
||||
.find("smart_http::receive_pack(")
|
||||
.expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack");
|
||||
assert!(
|
||||
lease_acquire < with_lease && with_lease < receive,
|
||||
"F3 gate bypassed: the write lease must be acquired, then carried by the \
|
||||
AdmissionGuard (.with_lease), BEFORE receive_pack runs the write"
|
||||
);
|
||||
|
||||
// The AdmissionGuard must actually hold the lease (Option<RepoWriteLease>) via a
|
||||
// with_lease setter, so it travels into the detached reaper. Removing the field or
|
||||
// method turns this red.
|
||||
assert!(
|
||||
smart_http.contains("_lease: Option<crate::state::RepoWriteLease>")
|
||||
&& smart_http.contains("pub fn with_lease("),
|
||||
"F3 gate missing: AdmissionGuard must hold an Option<RepoWriteLease> set via \
|
||||
with_lease, so the lease rides the guard into KillGroupOnDrop's reaper"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 U1 — every blocking walk in the `/ipfs` scan carries the request's walk
|
||||
/// admission.
|
||||
///
|
||||
/// The admission is an `Arc<WalkAdmission>` cloned into each `spawn_blocking`
|
||||
/// closure, so the global + per-source permits release only when the last holder
|
||||
/// drops: a client disconnect leaves the abandoned closure holding the slot, and a
|
||||
/// panicking closure leaves the handler holding it.
|
||||
///
|
||||
/// Only the walk site runs under `state.git_bin`, so only it can be pinned by the
|
||||
/// fake-git harness and mutation-verified dynamically (see
|
||||
/// `get_by_cid_walk_permit_held_through_blocking_walk`). The probe and the content
|
||||
/// read deliberately shell to the real `git`, which is why this structural check
|
||||
/// exists: it binds all three sites, and any blocking site added to this loop
|
||||
/// later, without reversing that deliberate independence.
|
||||
///
|
||||
/// MUTATION (RED): delete any one `Arc::clone(&admission)` binding, or drop the
|
||||
/// clone from inside its closure, and the count falls below three.
|
||||
#[test]
|
||||
fn inv22_ipfs_walk_admission_reaches_every_blocking_site() {
|
||||
let ipfs = src("api/ipfs.rs");
|
||||
|
||||
// The shared owner must exist and be built once per request.
|
||||
assert!(
|
||||
ipfs.contains("struct WalkAdmission") && ipfs.contains("Arc::new(WalkAdmission {"),
|
||||
"U1 gate missing: the /ipfs walk admission must be a shared WalkAdmission, \
|
||||
not a handler-local permit pair"
|
||||
);
|
||||
|
||||
// Every blocking site in the scan takes its own clone...
|
||||
let clones = ipfs.matches("Arc::clone(&admission)").count();
|
||||
assert!(
|
||||
clones >= 3,
|
||||
"U1 gate bypassed: expected an admission clone for each of the three \
|
||||
/ipfs spawn_blocking sites (probe, walk, read); found {clones}. A blocking \
|
||||
walk that does not hold the admission lets a disconnect or a panic free the \
|
||||
slot while its git child is still running."
|
||||
);
|
||||
|
||||
// ...and each clone is actually moved INTO the blocking closure, not merely
|
||||
// created in the async frame (which would hold nothing across the join).
|
||||
let held = ipfs.matches("let _admission = ").count();
|
||||
assert!(
|
||||
held >= 3,
|
||||
"U1 gate bypassed: each admission clone must be bound inside its \
|
||||
spawn_blocking closure so the blocking work owns it; found {held} of 3."
|
||||
);
|
||||
|
||||
// The count of blocking sites is itself the thing being covered: if a fourth
|
||||
// appears, it needs an admission clone too and this gate must be revisited.
|
||||
let sites = ipfs.matches("spawn_blocking(move ||").count();
|
||||
assert_eq!(
|
||||
sites, 3,
|
||||
"the /ipfs scan grew or lost a spawn_blocking site ({sites} found, expected 3); \
|
||||
give any new blocking walk its own Arc::clone(&admission) and update this gate"
|
||||
);
|
||||
}
|
||||
|
||||
/// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY,
|
||||
/// which is the moment receive-pack returns success, not the end of the handler and
|
||||
/// not after `guard.release()`.
|
||||
///
|
||||
/// The tail owes this push its pins, recovery copy, and announcements. Everything
|
||||
/// below the spawn stays in the cancellable request future, so anything the tail is
|
||||
/// spawned after is a window where a client disconnect drops that work while the pack
|
||||
/// is already durable on disk. `guard.release()` is such a window: on success it
|
||||
/// awaits the Tigris upload and then the advisory unlock.
|
||||
///
|
||||
/// The lower bound matters just as much as the upper one: `release` runs on failure
|
||||
/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing
|
||||
/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be
|
||||
/// what gates it, so the success check is explicit and this gate binds it: the spawn
|
||||
/// must sit inside `if push_succeeded`, and `release` must consume the same flag so
|
||||
/// the two cannot drift apart.
|
||||
///
|
||||
/// This is an ordering check rather than a cancellation-race test on purpose: it is
|
||||
/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which
|
||||
/// drives the actual disconnect through a parked `release`. Same instrument the F3
|
||||
/// gate above uses.
|
||||
///
|
||||
/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below
|
||||
/// `guard.release(` and the ordering assertion fails; take it out of the
|
||||
/// `if push_succeeded` block and the failed-push assertion fails.
|
||||
#[test]
|
||||
fn inv22_replication_tail_spawns_at_the_durability_boundary() {
|
||||
let repos = src("api/repos.rs");
|
||||
// Production half only — the tests below name these identifiers too.
|
||||
let production = repos
|
||||
.split("#[cfg(test)]")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
let success_flag = production
|
||||
.find("let push_succeeded = receive_result.is_ok();")
|
||||
.expect(
|
||||
"U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()",
|
||||
);
|
||||
let gate_open = production
|
||||
.find("if push_succeeded {")
|
||||
.expect("U5 gate missing: the tail spawn must be gated on the push having succeeded");
|
||||
let spawn = production
|
||||
.find("tokio::spawn(post_receive_replication_tail(")
|
||||
.expect("U5 gate missing: the replication tail must be spawned by git_receive_pack");
|
||||
let release = production
|
||||
.find("guard.release(push_succeeded)")
|
||||
.expect("U5 gate stale: release must consume the same success flag as the tail gate");
|
||||
let touch = production
|
||||
.find("state.db.touch_repo(")
|
||||
.expect("U5 gate stale: git_receive_pack no longer calls touch_repo");
|
||||
let webhook = production
|
||||
.find("webhooks::fire_event(")
|
||||
.expect("U5 gate stale: git_receive_pack no longer fires push webhooks");
|
||||
|
||||
assert!(
|
||||
success_flag < gate_open && gate_open < spawn,
|
||||
"U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \
|
||||
rejected push spawns a tail that pins and announces a half-applied repo"
|
||||
);
|
||||
// Still inside that block: no `}` may close it between the gate and the spawn.
|
||||
assert!(
|
||||
!production[gate_open + "if push_succeeded {".len()..spawn].contains('}'),
|
||||
"U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \
|
||||
rejected push now spawns a tail"
|
||||
);
|
||||
assert!(
|
||||
spawn < release && spawn < touch && spawn < webhook,
|
||||
"U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \
|
||||
and the webhook fan-out, so a disconnect in any of those windows cannot drop \
|
||||
this push's pins, recovery copy, and announcements"
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user