mirror of
https://github.com/Twigpine/node.git
synced 2026-11-09 18:08:02 -10:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 beardthelion
|
d30136efdd | ||
|
beardthelion
|
1544bedea7 | ||
|
carlitos973
|
15e017ad77 |
No files matched your search
@@ -380,21 +380,6 @@ GITLAWB_TRUSTED_PROXY=
|
||||
# Enable automatic background sync from known peers
|
||||
GITLAWB_AUTO_SYNC=false
|
||||
|
||||
# ── External upstream mirroring ───────────────────────────────────────────
|
||||
# Default-off scheduled fetch worker for repositories configured as INBOUND.
|
||||
# Enabling requires GITLAWB_ENFORCE_OWNER_PUSH=true; the node refuses to start
|
||||
# otherwise. The worker updates branches and tags atomically while preserving
|
||||
# Gitlawb-owned review/internal refs. Enabling also requires Git >= 2.37 on
|
||||
# the host for http.curloptResolve DNS pinning; the node refuses to start
|
||||
# when the installed Git is older.
|
||||
GITLAWB_UPSTREAM_MIRROR_ENABLED=false
|
||||
# Pause after each complete scan. Must be 5..=86400 seconds.
|
||||
GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS=300
|
||||
# Deadline for the external fetch child only; the preflight probes run on
|
||||
# their own short bound. Must be 1..=3153600000 seconds.
|
||||
GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS=600
|
||||
# Keyset page size; every page is processed before the next scan delay.
|
||||
GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE=100
|
||||
# ── Reconciliation sweep ─────────────────────────────────────────────────
|
||||
# Periodic durability sweep: re-derives the public pin set and the withheld-blob
|
||||
# recovery set each hour and fills gaps so a dropped replication job never means
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
name: CodeQL
|
||||
|
||||
# Advanced-setup replacement for CodeQL default setup. Default setup analyzes
|
||||
# every detected language on every PR, including Swift (macos-app/), which
|
||||
# autobuilds on a macOS runner for ~13 minutes per PR and has hung for 55 on
|
||||
# main; that single job was the longest check on most PRs. Here a PR analyzes
|
||||
# only the languages it touches (scripts/ci-surfaces.sh), while push to main
|
||||
# and the weekly schedule analyze every language, so main's code-scanning
|
||||
# baseline stays complete.
|
||||
#
|
||||
# Requires CodeQL default setup to be DISABLED (Settings -> Code security ->
|
||||
# Code scanning): GitHub rejects advanced-setup uploads while it is enabled.
|
||||
#
|
||||
# A PR that skips a language may show "configuration not found" on the CodeQL
|
||||
# check for that language; that is informational and expected.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "17 4 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
name: changed languages
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
codeql: ${{ steps.surfaces.outputs.codeql }}
|
||||
matrix: ${{ steps.surfaces.outputs.codeql_matrix }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Map changed paths to languages
|
||||
id: surfaces
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$EVENT_NAME" = "pull_request" ]; then
|
||||
# -z: literal paths; plain --name-only C-quotes unusual names.
|
||||
git diff -z --name-only --no-renames HEAD^1 HEAD > changed-paths
|
||||
tr '\0' '\n' < changed-paths >&2
|
||||
scripts/ci-surfaces.sh < changed-paths > surfaces.out
|
||||
else
|
||||
scripts/ci-surfaces.sh --all > surfaces.out
|
||||
fi
|
||||
cat surfaces.out
|
||||
grep -E '^(codeql|codeql_matrix)=' surfaces.out >> "$GITHUB_OUTPUT"
|
||||
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
needs: changes
|
||||
if: needs.changes.outputs.codeql == 'true'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
# Normal runs take <=14 min (Swift autobuild is the longest); cap a hung
|
||||
# autobuild instead of letting it hold a runner for an hour.
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
packages: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.changes.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
|
||||
- name: Autobuild
|
||||
if: matrix.build-mode == 'autobuild'
|
||||
uses: github/codeql-action/autobuild@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
@@ -30,62 +30,8 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Decide which checks this event needs. On pull_request, each job below runs
|
||||
# only when the PR touches the surface it validates, and the cargo test /
|
||||
# release-build jobs are scoped to the touched crates plus their reverse
|
||||
# dependencies (scripts/ci-surfaces.sh). Every other event (push to main,
|
||||
# merge_group, the release flow's workflow_dispatch) runs the full suite, so
|
||||
# main and the merge queue keep whole-workspace coverage and main-red-alert
|
||||
# still sees a complete run. A skipped job reports as "skipped", not green.
|
||||
changes:
|
||||
name: changed surfaces
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
rust: ${{ steps.surfaces.outputs.rust }}
|
||||
rust_packages: ${{ steps.surfaces.outputs.rust_packages }}
|
||||
windows: ${{ steps.surfaces.outputs.windows }}
|
||||
windows_packages: ${{ steps.surfaces.outputs.windows_packages }}
|
||||
docker: ${{ steps.surfaces.outputs.docker }}
|
||||
audit: ${{ steps.surfaces.outputs.audit }}
|
||||
core_deps: ${{ steps.surfaces.outputs.core_deps }}
|
||||
release_versioning: ${{ steps.surfaces.outputs.release_versioning }}
|
||||
release_tag: ${{ steps.surfaces.outputs.release_tag }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
# The pull_request checkout is the test-merge commit; depth 2 brings
|
||||
# its first parent (the base tip) so the diff is exactly this PR.
|
||||
fetch-depth: 2
|
||||
|
||||
# Every job's gate comes from the mapper, so pin its mapping first; a
|
||||
# failure here fails this job and the run is red, not silently narrowed.
|
||||
- name: Test the surface mapper
|
||||
run: scripts/test-ci-surfaces.sh
|
||||
|
||||
- name: Map changed paths to surfaces
|
||||
id: surfaces
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$EVENT_NAME" = "pull_request" ]; then
|
||||
# -z: literal paths; plain --name-only C-quotes unusual names.
|
||||
git diff -z --name-only --no-renames HEAD^1 HEAD > changed-paths
|
||||
tr '\0' '\n' < changed-paths >&2
|
||||
scripts/ci-surfaces.sh < changed-paths > surfaces.out
|
||||
else
|
||||
scripts/ci-surfaces.sh --all > surfaces.out
|
||||
fi
|
||||
cat surfaces.out
|
||||
cat surfaces.out >> "$GITHUB_OUTPUT"
|
||||
|
||||
fmt-clippy:
|
||||
name: fmt + clippy
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -111,8 +57,6 @@ jobs:
|
||||
|
||||
release-tag-validation:
|
||||
name: release tag validation
|
||||
needs: changes
|
||||
if: needs.changes.outputs.release_tag == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
@@ -142,8 +86,6 @@ jobs:
|
||||
|
||||
test:
|
||||
name: test (${{ matrix.toolchain }})
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
@@ -188,17 +130,10 @@ jobs:
|
||||
- name: cargo test
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test
|
||||
# `--workspace`, or `-p <crate>...` for the crates a PR touches plus
|
||||
# every crate that depends on them.
|
||||
PACKAGES: ${{ needs.changes.outputs.rust_packages }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086 # PACKAGES is a word list by design
|
||||
cargo test --locked $PACKAGES
|
||||
run: cargo test --locked --workspace
|
||||
|
||||
build-release:
|
||||
name: build --release
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
@@ -218,20 +153,10 @@ jobs:
|
||||
key: release
|
||||
|
||||
- name: cargo build --release
|
||||
env:
|
||||
PACKAGES: ${{ needs.changes.outputs.rust_packages }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086 # PACKAGES is a word list by design
|
||||
cargo build --locked --release $PACKAGES
|
||||
run: cargo build --locked --release --workspace
|
||||
|
||||
audit:
|
||||
name: cargo audit
|
||||
# cargo audit reads only Cargo.lock (+ .cargo/audit.toml), so a PR that
|
||||
# leaves both unchanged cannot change its verdict; a newly published
|
||||
# advisory against main is caught by the push-to-main run and the weekly
|
||||
# Scheduled Audit instead of turning every unrelated PR red.
|
||||
needs: changes
|
||||
if: needs.changes.outputs.audit == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -291,8 +216,6 @@ jobs:
|
||||
|
||||
msrv:
|
||||
name: MSRV (Rust 1.91)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -316,12 +239,6 @@ jobs:
|
||||
|
||||
docker-build:
|
||||
name: Docker build smoke test
|
||||
# Runs when the image's own inputs change: the Dockerfile, the manifests
|
||||
# its dependency layer copies, the lockfile, bootstrap-peers.json. A
|
||||
# source-only Rust change is already compiled by `build --release` (stable)
|
||||
# and `MSRV` (1.91, the image's toolchain); push to main still runs this.
|
||||
needs: changes
|
||||
if: needs.changes.outputs.docker == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -338,8 +255,6 @@ jobs:
|
||||
|
||||
test-windows:
|
||||
name: test (windows, non-blocking)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.windows == 'true'
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 45
|
||||
# Non-blocking, mirroring the informational `beta` lane above. We ship `gl`
|
||||
@@ -367,13 +282,7 @@ jobs:
|
||||
key: windows
|
||||
|
||||
- name: cargo test (shipped Windows crates)
|
||||
shell: bash
|
||||
env:
|
||||
# `-p gl -p git-remote-gitlawb`, or whichever of the two a PR affects.
|
||||
PACKAGES: ${{ needs.changes.outputs.windows_packages }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086 # PACKAGES is a word list by design
|
||||
cargo test --locked $PACKAGES
|
||||
run: cargo test --locked -p gl -p git-remote-gitlawb
|
||||
|
||||
# gitlawb-core is embedded by every consumer (gl, git-remote-gitlawb, the node
|
||||
# daemon), so it must stay lean. This gate fails if gitlawb-core's normal
|
||||
@@ -383,8 +292,6 @@ jobs:
|
||||
# cannot do. Regen instructions live in the allowlist header.
|
||||
core-deps-purity:
|
||||
name: gitlawb-core dependency purity
|
||||
needs: changes
|
||||
if: needs.changes.outputs.core_deps == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
@@ -417,8 +324,6 @@ jobs:
|
||||
# not its author knew this gate existed.
|
||||
release-versioning:
|
||||
name: release versioning coverage
|
||||
needs: changes
|
||||
if: needs.changes.outputs.release_versioning == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
|
||||
@@ -40,5 +40,5 @@ When a node denies a request, surface the denial to the user. Never render a den
|
||||
## Toolchain and CI reality
|
||||
|
||||
- MSRV is Rust 1.91 (`rust-version` in the workspace `Cargo.toml`). The MSRV CI job runs `cargo check --workspace --all-targets` on exactly 1.91, so check that an API you rely on is stable there; the full test suite runs on stable.
|
||||
- The PR workflow's jobs are `cargo fmt --check`, `cargo clippy --workspace --all-targets -- -D warnings`, `cargo test` (against Postgres), a release build, `cargo audit`, the MSRV check, and a Docker build. On a PR, `scripts/ci-surfaces.sh` runs each job only when the PR touches its inputs, and scopes `cargo test` and the release build to the touched crates plus every crate that depends on them. Root `Cargo.toml`/`Cargo.lock`/`.cargo/` changes run the Rust jobs (fmt, clippy, tests, release build, MSRV) on the whole workspace; every other job still runs only when its own inputs change. Edits to the CI wiring itself (`pr-checks.yml`, `codeql.yml`, `scripts/ci-surfaces.sh`), push to `main`, and the merge queue run every job on the whole workspace. Edits to any other workflow run none of these jobs on a PR; CodeQL only analyzes them as Actions code. Run `cargo test --workspace` locally regardless. The same script limits CodeQL (`codeql.yml`) to the languages a PR touches. The `main` branch ruleset currently requires one approving review but does not require these checks to pass before merge; a failing job should block review approval in practice, but nothing enforces that mechanically today.
|
||||
- The PR workflow runs these jobs on every PR: `cargo fmt --check`, `cargo clippy --workspace --all-targets -- -D warnings`, `cargo test --workspace` (against Postgres), a release build, `cargo audit`, the MSRV check, and a Docker build. The `main` branch ruleset currently requires one approving review but does not require these checks to pass before merge; a failing job should block review approval in practice, but nothing enforces that mechanically today.
|
||||
- Conventional commit titles (`feat:`, `fix:`, `docs:`, ...) are required by convention and drive releases, but no CI job checks them; a bad title fails review, not the pipeline.
|
||||
Generated
+3
-3
@@ -3085,7 +3085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6229,7 +6229,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7163,7 +7163,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -84,7 +84,7 @@ pub(crate) const PUBLIC_HTTP_URL_REQUIREMENT: &str = "must be a public http(s) U
|
||||
/// sits at a prefix-length-dependent offset (RFC 6052 §2.2) — so they return
|
||||
/// `None`. Any caller that needs them blocked must reject the wider
|
||||
/// `64:ff9b::/32` itself; `is_public_http_url` does this in its native-v6 arm.
|
||||
pub(crate) fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
|
||||
fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
|
||||
if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) {
|
||||
return Some(v4);
|
||||
}
|
||||
@@ -104,116 +104,6 @@ pub(crate) fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether an IP literal is outside the local/private ranges that an
|
||||
/// attacker-controlled outbound URL must never reach. Kept separate from URL
|
||||
/// parsing so callers that resolve a DNS name themselves can apply the same
|
||||
/// classification to every answer and then pin the approved addresses.
|
||||
pub(crate) fn is_public_ip(ip: std::net::IpAddr) -> bool {
|
||||
// Reject loopback/unspecified on the literal as given — catches `::1` and
|
||||
// `::` before the IPv4-folding below (`::1`.to_ipv4() would otherwise map
|
||||
// to a non-loopback `0.0.0.1`).
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
// Fold any IPv6 literal that embeds an IPv4 address (mapped, compatible,
|
||||
// 6to4, NAT64) down to that IPv4 so the v4 range checks catch
|
||||
// loopback/private addresses smuggled in via an IPv6 encoding, then
|
||||
// re-check loopback/unspecified.
|
||||
let ip = match ip {
|
||||
std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip),
|
||||
v4 => v4,
|
||||
};
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
match ip {
|
||||
std::net::IpAddr::V4(v4) => {
|
||||
let o = v4.octets();
|
||||
// RFC1918 private, link-local, CGNAT (100.64.0.0/10), or the
|
||||
// RFC1122 "this host" block 0.0.0.0/8 (never a valid destination;
|
||||
// 0.0.0.0 itself is already caught by the is_unspecified check).
|
||||
if v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| (o[0] == 100 && (o[1] & 0xc0) == 64)
|
||||
|| o[0] == 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
std::net::IpAddr::V6(v6) => {
|
||||
let s = v6.segments();
|
||||
// fc00::/7 (unique-local), fe80::/10 (link-local), or fec0::/10
|
||||
// (site-local, deprecated by RFC 3879; the link-local mask does
|
||||
// not cover it).
|
||||
if (s[0] & 0xfe00) == 0xfc00 || (s[0] & 0xffc0) == 0xfe80 || (s[0] & 0xffc0) == 0xfec0 {
|
||||
return false;
|
||||
}
|
||||
// Any NAT64 address (64:ff9b::/32) that is not the cleanly
|
||||
// decodable well-known /96 — e.g. the RFC 8215 local-use
|
||||
// 64:ff9b:1::/48 — carries a translated target whose embedded v4
|
||||
// sits at a prefix-length-dependent offset (RFC 6052 §2.2).
|
||||
// Rather than risk a wrong decode across every prefix length we
|
||||
// reject the whole NAT64 space here. The well-known /96 was already
|
||||
// folded to its v4 above and never reaches this arm.
|
||||
if s[0] == 0x0064 && s[1] == 0xff9b {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Mirror-upstream variant of [`is_public_ip`]: in addition to the shared
|
||||
/// peer classification, rejects IANA special-purpose ranges that are
|
||||
/// non-global, locally meaningful, or unsafe as owner-controlled outbound
|
||||
/// fetch endpoints. For IP-literal upstreams, configure-time URL validation
|
||||
/// and the fetch-time egress policy share this classifier, so a stored
|
||||
/// literal can never fail the worker it is scheduled for. Hostname upstreams
|
||||
/// are only shape-checked at configure time: each fetch resolves DNS and
|
||||
/// applies this classifier to the answers, so a hostname whose answers turn
|
||||
/// non-public still fails at fetch time.
|
||||
pub(crate) fn mirror_upstream_ip_permitted(ip: std::net::IpAddr) -> bool {
|
||||
if !is_public_ip(ip) {
|
||||
return false;
|
||||
}
|
||||
let classified = match ip {
|
||||
std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip),
|
||||
_ => ip,
|
||||
};
|
||||
match classified {
|
||||
std::net::IpAddr::V4(v4) => {
|
||||
let [a, b, c, _] = v4.octets();
|
||||
// IANA special-purpose ranges that are not globally reachable but
|
||||
// may be routed inside labs, clouds, or operator networks. They are
|
||||
// unsuitable for an owner-controlled outbound fetch even though
|
||||
// `Ipv4Addr::is_private` does not classify them as RFC1918.
|
||||
!(a >= 224
|
||||
|| (a == 192 && b == 0 && c == 0)
|
||||
|| (a == 192 && b == 0 && c == 2)
|
||||
|| (a == 192 && b == 88 && c == 99)
|
||||
|| (a == 198 && matches!(b, 18 | 19))
|
||||
|| (a == 198 && b == 51 && c == 100)
|
||||
|| (a == 203 && b == 0 && c == 113))
|
||||
}
|
||||
std::net::IpAddr::V6(v6) => {
|
||||
let s = v6.segments();
|
||||
// IANA special-purpose ranges that are non-global, locally
|
||||
// meaningful, or unsafe to treat as ordinary forge endpoints.
|
||||
// Reject the complete IETF protocol-assignment /23 rather than
|
||||
// trying to maintain its narrow globally reachable exceptions;
|
||||
// a public forge has no reason to depend on those anycast ranges.
|
||||
!(v6.is_multicast()
|
||||
|| (s[0] & 0xffc0) == 0xfec0
|
||||
|| (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 0)
|
||||
|| (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 1)
|
||||
|| (s[0] == 0x2001 && (s[1] & 0xfe00) == 0)
|
||||
|| (s[0] == 0x2001 && s[1] == 0x0db8)
|
||||
|| (s[0] == 0x3fff && (s[1] & 0xf000) == 0)
|
||||
|| s[0] == 0x5f00)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a peer `http_url` is a public http(s) endpoint safe to register.
|
||||
/// Rejects non-http(s) schemes, loopback/unspecified/private/link-local IPs,
|
||||
/// and `localhost` / `.localhost` / `.local` / `.internal` hostnames. Used at
|
||||
@@ -247,9 +137,60 @@ pub fn is_public_http_url(raw: &str) -> bool {
|
||||
// before parsing as an IP.
|
||||
let ip_candidate = host.trim_start_matches('[').trim_end_matches(']');
|
||||
if let Ok(ip) = ip_candidate.parse::<std::net::IpAddr>() {
|
||||
// IP literals share the single DNS/literal classification in
|
||||
// `is_public_ip`; hostname-only rules stay local to this function.
|
||||
return is_public_ip(ip);
|
||||
// Reject loopback/unspecified on the literal as given — catches `::1`
|
||||
// and `::` before the IPv4-folding below (`::1`.to_ipv4() would
|
||||
// otherwise map to a non-loopback `0.0.0.1`).
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
// Fold any IPv6 literal that embeds an IPv4 address (mapped, compatible,
|
||||
// 6to4, NAT64) down to that IPv4 so the v4 range checks catch
|
||||
// loopback/private addresses smuggled in via an IPv6 encoding, then
|
||||
// re-check loopback/unspecified.
|
||||
let ip = match ip {
|
||||
std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip),
|
||||
v4 => v4,
|
||||
};
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
match ip {
|
||||
std::net::IpAddr::V4(v4) => {
|
||||
let o = v4.octets();
|
||||
// RFC1918 private, link-local, CGNAT (100.64.0.0/10), or the
|
||||
// RFC1122 "this host" block 0.0.0.0/8 (never a valid destination;
|
||||
// 0.0.0.0 itself is already caught by the is_unspecified check).
|
||||
if v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| (o[0] == 100 && (o[1] & 0xc0) == 64)
|
||||
|| o[0] == 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
std::net::IpAddr::V6(v6) => {
|
||||
let s = v6.segments();
|
||||
// fc00::/7 (unique-local), fe80::/10 (link-local), or fec0::/10
|
||||
// (site-local, deprecated by RFC 3879 and so never a legitimate
|
||||
// peer address, but still routable on networks that kept it).
|
||||
if (s[0] & 0xfe00) == 0xfc00
|
||||
|| (s[0] & 0xffc0) == 0xfe80
|
||||
|| (s[0] & 0xffc0) == 0xfec0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// Any NAT64 address (64:ff9b::/32) that is not the cleanly
|
||||
// decodable well-known /96 — e.g. the RFC 8215 local-use
|
||||
// 64:ff9b:1::/48 — carries a translated target whose embedded v4
|
||||
// sits at a prefix-length-dependent offset (RFC 6052 §2.2).
|
||||
// Rather than risk a wrong decode across every prefix length we
|
||||
// reject the whole NAT64 space here. The well-known /96 was
|
||||
// already folded to its v4 above and never reaches this arm.
|
||||
if s[0] == 0x0064 && s[1] == 0xff9b {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !host.contains('.') {
|
||||
// A name with no dot at all. The `.local` / `.internal` rules above are
|
||||
// suffix checks, so a single label slips both, and the resolver decides
|
||||
@@ -675,8 +616,6 @@ mod tests {
|
||||
fn rejects_deprecated_site_local_v6() {
|
||||
// fec0::/10, deprecated by RFC 3879. The link-local mask (0xffc0 ==
|
||||
// 0xfe80) does not cover it: 0xfec0 & 0xffc0 is 0xfec0.
|
||||
assert!(!super::is_public_ip("fec0::1".parse().unwrap()));
|
||||
assert!(!super::is_public_ip("feff:ffff::1".parse().unwrap()));
|
||||
for bad in ["http://[fec0::1]/", "http://[feff:ffff::1]:7545"] {
|
||||
assert!(!is_public_http_url(bad), "{bad:?} must be rejected");
|
||||
}
|
||||
|
||||
+170
-2705
File diff suppressed because it is too large.
Load diff
@@ -11,8 +11,6 @@ use uuid::Uuid;
|
||||
use crate::db::RefCertificate;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Issue a signed ref-update certificate for a successful push. The
|
||||
/// live receive-pack handler calls this on every successful push.
|
||||
/// Build the v1 signing payload — the EXACT bytes the node signs and
|
||||
/// gl's `verify_signature` reconstructs. This is the single source
|
||||
/// of truth shared by `issue_ref_certificate` and the frozen
|
||||
@@ -83,36 +81,8 @@ pub(crate) fn v2_signing_payload(
|
||||
|
||||
/// Issue a signed ref-update certificate for a successful push.
|
||||
///
|
||||
/// `cert_id` is the deterministic id derived from `(request_id,
|
||||
/// ref_name)` (see [`crate::db::ref_cert_id_for`]). It is required so
|
||||
/// the recovery drain and the live handler produce the same primary
|
||||
/// key: a live push followed by a recovery pass collapses to a
|
||||
/// single cert row, and a re-push to the same `(repo, ref)` updates
|
||||
/// the existing row's `old_sha` / `new_sha` / `pusher_did` /
|
||||
/// `issued_at` / `signature` to the new transition while preserving
|
||||
/// the original `id` (the `insert_ref_certificate` upsert is
|
||||
/// keyed on `(repo_id, ref_name)` and only updates fields when the
|
||||
/// new `issued_at` is strictly greater).
|
||||
///
|
||||
/// #26 Split PR 1 P1-B: the live handler routes through this
|
||||
/// function (the upsert). After the reviewer-1 round-2 fix, the
|
||||
/// recovery drain also routes through this function (P1: refresh a
|
||||
/// stale cert), so both paths use the same deterministic `cert_id`
|
||||
/// and the same upsert. A re-pass is always safe:
|
||||
///
|
||||
/// - Live handler → live upsert: re-push updates the row, preserves
|
||||
/// the original `id`. The contract pinned by
|
||||
/// `insert_ref_certificate_upserts_on_repo_ref` is restored.
|
||||
/// - Live handler → recovery: live's `ON CONFLICT (id) DO UPDATE`
|
||||
/// preserves the original `id`; the recovery's same upsert is
|
||||
/// a no-op for an equal-`issued_at` re-run and a refresh for a
|
||||
/// strictly-newer one.
|
||||
/// - Recovery → live handler: the recovery wrote a row with the
|
||||
/// deterministic `id`; the live upsert (which preserves `id` and
|
||||
/// only updates other fields when `issued_at` is strictly newer)
|
||||
/// is a no-op for an equal-`issued_at` re-run and a refresh for
|
||||
/// a strictly-newer one.
|
||||
#[allow(dead_code)] // round-trip test in db/mod.rs pins the upsert contract; the live path and the drain use issue_ref_certificate_with_issued_at
|
||||
/// Builds a canonical JSON payload, signs it with the node's Ed25519 key,
|
||||
/// persists the certificate, and returns it.
|
||||
pub async fn issue_ref_certificate(
|
||||
state: &AppState,
|
||||
repo_id: &str,
|
||||
@@ -120,92 +90,9 @@ pub async fn issue_ref_certificate(
|
||||
old_sha: &str,
|
||||
new_sha: &str,
|
||||
pusher_did: &str,
|
||||
cert_id: &str,
|
||||
) -> Result<RefCertificate> {
|
||||
issue_ref_certificate_with_issued_at(
|
||||
state, repo_id, ref_name, old_sha, new_sha, pusher_did, cert_id, None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// #26 Split PR 1 round 4 — variant that lets the caller stamp the
|
||||
/// cert's `issued_at` with a transition-time timestamp instead of
|
||||
/// `Utc::now()`. The recovery drain passes the persisted
|
||||
/// `row.created_at` so a replay after a later live cert does not
|
||||
/// outrank the live cert in the `EXCLUDED.issued_at >
|
||||
/// ref_certificates.issued_at` upsert guard.
|
||||
///
|
||||
/// The live handler uses the default `issue_ref_certificate` (no
|
||||
/// override), which keeps `Utc::now()` — the reviewer's invariant
|
||||
/// is that `issued_at` reflects the transition time, and for a
|
||||
/// live push the transition time and the wall-clock are the same.
|
||||
///
|
||||
/// `issued_at_override` is honored verbatim; passing a value not in
|
||||
/// RFC 3339 form is a logic bug (the upsert will mis-order), so
|
||||
/// callers must use the row's persisted `created_at`.
|
||||
///
|
||||
/// # Clippy allow — too many arguments
|
||||
/// This is the explicit "stamp a transition-time `issued_at`"
|
||||
/// variant of `issue_ref_certificate`. The drain
|
||||
/// (`durable_outbox::derive_one`) is the in-crate caller; the
|
||||
/// test `replay_of_stale_row_does_not_overwrite_live_cert_b` pins
|
||||
/// the contract that a recovery replay's `issued_at` does NOT
|
||||
/// outrank a later live cert. Adding a struct-arg would be a
|
||||
/// larger refactor for two callers (live + drain) and obscure the
|
||||
/// parallel to `issue_ref_certificate` (which is `#[allow]`'d for
|
||||
/// the same reason historically).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn issue_ref_certificate_with_issued_at(
|
||||
state: &AppState,
|
||||
repo_id: &str,
|
||||
ref_name: &str,
|
||||
old_sha: &str,
|
||||
new_sha: &str,
|
||||
pusher_did: &str,
|
||||
cert_id: &str,
|
||||
issued_at_override: Option<String>,
|
||||
) -> Result<RefCertificate> {
|
||||
let cert = build_ref_certificate(
|
||||
state,
|
||||
repo_id,
|
||||
ref_name,
|
||||
old_sha,
|
||||
new_sha,
|
||||
pusher_did,
|
||||
Some(cert_id.to_string()),
|
||||
issued_at_override,
|
||||
)
|
||||
.await?;
|
||||
state.db.insert_ref_certificate(&cert).await
|
||||
}
|
||||
|
||||
/// Shared cert construction: build the JSON payload, sign it with the
|
||||
/// node key, and assemble the `RefCertificate` row. `cert_id_override`
|
||||
/// lets the recovery path plug in a deterministic id; the live path
|
||||
/// passes `None` and gets a fresh UUID. `issued_at_override` lets
|
||||
/// the recovery path stamp the cert with the original transition
|
||||
/// time so the upsert's `issued_at > issued_at` guard correctly
|
||||
/// orders transitions regardless of write order.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_ref_certificate(
|
||||
state: &AppState,
|
||||
repo_id: &str,
|
||||
ref_name: &str,
|
||||
old_sha: &str,
|
||||
new_sha: &str,
|
||||
pusher_did: &str,
|
||||
cert_id_override: Option<String>,
|
||||
issued_at_override: Option<String>,
|
||||
) -> Result<RefCertificate> {
|
||||
let node_did = state.node_did.to_string();
|
||||
// P1 (reviewer-1 round 4): when the caller passes a transition-
|
||||
// time `issued_at` (the recovery drain passes `row.created_at`),
|
||||
// use it verbatim so the upsert's per-column guard
|
||||
// `EXCLUDED.issued_at > ref_certificates.issued_at` correctly
|
||||
// orders transitions regardless of write order. The live handler
|
||||
// passes `None` and gets `Utc::now()` — for a live push the
|
||||
// transition time and the wall-clock are the same.
|
||||
let issued_at = issued_at_override.unwrap_or_else(|| Utc::now().to_rfc3339());
|
||||
let issued_at = Utc::now().to_rfc3339();
|
||||
|
||||
// Build the canonical signing payload via the shared builder so
|
||||
// the frozen-vector test below and the live signer cannot drift
|
||||
@@ -217,9 +104,8 @@ async fn build_ref_certificate(
|
||||
|
||||
let signature = state.node_keypair.sign_b64(&payload_bytes);
|
||||
|
||||
let id = cert_id_override.unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
let cert = RefCertificate {
|
||||
id,
|
||||
id: Uuid::new_v4().to_string(),
|
||||
repo_id: repo_id.to_string(),
|
||||
ref_name: ref_name.to_string(),
|
||||
old_sha: old_sha.to_string(),
|
||||
@@ -490,16 +376,14 @@ mod v1_payload_tests {
|
||||
let old = "0".repeat(40);
|
||||
let new = "a".repeat(40);
|
||||
let pusher = "did:key:z6MkPusher";
|
||||
let cert_id = "cert-issuer-observe";
|
||||
|
||||
// Drive the live issuer and read the persisted row back by
|
||||
// id, so the test covers the stored `version` column and
|
||||
// not just the in-process return value.
|
||||
let cert = crate::cert::issue_ref_certificate(
|
||||
&state, repo_id, ref_name, &old, &new, pusher, cert_id,
|
||||
)
|
||||
.await
|
||||
.expect("issue_ref_certificate must succeed");
|
||||
let cert =
|
||||
crate::cert::issue_ref_certificate(&state, repo_id, ref_name, &old, &new, pusher)
|
||||
.await
|
||||
.expect("issue_ref_certificate must succeed");
|
||||
|
||||
// The cert returned by the issuer is the source of truth
|
||||
// for both the version claim and the signed bytes.
|
||||
|
||||
@@ -152,46 +152,6 @@ pub struct Config {
|
||||
#[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)]
|
||||
pub auto_sync: bool,
|
||||
|
||||
/// Periodically fetch configured INBOUND repositories from their external
|
||||
/// HTTPS upstream. Disabled by default: enabling this also requires
|
||||
/// `GITLAWB_ENFORCE_OWNER_PUSH=true`, so an upstream-authoritative mirror
|
||||
/// cannot accept writes from arbitrary signed identities between fetches.
|
||||
#[arg(long, env = "GITLAWB_UPSTREAM_MIRROR_ENABLED", default_value_t = false)]
|
||||
pub upstream_mirror_enabled: bool,
|
||||
|
||||
/// Pause between complete upstream-mirror scans. A scan is sequential and
|
||||
/// paginated; the pause begins only after the prior scan has finished, so a
|
||||
/// slow forge cannot create overlapping worker cycles.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS",
|
||||
default_value_t = 300,
|
||||
value_parser = clap::value_parser!(u64).range(5..=86_400)
|
||||
)]
|
||||
pub upstream_mirror_interval_secs: u64,
|
||||
|
||||
/// Maximum wall-clock time for one external `git fetch`. The child runs in
|
||||
/// its own process group and is fully reaped on timeout. The preflight
|
||||
/// URL-rewrite and local-config probes run on their own short bounds and
|
||||
/// do not consume this budget.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS",
|
||||
default_value_t = 600,
|
||||
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
|
||||
)]
|
||||
pub upstream_mirror_fetch_timeout_secs: u64,
|
||||
|
||||
/// Number of durable INBOUND rows read per database page. Every page is
|
||||
/// processed before the scan sleeps; this bounds query memory without
|
||||
/// permanently starving repositories beyond the first page.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE",
|
||||
default_value_t = 100,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_000)
|
||||
)]
|
||||
pub upstream_mirror_page_size: usize,
|
||||
/// Enable the periodic reconciliation sweep that re-derives pin/seal sets
|
||||
/// and fills durability gaps. Defaults to true; set to false to disable
|
||||
/// the sweep even when a pin backend (IPFS/Pinata) is configured.
|
||||
@@ -837,45 +797,6 @@ pub struct Config {
|
||||
)]
|
||||
pub pin_repair_sweep_delay_secs: u64,
|
||||
|
||||
/// #26 Split PR 1 step 4 — receive-pack queue retention window.
|
||||
/// Terminal `complete` and `rejected_at_git` rows older than
|
||||
/// this are eligible for the periodic purge. `quarantined`
|
||||
/// rows are never purged on a timer. The v30 partial index
|
||||
/// `idx_receive_pack_requests_completed_at` keeps the scan
|
||||
/// cheap regardless of the value.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_QUEUE_RETENTION_DAYS",
|
||||
default_value_t = 7,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<i64>::new().range(1..=365)
|
||||
)]
|
||||
pub queue_retention_days: i64,
|
||||
|
||||
/// #26 Split PR 1 step 4 — receive-pack queue purge batch size.
|
||||
/// Each periodic purge pass deletes at most this many terminal
|
||||
/// rows per batch. The drain and the purge share the same
|
||||
/// `DRAIN_PER_PASS_LIMIT` budget; see the spawn function in
|
||||
/// `main.rs` for the wiring.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_QUEUE_PURGE_BATCH",
|
||||
default_value_t = 1000,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<i64>::new().range(1..=100_000)
|
||||
)]
|
||||
pub queue_purge_batch: i64,
|
||||
|
||||
/// #26 Split PR 1 step 5 — effects-executor retry bound. The
|
||||
/// drain flips a request to `quarantined` after this many
|
||||
/// `EffectsOutcome::Retry` returns, closing the
|
||||
/// infinite-retry DoS window. Default 8 matches the spec; set
|
||||
/// lower in tests.
|
||||
#[arg(
|
||||
long,
|
||||
env = "GITLAWB_EFFECTS_MAX_ATTEMPTS",
|
||||
default_value_t = 8,
|
||||
value_parser = clap::builder::RangedU64ValueParser::<i32>::new().range(1..=1000)
|
||||
)]
|
||||
pub effects_max_attempts: i32,
|
||||
/// Per-client-IP rate limit for the anonymous task read routes
|
||||
/// (`GET /api/v1/tasks`, `GET /api/v1/tasks/{id}`), in requests per hour.
|
||||
/// Both are publicly reachable (`optional_signature`). `GET /api/v1/tasks`
|
||||
@@ -960,13 +881,6 @@ impl Config {
|
||||
Self::DB_LOCK_POOL_NON_PUSH_HEADROOM
|
||||
));
|
||||
}
|
||||
if self.upstream_mirror_enabled && !self.enforce_owner_push {
|
||||
return Err(
|
||||
"GITLAWB_UPSTREAM_MIRROR_ENABLED requires GITLAWB_ENFORCE_OWNER_PUSH=true: \
|
||||
an upstream-authoritative mirror must not accept writes from arbitrary signed identities"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
let main_floor = Self::DB_POOL_APP_HEADROOM as u64;
|
||||
if (self.db_max_connections as u64) < main_floor {
|
||||
return Err(format!(
|
||||
@@ -1166,51 +1080,6 @@ pub(crate) fn validate_web_url(raw: &str) -> Result<(), String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn upstream_mirror_worker_is_default_off_and_bounded() {
|
||||
let mut config = Config::parse_from(["gitlawb-node"]);
|
||||
assert!(!config.upstream_mirror_enabled);
|
||||
assert_eq!(config.upstream_mirror_interval_secs, 300);
|
||||
assert_eq!(config.upstream_mirror_fetch_timeout_secs, 600);
|
||||
assert_eq!(config.upstream_mirror_page_size, 100);
|
||||
config.validate().unwrap();
|
||||
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--upstream-mirror-interval-secs", "4",])
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Config::try_parse_from(["gitlawb-node", "--upstream-mirror-page-size", "0",]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_mirror_worker_requires_owner_push_enforcement() {
|
||||
// enforce_owner_push defaults true on main, so the only unsafe shape
|
||||
// is the explicit opt-out: mirror enabled plus enforcement refused.
|
||||
let mut unsafe_config = Config::parse_from([
|
||||
"gitlawb-node",
|
||||
"--upstream-mirror-enabled",
|
||||
"--enforce-owner-push",
|
||||
"false",
|
||||
]);
|
||||
let error = unsafe_config.validate().unwrap_err();
|
||||
assert!(error.contains("GITLAWB_ENFORCE_OWNER_PUSH=true"));
|
||||
|
||||
Config::parse_from([
|
||||
"gitlawb-node",
|
||||
"--upstream-mirror-enabled",
|
||||
"--enforce-owner-push",
|
||||
"true",
|
||||
])
|
||||
.validate()
|
||||
.unwrap();
|
||||
// The default posture alone also validates: enforcement is on.
|
||||
Config::parse_from(["gitlawb-node", "--upstream-mirror-enabled"])
|
||||
.validate()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lock_pool_size_defaults_to_40_and_rejects_zero() {
|
||||
assert_eq!(
|
||||
@@ -1439,30 +1308,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_lifecycle_knobs_default_conservatively() {
|
||||
let c = Config::parse_from(["gitlawb-node"]);
|
||||
// 7-day retention matches the v30 partial index comment
|
||||
// and the spec at .gravirei/plans/state-model-durable-post-receive.md.
|
||||
assert_eq!(c.queue_retention_days, 7);
|
||||
// 1000 rows per pass matches DRAIN_PER_PASS_LIMIT in durable_outbox.
|
||||
assert_eq!(c.queue_purge_batch, 1000);
|
||||
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--queue-retention-days", "30"])
|
||||
.queue_retention_days,
|
||||
30
|
||||
);
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--queue-retention-days", "0"]).is_err());
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--queue-retention-days", "366"]).is_err());
|
||||
|
||||
assert_eq!(
|
||||
Config::parse_from(["gitlawb-node", "--queue-purge-batch", "500"]).queue_purge_batch,
|
||||
500
|
||||
);
|
||||
assert!(Config::try_parse_from(["gitlawb-node", "--queue-purge-batch", "0"]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() {
|
||||
assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4);
|
||||
|
||||
+134
-4093
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -220,7 +220,6 @@ pub fn response_served_pack(output: &[u8]) -> bool {
|
||||
///
|
||||
/// Accepts a push. The caller MUST verify HTTP Signature auth before
|
||||
/// calling this function.
|
||||
#[allow(dead_code)] // used by tests; production uses receive_pack_raw
|
||||
pub async fn receive_pack(
|
||||
git_bin: &str,
|
||||
repo_path: &Path,
|
||||
@@ -245,219 +244,6 @@ pub async fn receive_pack(
|
||||
.body(Body::from(output))?)
|
||||
}
|
||||
|
||||
/// Run `git-receive-pack` and return the raw stdout bytes together with
|
||||
/// the process exit status. Unlike [`receive_pack`], this does NOT bail
|
||||
/// on a non-zero exit: the caller needs the stdout (which contains the
|
||||
/// report-status with per-ref ok/ng results) even when the process
|
||||
/// exits non-zero. A timeout still returns `Err`.
|
||||
///
|
||||
/// `reflog_action` (typically `gitlawb-request:<request_id>`) is
|
||||
/// exported as `GIT_REFLOG_ACTION` so the ref transaction writes
|
||||
/// request-bound landing evidence into the reflog. Recovery requires
|
||||
/// that evidence before auto-promoting a stranded row; without it a
|
||||
/// later request recreating the same old/new tuple is
|
||||
/// indistinguishable from the stranded one.
|
||||
#[allow(dead_code)]
|
||||
pub async fn receive_pack_raw(
|
||||
git_bin: &str,
|
||||
repo_path: &Path,
|
||||
request_body: Bytes,
|
||||
timeout: Duration,
|
||||
admission: Option<AdmissionGuard>,
|
||||
) -> Result<(Vec<u8>, bool)> {
|
||||
receive_pack_raw_with_reflog(git_bin, repo_path, request_body, timeout, admission, None).await
|
||||
}
|
||||
|
||||
/// [`receive_pack_raw`] with explicit reflog binding.
|
||||
pub async fn receive_pack_raw_with_reflog(
|
||||
git_bin: &str,
|
||||
repo_path: &Path,
|
||||
request_body: Bytes,
|
||||
timeout: Duration,
|
||||
admission: Option<AdmissionGuard>,
|
||||
reflog_action: Option<&str>,
|
||||
) -> Result<(Vec<u8>, bool)> {
|
||||
let mut command = Command::new(git_bin);
|
||||
command
|
||||
.arg("receive-pack")
|
||||
.arg("--stateless-rpc")
|
||||
.arg(repo_path);
|
||||
if let Some(action) = reflog_action {
|
||||
command.env("GIT_REFLOG_ACTION", action);
|
||||
}
|
||||
let (out, err, status, _admission) = drive_git_child_raw(
|
||||
command,
|
||||
request_body,
|
||||
timeout,
|
||||
"git-receive-pack",
|
||||
admission,
|
||||
)
|
||||
.await?;
|
||||
// On timeout, drive_git_child_raw returns Err — we never reach here.
|
||||
// On success/non-zero exit, we have the stdout and exit status.
|
||||
if !status.success() {
|
||||
let stderr = String::from_utf8_lossy(&err);
|
||||
tracing::warn!(stderr = %stderr, "git-receive-pack exited non-zero");
|
||||
}
|
||||
Ok((out, status.success()))
|
||||
}
|
||||
|
||||
/// Parse the git-receive-pack report-status output to determine per-ref
|
||||
/// success/failure. Returns `(unpack_ok, per_ref_results)` where
|
||||
/// `per_ref_results` is a list of `(ref_name, is_ok)`.
|
||||
///
|
||||
/// The report-status format (after the sideband framing) is:
|
||||
/// ```text
|
||||
/// unpack ok\n (or: unpack fail\n)
|
||||
/// ok <refname>\n (per successful ref)
|
||||
/// ng <refname> <reason>\n (per rejected ref)
|
||||
/// \n (empty line terminates)
|
||||
/// ```
|
||||
///
|
||||
/// Returns `None` if the output cannot be parsed (e.g. the client did
|
||||
/// not request report-status, or the output is truncated). In that
|
||||
/// case the caller should treat all refs as uncertain.
|
||||
pub fn parse_report_status(output: &[u8]) -> Option<(bool, Vec<(String, bool)>)> {
|
||||
let text = std::str::from_utf8(output).ok()?;
|
||||
// Strip sideband framing: each line starts with a pkt-line length
|
||||
// prefix and a channel byte (1 = stdout, 2 = stderr). The actual
|
||||
// data starts after the first `0000` flush packet or after we
|
||||
// strip sideband bytes.
|
||||
//
|
||||
// Framing completeness is part of authority, at whichever level the
|
||||
// framing lives. A double-framed capture carries the report stream
|
||||
// (itself pkt-line framed, flush-terminated) inside the sideband
|
||||
// envelope — real git output has been observed with NO outer flush,
|
||||
// so the outer level cannot require one; the inner frame's flush is
|
||||
// authoritative there. A single-framed (or unframed) payload has no
|
||||
// inner frame, so its only completion marker is the outer flush:
|
||||
// output ending after a complete `unpack ok` / `ok <ref>` packet but
|
||||
// before the outer `0000` is a truncated report and yields `None`
|
||||
// (uncertain), not a successful report.
|
||||
let (stripped, outer_flush) = strip_sideband(text, false)?;
|
||||
// The report-status vocabulary (`unpack ...`, `ok ...`, `ng ...`)
|
||||
// never opens with four hex digits, so a hex quartet here is inner
|
||||
// pkt-line framing, not report text.
|
||||
let stripped = if stripped.len() >= 4
|
||||
&& stripped.as_bytes()[..4]
|
||||
.iter()
|
||||
.all(|b| b.is_ascii_hexdigit())
|
||||
{
|
||||
strip_sideband(&stripped, true)?.0
|
||||
} else if !outer_flush {
|
||||
return None;
|
||||
} else {
|
||||
stripped
|
||||
};
|
||||
// The report is framed TWICE when the client negotiated side-band-64k,
|
||||
// which `git push` over smart HTTP does — so this is the common case, not
|
||||
// an exotic one. The outer frame is the side-band envelope; band 1 carries
|
||||
// the report-status stream, which is ITSELF pkt-line encoded. Real bytes
|
||||
// from git 2.50.1 rejecting one ref of a two-ref push:
|
||||
//
|
||||
// 0057\x01000eunpack ok\n0028ng refs/heads/main non-fast-forward\n...
|
||||
//
|
||||
// After one pass the first line reads `000eunpack ok`, which fails the
|
||||
// `unpack ok` check below and makes this return None — i.e. "no report",
|
||||
// which the caller treats as inconclusive and keeps every declared ref.
|
||||
// The per-ref gate would then be inert for exactly the pushes it exists to
|
||||
// filter. The second pass above already removed the inner pkt-line
|
||||
// framing where present (and required its flush); it is a no-op
|
||||
// on the single-framed shape, because plain report text does not begin
|
||||
// with four hex digits.
|
||||
let lines: Vec<&str> = stripped.lines().collect();
|
||||
if lines.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// First non-empty line is "unpack ok" or "unpack fail".
|
||||
let unpack_line = lines.iter().find(|l| !l.is_empty())?;
|
||||
let unpack_ok = if unpack_line.starts_with("unpack ok") {
|
||||
true
|
||||
} else if unpack_line.starts_with("unpack fail") {
|
||||
false
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let mut results = Vec::new();
|
||||
for line in &lines[1..] {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix("ok ") {
|
||||
results.push((rest.to_string(), true));
|
||||
} else if let Some(rest) = line.strip_prefix("ng ") {
|
||||
// "ng <refname> <reason>" — skip the reason
|
||||
let ref_name = rest.split_whitespace().next()?.to_string();
|
||||
results.push((ref_name, false));
|
||||
}
|
||||
}
|
||||
|
||||
Some((unpack_ok, results))
|
||||
}
|
||||
|
||||
/// Strip git sideband framing from a pkt-line encoded output.
|
||||
/// Sideband-encoded lines start with a 4-hex-digit length, then a
|
||||
/// channel byte (0x01=stdout, 0x02=stderr), then payload. Returns the
|
||||
/// decoded payload plus whether the stream saw its terminating flush
|
||||
/// packet, or `None` if the framing is malformed. With `require_flush`,
|
||||
/// a stream that never sees its flush is also `None` — the caller uses
|
||||
/// this for the level whose completion marker is authoritative (see
|
||||
/// [`parse_report_status`]).
|
||||
fn strip_sideband(text: &str, require_flush: bool) -> Option<(String, bool)> {
|
||||
let mut output = String::new();
|
||||
let mut pos = 0;
|
||||
let bytes = text.as_bytes();
|
||||
let mut saw_flush = false;
|
||||
|
||||
loop {
|
||||
if pos == bytes.len() {
|
||||
break;
|
||||
}
|
||||
if pos + 4 > bytes.len() {
|
||||
// Truncated framing mid-header: fail closed.
|
||||
return None;
|
||||
}
|
||||
let len_str = std::str::from_utf8(&bytes[pos..pos + 4]).ok()?;
|
||||
let len = usize::from_str_radix(len_str, 16).ok()?;
|
||||
if len == 0 {
|
||||
// Flush packet — end of sideband stream
|
||||
saw_flush = true;
|
||||
break;
|
||||
}
|
||||
if len < 4 || pos + len > bytes.len() {
|
||||
// Truncated pkt-line extends past EOF: incomplete framing.
|
||||
return None;
|
||||
}
|
||||
let pkt_data = std::str::from_utf8(&bytes[pos + 4..pos + len]).ok()?;
|
||||
pos += len;
|
||||
|
||||
// Sideband: first byte is channel (1=stdout, 2=stderr)
|
||||
if let Some(payload) = pkt_data.strip_prefix('\x01') {
|
||||
output.push_str(payload);
|
||||
} else if pkt_data.starts_with('\x02') {
|
||||
// stderr — skip (git error messages)
|
||||
} else {
|
||||
// Not sideband encoded — pass through
|
||||
output.push_str(pkt_data);
|
||||
}
|
||||
}
|
||||
|
||||
if require_flush && !saw_flush {
|
||||
// Truncated at a packet boundary: every packet parsed, but the
|
||||
// stream never completed. Indistinguishable from a report git was
|
||||
// killed mid-write, so no authority.
|
||||
return None;
|
||||
}
|
||||
if output.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some((output, saw_flush))
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends SIGTERM to a child's whole process group on drop, unless disarmed first.
|
||||
///
|
||||
/// A served `git upload-pack`/`receive-pack` forks helpers such as `pack-objects`.
|
||||
@@ -792,121 +578,6 @@ async fn drive_git_child(
|
||||
Ok((out, admission))
|
||||
}
|
||||
|
||||
/// Like [`drive_git_child`], but returns stdout and stderr even on a
|
||||
/// non-zero exit status. Used by [`receive_pack_raw`] so the caller
|
||||
/// can parse the report-status output from a failed `git-receive-pack`.
|
||||
async fn drive_git_child_raw(
|
||||
mut command: Command,
|
||||
input: Bytes,
|
||||
timeout: Duration,
|
||||
what: &str,
|
||||
admission: Option<AdmissionGuard>,
|
||||
) -> Result<(
|
||||
Vec<u8>,
|
||||
Vec<u8>,
|
||||
std::process::ExitStatus,
|
||||
Option<AdmissionGuard>,
|
||||
)> {
|
||||
command
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
#[cfg(unix)]
|
||||
command.process_group(0);
|
||||
|
||||
let mut child = command.spawn()?;
|
||||
|
||||
let mut stdin = child.stdin.take();
|
||||
let mut stdout = child.stdout.take().context("git stdout was not piped")?;
|
||||
let mut stderr = child.stderr.take().context("git stderr was not piped")?;
|
||||
|
||||
#[cfg(unix)]
|
||||
let pgid = child.id().map(|id| id as i32);
|
||||
#[cfg(unix)]
|
||||
let mut group_guard = KillGroupOnDrop {
|
||||
child: Some(child),
|
||||
pgid,
|
||||
admission,
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut err = Vec::new();
|
||||
|
||||
let interact = async {
|
||||
let write = async {
|
||||
match stdin.take() {
|
||||
Some(mut s) => s.write_all(&input).await,
|
||||
None => Ok(()),
|
||||
}
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let child_ref = group_guard.child_mut();
|
||||
#[cfg(not(unix))]
|
||||
let child_ref = &mut child;
|
||||
let (write_result, r_out, r_err, status) = tokio::join!(
|
||||
write,
|
||||
stdout.read_to_end(&mut out),
|
||||
stderr.read_to_end(&mut err),
|
||||
child_ref.wait(),
|
||||
);
|
||||
r_out?;
|
||||
r_err?;
|
||||
Ok::<_, anyhow::Error>((write_result, status?))
|
||||
};
|
||||
|
||||
let timed = tokio::time::timeout(timeout, interact).await;
|
||||
let (write_result, status, admission) = match timed {
|
||||
Ok(result) => {
|
||||
#[cfg(unix)]
|
||||
let admission = group_guard.disarm();
|
||||
let (write_result, status) = result?;
|
||||
(write_result, status, admission)
|
||||
}
|
||||
Err(_elapsed) => {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
reap_group_on_timeout(group_guard.child_mut()).await;
|
||||
drop(group_guard.disarm());
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = child.start_kill();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await;
|
||||
drop(admission);
|
||||
}
|
||||
return Err(GitServiceTimeout.into());
|
||||
}
|
||||
};
|
||||
|
||||
// Prefer the child's collected answer over a stdin-write error: an early
|
||||
// rejection (unpack failure, command-level deny) makes git print a complete
|
||||
// report-status and exit WITHOUT draining stdin, so a body larger than the
|
||||
// pipe buffer EPIPEs the write while the answer is already on stdout. The
|
||||
// child is reaped (the join above waited on it), so `out`/`err`/`status`
|
||||
// are complete and authoritative — returning Err here would discard them:
|
||||
// `receive_pack_raw_with_reflog` would propagate the write error and the
|
||||
// handler would mark every prepared row `uncertain` instead of driving
|
||||
// `cancelled` per the report. That durable fate is the effect: on a
|
||||
// non-zero exit the client still gets `Err(AppError::Git)` (the handler
|
||||
// maps the failed run itself, not this write), while on exit 0 an error
|
||||
// here would answer a push git actually landed with a server error. The
|
||||
// non-raw `drive_git_child` orders the same way (exit status and git's own
|
||||
// failure first, stdin-write error last — pinned by
|
||||
// `run_git_service_surfaces_git_stderr_over_a_stdin_epipe`), but it must
|
||||
// fail on non-zero because its callers do not parse output from a failed
|
||||
// child; the raw driver exists precisely to parse that output.
|
||||
if let Err(e) = &write_result {
|
||||
tracing::warn!(
|
||||
err = %e,
|
||||
what = %what,
|
||||
"git stdin write failed after child exit; surfacing collected output"
|
||||
);
|
||||
}
|
||||
|
||||
Ok((out, err, status, admission))
|
||||
}
|
||||
|
||||
/// Map a validated service name to its git subcommand. Only the two fixed
|
||||
/// subcommands can come out, so an unrecognized service can never reach git's argv.
|
||||
fn service_to_command(service: &str) -> Result<&'static str> {
|
||||
@@ -1197,86 +868,6 @@ mod tests {
|
||||
use std::process::Command;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Byte-for-byte capture from `git receive-pack` 2.50.1 rejecting one ref of
|
||||
/// a two-ref push, with side-band-64k negotiated — what `git push` over
|
||||
/// smart HTTP actually sends. The report is framed twice: the outer
|
||||
/// side-band envelope, then the report-status stream's own pkt-lines.
|
||||
///
|
||||
/// This fixture is the point of the test. A hand-written single-framed
|
||||
/// string parses fine with only one strip pass, so a fixture invented to
|
||||
/// match the parser hides the exact case the parser is for.
|
||||
const REAL_SIDEBAND_REPORT: &[u8] =
|
||||
b"0057\x01000eunpack ok\n0028ng refs/heads/main non-fast-forward\n0018ok refs/heads/third\n0000";
|
||||
|
||||
#[test]
|
||||
fn a_real_double_framed_sideband_report_is_read_not_treated_as_absent() {
|
||||
let (unpack_ok, refs) =
|
||||
parse_report_status(REAL_SIDEBAND_REPORT).expect("a real git report must parse");
|
||||
assert!(unpack_ok, "unpack line must be read through both frames");
|
||||
assert_eq!(
|
||||
refs,
|
||||
vec![
|
||||
("refs/heads/main".to_string(), false),
|
||||
("refs/heads/third".to_string(), true),
|
||||
],
|
||||
"the rejected ref must be distinguished from the accepted one"
|
||||
);
|
||||
}
|
||||
|
||||
/// The single-framed shape must keep working: the second pass has to be a
|
||||
/// no-op there, not a corruption. Plain report text does not begin with
|
||||
/// four hex digits, which is what makes that safe.
|
||||
#[test]
|
||||
fn a_single_framed_report_still_parses_after_the_second_pass() {
|
||||
// Framed programmatically rather than by a hand-counted hex prefix: a
|
||||
// wrong length makes the parser return None, which would look exactly
|
||||
// like the regression this pair of tests exists to catch.
|
||||
let payload = "\x01unpack ok\nok refs/heads/x\n";
|
||||
let single = format!("{:04x}{payload}0000", payload.len() + 4);
|
||||
let (unpack_ok, refs) =
|
||||
parse_report_status(single.as_bytes()).expect("single-framed must parse");
|
||||
assert!(unpack_ok);
|
||||
assert_eq!(refs, vec![("refs/heads/x".to_string(), true)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_mid_packet_report_is_indeterminate_not_partial() {
|
||||
// Truncate a valid single-framed report mid-packet (overrun):
|
||||
// must return None so the handler persists the whole request as
|
||||
// indeterminate instead of committing a partial ok set.
|
||||
let payload = "\x01unpack ok\nok refs/heads/x\n";
|
||||
let single = format!("{:04x}{payload}0000", payload.len() + 4);
|
||||
let truncated = &single.as_bytes()[..single.len() - 6];
|
||||
assert!(
|
||||
parse_report_status(truncated).is_none(),
|
||||
"mid-packet truncation must be indeterminate"
|
||||
);
|
||||
}
|
||||
|
||||
/// Truncation exactly at a packet boundary (every packet complete, but
|
||||
/// the outer flush never arrived — e.g. git killed after the last `ok`
|
||||
/// line) must also be indeterminate: the payload is complete but the
|
||||
/// frame is not, and framing completeness is part of authority. The
|
||||
/// mid-packet sibling above cannot catch this; only the flush gate can.
|
||||
/// Removing the `require_flush` enforcement returns `Some` and turns
|
||||
/// this red.
|
||||
#[test]
|
||||
fn truncated_at_packet_boundary_report_is_indeterminate_not_partial() {
|
||||
let payload = "\x01unpack ok\nok refs/heads/x\n";
|
||||
let single = format!("{:04x}{payload}0000", payload.len() + 4);
|
||||
let no_flush = &single.as_bytes()[..single.len() - 4];
|
||||
assert!(
|
||||
parse_report_status(no_flush).is_none(),
|
||||
"a report missing its outer flush must be indeterminate, \
|
||||
even with every packet intact"
|
||||
);
|
||||
// Sanity: the untruncated fixture still parses (flush present).
|
||||
assert!(
|
||||
parse_report_status(single.as_bytes()).is_some(),
|
||||
"the same bytes with the flush must parse"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_to_command_maps_allowlist_and_rejects_passthrough() {
|
||||
assert_eq!(
|
||||
@@ -3050,62 +2641,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The raw driver's counterpart: a rejection-shaped early exit where git
|
||||
// prints a COMPLETE report-status and exits without draining stdin, so a
|
||||
// body larger than the pipe buffer (~64 KiB) EPIPEs the write while the
|
||||
// answer is already on stdout. `receive_pack_raw` must surface that report
|
||||
// (Ok + stdout + exit status), not discard it as a stdin-write error — an
|
||||
// Err here loses parseable output: `receive_pack_raw_with_reflog`
|
||||
// propagates it and the handler marks every prepared row `uncertain`
|
||||
// instead of driving `cancelled` from the report (the durable fate is the
|
||||
// effect; the client-visible status on a failed run is `AppError::Git`
|
||||
// either way). Goes RED if the stdin-write check returns Err after the
|
||||
// join. Both exit codes are pinned: receive-pack rejections and successes
|
||||
// both carry reports the handler must parse.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn receive_pack_raw_surfaces_report_status_over_a_stdin_epipe() {
|
||||
for exit_code in [0, 1] {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
std::fs::write(tmp.path().join("report.bin"), REAL_SIDEBAND_REPORT).unwrap();
|
||||
// Never reads stdin: the pending body blocks the pipe, then EPIPEs
|
||||
// once the fake exits. `cat` delivers the report first.
|
||||
let git_bin = write_fake_git(
|
||||
tmp.path(),
|
||||
&format!("#!/bin/sh\ncat \"$(dirname \"$0\")/report.bin\"\nexit {exit_code}\n"),
|
||||
);
|
||||
let body = Bytes::from(vec![0u8; 256 * 1024]);
|
||||
let (out, ok) = receive_pack_raw(
|
||||
git_bin.to_str().unwrap(),
|
||||
tmp.path(),
|
||||
body,
|
||||
Duration::from_secs(60),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect(
|
||||
"a report-status printed before exit must not be discarded as a \
|
||||
stdin-write error",
|
||||
);
|
||||
assert_eq!(
|
||||
ok,
|
||||
exit_code == 0,
|
||||
"the exit status rides the Ok return for both report shapes"
|
||||
);
|
||||
let (unpack_ok, results) = parse_report_status(&out)
|
||||
.expect("git's report survives the EPIPE and stays parseable");
|
||||
assert!(unpack_ok, "unpack ok parses");
|
||||
assert_eq!(
|
||||
results,
|
||||
vec![
|
||||
("refs/heads/main".to_string(), false),
|
||||
("refs/heads/third".to_string(), true),
|
||||
],
|
||||
"the per-ref report reaches the handler instead of an HTTP error"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── #192 F1: response_served_pack detector (real git output) ────────────
|
||||
|
||||
// A completing request (want+flush+done) yields NAK + a real packfile;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -43,53 +43,6 @@ const WALK_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
#[cfg(unix)]
|
||||
const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Build a Git child command, optionally with an isolated environment for an
|
||||
/// owner-selected HTTPS destination. The isolated form intentionally drops
|
||||
/// ambient proxy, credential-helper, askpass, and Git config environment that
|
||||
/// could otherwise forward node-operator secrets to the selected upstream.
|
||||
fn git_child_command(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
isolated_https: bool,
|
||||
isolated_config: &[(&str, &str)],
|
||||
) -> std::process::Command {
|
||||
let mut command = std::process::Command::new(git_bin);
|
||||
command
|
||||
.args(args)
|
||||
.current_dir(repo_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
if isolated_https {
|
||||
let path = std::env::var_os("PATH");
|
||||
command.env_clear();
|
||||
if let Some(path) = path {
|
||||
command.env("PATH", path);
|
||||
}
|
||||
command
|
||||
.env("GIT_CONFIG_NOSYSTEM", "1")
|
||||
.env(
|
||||
"GIT_CONFIG_GLOBAL",
|
||||
if cfg!(windows) { "NUL" } else { "/dev/null" },
|
||||
)
|
||||
.env("GIT_TERMINAL_PROMPT", "0")
|
||||
.env("GIT_ASKPASS", "false")
|
||||
.env("SSH_ASKPASS", "false")
|
||||
.env("LANG", "C")
|
||||
.env("LC_ALL", "C");
|
||||
if !isolated_config.is_empty() {
|
||||
command.env("GIT_CONFIG_COUNT", isolated_config.len().to_string());
|
||||
for (index, (key, value)) in isolated_config.iter().enumerate() {
|
||||
command
|
||||
.env(format!("GIT_CONFIG_KEY_{index}"), key)
|
||||
.env(format!("GIT_CONFIG_VALUE_{index}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
command
|
||||
}
|
||||
|
||||
/// Run one git child under a shared `deadline` with process-group teardown,
|
||||
/// BLOCKING, and return its stdout. The child runs in its own process group; a
|
||||
/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs,
|
||||
@@ -128,22 +81,24 @@ fn child_terminated_without_reaping(pid: i32) -> bool {
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn run_bounded_git_raw_impl(
|
||||
pub(crate) fn run_bounded_git_raw(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
isolated_https: bool,
|
||||
isolated_config: &[(&str, &str)],
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::sync::mpsc::RecvTimeoutError;
|
||||
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let mut command = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config);
|
||||
let mut child = command
|
||||
let mut child = std::process::Command::new(git_bin)
|
||||
.args(args)
|
||||
.current_dir(repo_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.process_group(0)
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn git {label}"))?;
|
||||
@@ -253,51 +208,6 @@ fn run_bounded_git_raw_impl(
|
||||
Ok((status, out, err))
|
||||
}
|
||||
|
||||
pub(crate) fn run_bounded_git_raw(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, false, &[])
|
||||
}
|
||||
|
||||
/// Run a bounded Git child without inheriting ambient Git/proxy/credential
|
||||
/// configuration. Reserved for owner-selected external HTTPS destinations;
|
||||
/// ordinary local Git operations retain their existing environment.
|
||||
pub(crate) fn run_bounded_git_raw_isolated_https(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, true, &[])
|
||||
}
|
||||
|
||||
/// Isolated HTTPS runner with command-scope Git configuration supplied as
|
||||
/// separate key/value environment entries. Unlike `git -c key=value`, this
|
||||
/// preserves valid URL subsection keys whose path contains `=`.
|
||||
pub(crate) fn run_bounded_git_raw_isolated_https_with_config(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
config: &[(&str, &str)],
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
run_bounded_git_raw_impl(
|
||||
git_bin,
|
||||
args,
|
||||
repo_path,
|
||||
stdin_bytes,
|
||||
deadline,
|
||||
true,
|
||||
config,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin
|
||||
/// wrapper the walk callers use. Probes that must distinguish exit classes —
|
||||
/// `git cat-file` absence vs an object-store access failure — call
|
||||
@@ -331,20 +241,23 @@ pub(crate) fn run_bounded_git(
|
||||
/// the Unix version's signature and result semantics so every caller compiles on all
|
||||
/// targets (#174).
|
||||
#[cfg(not(unix))]
|
||||
fn run_bounded_git_raw_impl(
|
||||
pub(crate) fn run_bounded_git_raw(
|
||||
git_bin: &str,
|
||||
args: &[&str],
|
||||
repo_path: &Path,
|
||||
stdin_bytes: &[u8],
|
||||
deadline: Instant,
|
||||
isolated_https: bool,
|
||||
isolated_config: &[(&str, &str)],
|
||||
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::mpsc::RecvTimeoutError;
|
||||
|
||||
let label = args.first().copied().unwrap_or("git");
|
||||
let mut child = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config)
|
||||
let mut child = std::process::Command::new(git_bin)
|
||||
.args(args)
|
||||
.current_dir(repo_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn git {label}"))?;
|
||||
|
||||
@@ -421,111 +334,6 @@ pub(crate) fn run_bounded_git(
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Fail closed unless every ref ultimately resolves to a commit (a ref pointing
|
||||
/// directly at a blob or tree, or an annotated tag — even a nested one — of such
|
||||
/// an object is refused). `git rev-list --all` silently *skips* such refs, but
|
||||
/// `git upload-pack` (serve) and the whole-repo pin fallback
|
||||
/// (`git cat-file --batch-all-objects`) still expose their target object, so a
|
||||
/// tolerant walk would under-withhold. Refuse rather than leak.
|
||||
///
|
||||
/// Each ref is peeled fully with `<ref>^{}` through `git cat-file --batch-check`.
|
||||
/// Full peeling is why this is not `for-each-ref %(*objecttype)`, which
|
||||
/// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a-
|
||||
/// commit as a non-commit.
|
||||
///
|
||||
/// Internal node-managed refs are exempt from this check since they point
|
||||
/// to blobs by design, not repository content: request markers
|
||||
/// (`refs/gitlawb/requests/*`, per-push marker blobs) and issue records
|
||||
/// (`refs/gitlawb/issues/*`, JSON blobs). The exemption lists exactly
|
||||
/// those two namespaces: any other `refs/gitlawb/*` ref or non-commit
|
||||
/// content ref still fails closed. Pushes into `refs/gitlawb/*` are
|
||||
/// refused at the push edge, so no new bypass ref can appear; the
|
||||
/// visibility logic only cares about content refs, not these two
|
||||
/// internal metadata namespaces whose binding and advertisement hiding
|
||||
/// are verified on the push path.
|
||||
fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> {
|
||||
let refs_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["for-each-ref", "--format=%(refname)"],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
let refs_stdout = String::from_utf8_lossy(&refs_out);
|
||||
let refnames: Vec<&str> = refs_stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
if refnames.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Filter out only the two node-managed blob namespaces before checking
|
||||
// object types. Scoped to `refs/gitlawb/requests/` and
|
||||
// `refs/gitlawb/issues/` — not all of `refs/gitlawb/*` — so any
|
||||
// other internal or content ref pointing at a non-commit still
|
||||
// fails closed.
|
||||
let content_refs: Vec<&str> = refnames
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
!r.starts_with("refs/gitlawb/requests/") && !r.starts_with("refs/gitlawb/issues/")
|
||||
})
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
if content_refs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Peel every content ref in one `git cat-file --batch-check` pass: one `<refname>^{}`
|
||||
// query per line, one output line per input line, in order. cat-file echoes the
|
||||
// full query on a `<query> missing` line, so output scales with refname length;
|
||||
// run_bounded_git drains stdout concurrently with the stdin write, so the pipe
|
||||
// cannot deadlock, and the whole peel is bounded by the shared walk deadline.
|
||||
let queries = content_refs
|
||||
.iter()
|
||||
.map(|r| format!("{r}^{{}}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let peel_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["cat-file", "--batch-check=%(objecttype)"],
|
||||
repo_path,
|
||||
queries.as_bytes(),
|
||||
deadline,
|
||||
)?;
|
||||
|
||||
let peel_stdout = String::from_utf8_lossy(&peel_out);
|
||||
let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect();
|
||||
// A short read means at least one ref went unclassified — fail closed.
|
||||
if types.len() != content_refs.len() {
|
||||
anyhow::bail!(
|
||||
"git cat-file returned {} lines for {} refs; \
|
||||
refusing to produce a partial (under-withheld) set",
|
||||
types.len(),
|
||||
content_refs.len()
|
||||
);
|
||||
}
|
||||
for (refname, kind) in content_refs.iter().zip(types.iter()) {
|
||||
// git emits `<query> missing` (not the objecttype) when the peel target
|
||||
// is absent; the status word is the last token.
|
||||
if kind.split_ascii_whitespace().last() == Some("missing") {
|
||||
anyhow::bail!(
|
||||
"ref {refname} does not resolve to an object; \
|
||||
refusing to produce a partial (under-withheld) set"
|
||||
);
|
||||
}
|
||||
if *kind != "commit" {
|
||||
anyhow::bail!(
|
||||
"ref {refname} resolves to a {kind}, not a commit; \
|
||||
refusing to produce a partial (under-withheld) set"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// List every (blob_oid, "/repo/relative/path") pair reachable from any commit in
|
||||
/// `repo_path` — every ref *and* every historical commit those refs reach, not just
|
||||
/// the ref tips. `git upload-pack` (serve) and the whole-repo pin fallback
|
||||
@@ -1616,17 +1424,84 @@ fn pair_decision(
|
||||
}
|
||||
}
|
||||
|
||||
/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's
|
||||
/// own root tree (it lists entries *under* a tree), so it is added explicitly here.
|
||||
/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the
|
||||
/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the
|
||||
/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a
|
||||
/// long history has tens of thousands of reachable commits, and passing them all as
|
||||
/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats
|
||||
/// as a walk error and fail-closed 404s an authorized reader of a reachable/root
|
||||
/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin
|
||||
/// write, so a large history cannot deadlock the pipes. A commit whose root tree git
|
||||
/// cannot resolve fails the pass (bail), failing closed.
|
||||
/// Reachable blob AND withheld-subtree tree OIDs the anonymous (or `caller`)
|
||||
/// replication audience must not receive. The single source for
|
||||
/// `replication_withheld_set`: a blob-only set handed to [`replicable_objects`]
|
||||
/// reintroduces #172 (withheld trees pin to public IPFS).
|
||||
///
|
||||
/// Strict where [`tree_paths`] is lenient. This feeds a withheld filter, so
|
||||
/// absence means replicate; a missed reachable tree would leak. The walk
|
||||
/// therefore matches [`blob_paths`]: [`assert_all_refs_are_commits`] first, then
|
||||
/// a failing `rev-list --all [HEAD]` is an error, then [`object_paths`] (already
|
||||
/// fail-closed on ls-tree / non-UTF-8) plus [`root_tree_pairs`]. [`tree_paths`]
|
||||
/// stays the lenient allow-list walk for GET /ipfs/{cid} and the full-scan
|
||||
/// allowed-tree set, where absence withholds.
|
||||
fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> {
|
||||
let refs_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["for-each-ref", "--format=%(refname)"],
|
||||
repo_path,
|
||||
b"",
|
||||
deadline,
|
||||
)?;
|
||||
let refs_stdout = String::from_utf8_lossy(&refs_out);
|
||||
let refnames: Vec<&str> = refs_stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
if refnames.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Peel every ref in one `git cat-file --batch-check` pass: one `<refname>^{}`
|
||||
// query per line, one output line per input line, in order. cat-file echoes the
|
||||
// full query on a `<query> missing` line, so output scales with refname length;
|
||||
// run_bounded_git drains stdout concurrently with the stdin write, so the pipe
|
||||
// cannot deadlock, and the whole peel is bounded by the shared walk deadline.
|
||||
let queries = refnames
|
||||
.iter()
|
||||
.map(|r| format!("{r}^{{}}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let peel_out = run_bounded_git(
|
||||
git_bin,
|
||||
&["cat-file", "--batch-check=%(objecttype)"],
|
||||
repo_path,
|
||||
queries.as_bytes(),
|
||||
deadline,
|
||||
)?;
|
||||
|
||||
let peel_stdout = String::from_utf8_lossy(&peel_out);
|
||||
let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect();
|
||||
// A short read means at least one ref went unclassified — fail closed.
|
||||
if types.len() != refnames.len() {
|
||||
anyhow::bail!(
|
||||
"git cat-file returned {} lines for {} refs; \
|
||||
refusing to produce a partial (under-withheld) set",
|
||||
types.len(),
|
||||
refnames.len()
|
||||
);
|
||||
}
|
||||
for (refname, kind) in refnames.iter().zip(types.iter()) {
|
||||
// git emits `<query> missing` (not the objecttype) when the peel target
|
||||
// is absent; the status word is the last token.
|
||||
if kind.split_ascii_whitespace().last() == Some("missing") {
|
||||
anyhow::bail!(
|
||||
"ref {refname} does not resolve to an object; \
|
||||
refusing to produce a partial (under-withheld) set"
|
||||
);
|
||||
}
|
||||
if *kind != "commit" {
|
||||
anyhow::bail!(
|
||||
"ref {refname} resolves to a {kind}, not a commit; \
|
||||
refusing to produce a partial (under-withheld) set"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn root_tree_pairs(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
@@ -1658,18 +1533,6 @@ fn root_tree_pairs(
|
||||
Ok(set)
|
||||
}
|
||||
|
||||
/// Reachable blob AND withheld-subtree tree OIDs the anonymous (or `caller`)
|
||||
/// replication audience must not receive. The single source for
|
||||
/// `replication_withheld_set`: a blob-only set handed to [`replicable_objects`]
|
||||
/// reintroduces #172 (withheld trees pin to public IPFS).
|
||||
///
|
||||
/// Strict where [`tree_paths`] is lenient. This feeds a withheld filter, so
|
||||
/// absence means replicate; a missed reachable tree would leak. The walk
|
||||
/// therefore matches [`blob_paths`]: [`assert_all_refs_are_commits`] first, then
|
||||
/// a failing `rev-list --all [HEAD]` is an error, then [`object_paths`] (already
|
||||
/// fail-closed on ls-tree / non-UTF-8) plus [`root_tree_pairs`]. [`tree_paths`]
|
||||
/// stays the lenient allow-list walk for GET /ipfs/{cid} and the full-scan
|
||||
/// allowed-tree set, where absence withholds.
|
||||
pub fn withheld_object_oids_bounded(
|
||||
repo_path: &Path,
|
||||
git_bin: &str,
|
||||
@@ -6100,77 +5963,6 @@ esac\n";
|
||||
);
|
||||
}
|
||||
|
||||
/// A planted non-exempt `refs/gitlawb/*` blob ref fails the commit walk
|
||||
/// closed, while the two exempt namespaces (`requests/`, `issues/`) pass.
|
||||
/// The exemption narrowing removed a serving path, so this test pins the
|
||||
/// denial: widening the exemption back to all of `refs/gitlawb/*` turns
|
||||
/// the expect_err red. Real git, scratch bare repo.
|
||||
#[test]
|
||||
fn planted_non_exempt_gitlawb_blob_ref_fails_closed() {
|
||||
use std::time::{Duration, Instant};
|
||||
let td = TempDir::new().unwrap();
|
||||
let bare = td.path().join("repo.git");
|
||||
std::fs::create_dir_all(&bare).unwrap();
|
||||
let run = |args: &[&str]| {
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git {args:?} failed"
|
||||
);
|
||||
};
|
||||
let run_out = |args: &[&str]| {
|
||||
let out = Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(out.status.success(), "git {args:?} failed");
|
||||
String::from_utf8_lossy(&out.stdout).trim().to_string()
|
||||
};
|
||||
run(&["init", "-q", "--bare", "."]);
|
||||
// A blob object to point refs at (node markers and issues are blobs).
|
||||
std::fs::write(bare.join("marker-body"), b"marker\n").unwrap();
|
||||
let blob = run_out(&["hash-object", "-w", "marker-body"]);
|
||||
let check = |want_ok: bool, why: &str| {
|
||||
let result =
|
||||
assert_all_refs_are_commits(&bare, "git", Instant::now() + Duration::from_secs(30));
|
||||
if want_ok {
|
||||
result.expect(why);
|
||||
} else {
|
||||
let err = result.expect_err(why);
|
||||
assert!(
|
||||
err.to_string().contains("refs/gitlawb/evil"),
|
||||
"the denial must name the offending ref, got: {err:#}"
|
||||
);
|
||||
}
|
||||
};
|
||||
// The bare `refs/gitlawb` name fails closed too (checked first: git
|
||||
// forbids a bare ref coexisting with the subtree below, D/F conflict).
|
||||
run(&["update-ref", "refs/gitlawb", &blob]);
|
||||
let result =
|
||||
assert_all_refs_are_commits(&bare, "git", Instant::now() + Duration::from_secs(30));
|
||||
let err = result.expect_err("bare refs/gitlawb blob ref must fail closed");
|
||||
assert!(
|
||||
err.to_string().contains("refs/gitlawb"),
|
||||
"the denial must name the offending ref, got: {err:#}"
|
||||
);
|
||||
run(&["update-ref", "-d", "refs/gitlawb", "--no-deref"]);
|
||||
// Exempt namespaces with blob targets pass (node-managed by design).
|
||||
run(&["update-ref", "refs/gitlawb/requests/r1", &blob]);
|
||||
run(&["update-ref", "refs/gitlawb/issues/i1", &blob]);
|
||||
check(true, "requests//issues/ blob refs are exempt and must pass");
|
||||
// A non-exempt blob ref fails closed.
|
||||
run(&["update-ref", "refs/gitlawb/evil", &blob]);
|
||||
check(false, "a non-exempt refs/gitlawb blob ref must fail closed");
|
||||
// Cleanup proves the denial is about that ref, not residue.
|
||||
run(&["update-ref", "-d", "refs/gitlawb/evil", "--no-deref"]);
|
||||
check(true, "removing the evil ref must restore the walk");
|
||||
}
|
||||
|
||||
/// #218 review round 9 (guidance #1 — single fail-closed
|
||||
/// classification contract): for every non-commit ref shape the
|
||||
/// parser can produce, the FOUR consumers of `(oid, path)` —
|
||||
|
||||
+49
-260
@@ -7,7 +7,6 @@ mod bootstrap;
|
||||
mod cert;
|
||||
mod config;
|
||||
mod db;
|
||||
mod durable_outbox;
|
||||
mod encrypted_pin;
|
||||
mod error;
|
||||
mod git;
|
||||
@@ -25,7 +24,6 @@ mod state;
|
||||
mod sync;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
mod upstream_mirror;
|
||||
mod visibility;
|
||||
mod webhooks;
|
||||
|
||||
@@ -631,13 +629,6 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config);
|
||||
let _queue_lifecycle_sweep = spawn_queue_lifecycle_sweep(&state, &config);
|
||||
// NOTE: the due-request worker is spawned AFTER the one-shot startup
|
||||
// reconcile/drain below, not here: its 5s interval would otherwise tick
|
||||
// during operator/PoS setup and interleave with boot recovery on backlog
|
||||
// rows (state gates prevent data loss, but the interleave costs recovery
|
||||
// latency and widens the reconcile/drain race window).
|
||||
let _marker_cleanup_worker = spawn_marker_cleanup_worker(&state);
|
||||
|
||||
let router = server::build_router(state.clone());
|
||||
// Re-register the socket bound at startup — same fd, so there was never a
|
||||
@@ -695,22 +686,6 @@ async fn main() -> Result<()> {
|
||||
info!("auto-sync worker started");
|
||||
}
|
||||
|
||||
// External-forge mirroring is a separate, default-off worker. Startup
|
||||
// validation already requires owner-only pushes when it is enabled; the
|
||||
// worker additionally verifies the Git runtime can pin DNS answers before
|
||||
// any URL reaches `git fetch`.
|
||||
if config.upstream_mirror_enabled {
|
||||
upstream_mirror::start(
|
||||
Arc::clone(&state.db),
|
||||
Arc::clone(&state.config),
|
||||
state.repo_store.clone(),
|
||||
state.repo_write_leases.clone(),
|
||||
Arc::clone(&state.git_write_semaphore),
|
||||
state.subscribe_shutdown(),
|
||||
)
|
||||
.context("starting upstream mirror worker")?;
|
||||
}
|
||||
|
||||
// Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps
|
||||
// so a dropped replication job never means data loss.
|
||||
{
|
||||
@@ -769,63 +744,6 @@ async fn main() -> Result<()> {
|
||||
let grace = std::time::Duration::from_secs(config.shutdown_grace_secs);
|
||||
info!(grace_secs = config.shutdown_grace_secs, "axum server ready");
|
||||
|
||||
// #26 Split PR 1: drain any `applied` rows left by a previous
|
||||
// process that crashed after Git applied a ref but before the
|
||||
// bookkeeping landed. Runs once, BEFORE the server accepts new
|
||||
// pushes, so a recovery re-derivation does not race a fresh push.
|
||||
// Non-fatal: a transient drain failure is logged and the rows
|
||||
// remain `applied` for the next startup to pick up.
|
||||
//
|
||||
// P1-A: the reconcile step runs FIRST and promotes any `prepared`
|
||||
// or `uncertain` row whose target SHA actually landed on disk.
|
||||
// This is the path that recovers a ref when the post-receive
|
||||
// `mark_pending_ref_transitions_applied` call errored or was
|
||||
// interrupted after `receive_pack` returned Ok, or when
|
||||
// receive-pack returned Err but some refs may have landed
|
||||
// (the `uncertain` state). Without this step, the drain (gated
|
||||
// on `state = 'applied'`) would never see those rows.
|
||||
//
|
||||
// P2 (reviewer-1/2 round 3): use the multi-pass reconcile so
|
||||
// prepared/uncertain rows beyond the first 1000-row page are
|
||||
// processed in the same startup, rather than waiting for the
|
||||
// next restart (where they might age out of MAX_RECONCILE_AGE).
|
||||
match durable_outbox::reconcile_prepared_from_disk_all(
|
||||
state.clone(),
|
||||
durable_outbox::DRAIN_PER_PASS_LIMIT,
|
||||
durable_outbox::DRAIN_MAX_PASSES,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(0) => {}
|
||||
Ok(n) => info!(
|
||||
n,
|
||||
"reconciled prepared/uncertain -> applied via on-disk ref match"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
err = %e,
|
||||
"pending ref transition reconcile failed at startup (non-fatal; will retry on next start)"
|
||||
),
|
||||
}
|
||||
match durable_outbox::drain_receive_pack_requests_all(
|
||||
state.clone(),
|
||||
durable_outbox::DRAIN_PER_PASS_LIMIT,
|
||||
durable_outbox::DRAIN_MAX_PASSES,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(0) => {}
|
||||
Ok(n) => info!(n, "drained pending ref transitions from prior run"),
|
||||
Err(e) => warn!(
|
||||
err = %e,
|
||||
"pending ref transition drain failed at startup (non-fatal; will retry on next start)"
|
||||
),
|
||||
}
|
||||
|
||||
// Start the due-request worker only now that boot recovery is done (see
|
||||
// the note at the other spawns above): its first tick must not interleave
|
||||
// with the one-shot reconcile/drain on backlog rows.
|
||||
let _due_request_worker = spawn_due_request_worker(&state);
|
||||
|
||||
// `into_make_service_with_connect_info` exposes the socket peer address as
|
||||
// `ConnectInfo<SocketAddr>` so the push limiter can key on the real client
|
||||
// when no trusted proxy header applies (see `rate_limit::client_key`).
|
||||
@@ -858,158 +776,6 @@ async fn main() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this
|
||||
/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`,
|
||||
/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not
|
||||
/// the raw-content CID. The opportunistic repair on the pin path only fires when a push
|
||||
/// re-carries the object, which normal git negotiation makes it not do, so those rows
|
||||
/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this
|
||||
/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's
|
||||
/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of
|
||||
/// rewinding. It re-arms after every run, including a failed one, so it never returns
|
||||
/// and there is no awaited value to log here; the shutdown watcher below is what ends
|
||||
/// it.
|
||||
///
|
||||
/// A named function rather than an inline block in `main` so the WIRING has a seam a
|
||||
/// test can call: that the task is spawned at all, that it reads its batch and delay
|
||||
/// from the config knobs rather than some other field, that the caller is not blocked
|
||||
/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own
|
||||
/// behavior is covered elsewhere; this is the boot-path half.
|
||||
fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> {
|
||||
let db = state.db.clone();
|
||||
let repos_dir = config.repos_dir.clone();
|
||||
let git_bin = state.git_bin.clone();
|
||||
let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs);
|
||||
let batch = config.pin_repair_sweep_batch;
|
||||
let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs);
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = ipfs_pin::run_sweep_rearmed(
|
||||
&repos_dir, &git_bin, git_timeout, batch, delay,
|
||||
ipfs_pin::SWEEP_REARM_DELAY, &db,
|
||||
) => {}
|
||||
// Shutdown mid-walk simply drops the run; the persisted cursor means the
|
||||
// next boot picks up where this one stopped.
|
||||
_ = shutdown_rx.changed() => {}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// #26 Split PR 1 step 4 — periodic queue-lifecycle purge. Runs on
|
||||
/// the same detached task pattern as `spawn_legacy_cid_sweep`:
|
||||
/// tokio::spawn with a shutdown watcher, never on the boot path. The
|
||||
/// interval is fixed at 24 hours (the spec calls for "one per
|
||||
/// cluster per day"); the inter-batch delay is implicit in the
|
||||
/// batch size plus the wall-clock cost of each pass. The drain
|
||||
/// (`drain_receive_pack_requests_all`) and the purge
|
||||
/// (`purge_request_queue`) share the same `DRAIN_PER_PASS_LIMIT`
|
||||
/// budget so a 1000-row purge pass takes roughly the same time as
|
||||
/// a 1000-row drain pass.
|
||||
fn spawn_queue_lifecycle_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> {
|
||||
let app_state = state.clone();
|
||||
let retention_days = config.queue_retention_days;
|
||||
let batch = config.queue_purge_batch;
|
||||
let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs.min(30));
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(24 * 3600));
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
// Consume the immediate first tick: Tokio intervals fire
|
||||
// immediately on creation, which would run the purge
|
||||
// concurrently with startup reconcile/drain. The child purge
|
||||
// is terminal-parent-gated in SQL, but delaying the first
|
||||
// sweep a full period removes the race entirely.
|
||||
interval.tick().await;
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(e) = durable_outbox::purge_request_queue(
|
||||
&app_state.db,
|
||||
retention_days,
|
||||
batch,
|
||||
).await {
|
||||
tracing::warn!(err = %e, "queue lifecycle purge failed; will retry on next tick");
|
||||
}
|
||||
// Drain marker tombstones with the bounded deleter;
|
||||
// failures retain the tombstone for the next tick.
|
||||
if let Err(e) = durable_outbox::drain_marker_cleanup_queue(
|
||||
&app_state,
|
||||
batch,
|
||||
git_timeout,
|
||||
).await {
|
||||
tracing::warn!(err = %e, "marker cleanup drain failed");
|
||||
}
|
||||
}
|
||||
_ = shutdown_rx.changed() => {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Shutdown-aware background due-request loop. The startup drain runs
|
||||
/// once; without this, `effects_pending` rows with a future
|
||||
/// `next_attempt_at` would wait until the next process restart.
|
||||
/// Polls the indexed due query with bounded batches and failure
|
||||
/// isolation (one request's error never aborts the batch).
|
||||
fn spawn_due_request_worker(state: &AppState) -> tokio::task::JoinHandle<()> {
|
||||
let app_state = state.clone();
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
match durable_outbox::drain_receive_pack_requests(
|
||||
app_state.clone(),
|
||||
durable_outbox::DRAIN_PER_PASS_LIMIT,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((0, _)) => {}
|
||||
Ok((n, _)) => tracing::info!(n, "due request worker drained"),
|
||||
Err(e) => tracing::warn!(err = %e, "due request worker drain failed"),
|
||||
}
|
||||
}
|
||||
_ = shutdown_rx.changed() => break,
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Dedicated marker-tombstone worker at a shorter cadence than the
|
||||
/// daily purge, so external Git refs do not linger a full period
|
||||
/// after their SQL owner is gone.
|
||||
fn spawn_marker_cleanup_worker(state: &AppState) -> tokio::task::JoinHandle<()> {
|
||||
let app_state = state.clone();
|
||||
let git_timeout =
|
||||
std::time::Duration::from_secs(app_state.config.git_service_timeout_secs.min(30));
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(e) = durable_outbox::drain_marker_cleanup_queue(
|
||||
&app_state,
|
||||
durable_outbox::DRAIN_PER_PASS_LIMIT,
|
||||
git_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(err = %e, "marker cleanup worker failed");
|
||||
}
|
||||
}
|
||||
_ = shutdown_rx.changed() => break,
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn spawn_shutdown_signal(tx: watch::Sender<bool>) {
|
||||
tokio::spawn(async move {
|
||||
#[cfg(unix)]
|
||||
@@ -1497,6 +1263,55 @@ mod rate_limiter_sweep_tests {
|
||||
}
|
||||
|
||||
/// it.
|
||||
fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> {
|
||||
let db = state.db.clone();
|
||||
let repos_dir = config.repos_dir.clone();
|
||||
let git_bin = state.git_bin.clone();
|
||||
let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs);
|
||||
let batch = config.pin_repair_sweep_batch;
|
||||
let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs);
|
||||
let mut shutdown_rx = state.subscribe_shutdown();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = ipfs_pin::run_sweep_rearmed(
|
||||
&repos_dir, &git_bin, git_timeout, batch, delay,
|
||||
ipfs_pin::SWEEP_REARM_DELAY, &db,
|
||||
) => {}
|
||||
_ = shutdown_rx.changed() => {}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn gossip_ping_round(
|
||||
db: &Db,
|
||||
client: &reqwest::Client,
|
||||
failed_once: &mut HashSet<String>,
|
||||
peers: Vec<db::PeerRecord>,
|
||||
) {
|
||||
{
|
||||
let current_dids: HashSet<&str> = peers.iter().map(|peer| peer.did.as_str()).collect();
|
||||
failed_once.retain(|did| current_dids.contains(did.as_str()));
|
||||
}
|
||||
for peer in peers {
|
||||
let ok = ping_peer_readiness(client, &peer.http_url).await;
|
||||
match peer_ping_db_update(failed_once, &peer.did, ok) {
|
||||
Some(reachable) => {
|
||||
if let Err(error) = db.mark_peer_ping(&peer.did, reachable).await {
|
||||
warn!(did = %peer.did, err = %error, "failed to persist peer readiness");
|
||||
}
|
||||
}
|
||||
None => warn!(
|
||||
did = %peer.did,
|
||||
"peer readiness probe failed once; preserving previous federation status"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide whether one readiness sample should update the persisted federation
|
||||
/// gate. A success is authoritative immediately. A failure must repeat on the
|
||||
/// next gossip tick before it can mark a peer unreachable, so one transient
|
||||
/// database hiccup does not hide the peer's repositories for five minutes.
|
||||
fn peer_ping_db_update(failed_once: &mut HashSet<String>, did: &str, ready: bool) -> Option<bool> {
|
||||
if ready {
|
||||
failed_once.remove(did);
|
||||
@@ -1673,32 +1488,6 @@ fn load_or_create_keypair_at(key_path: &std::path::Path) -> Result<Keypair> {
|
||||
Ok(kp)
|
||||
}
|
||||
|
||||
async fn gossip_ping_round(
|
||||
db: &Db,
|
||||
client: &reqwest::Client,
|
||||
failed_once: &mut HashSet<String>,
|
||||
peers: Vec<db::PeerRecord>,
|
||||
) {
|
||||
{
|
||||
let current_dids: HashSet<&str> = peers.iter().map(|peer| peer.did.as_str()).collect();
|
||||
failed_once.retain(|did| current_dids.contains(did.as_str()));
|
||||
}
|
||||
for peer in peers {
|
||||
let ok = ping_peer_readiness(client, &peer.http_url).await;
|
||||
match peer_ping_db_update(failed_once, &peer.did, ok) {
|
||||
Some(reachable) => {
|
||||
if let Err(error) = db.mark_peer_ping(&peer.did, reachable).await {
|
||||
warn!(did = %peer.did, err = %error, "failed to persist peer readiness");
|
||||
}
|
||||
}
|
||||
None => warn!(
|
||||
did = %peer.did,
|
||||
"peer readiness probe failed once; preserving previous federation status"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod gossip_ssrf_tests {
|
||||
use super::{
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use gitlawb_core::identity::Keypair;
|
||||
@@ -436,10 +435,9 @@ async fn process_batch(
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
// Prune vs terminal classification lives in
|
||||
// `handle_sync_item_error` (defer = stays pending for the
|
||||
// next tick; anything else = failed).
|
||||
handle_sync_item_error(db, &item, &origin_url, e).await;
|
||||
warn!(repo = %item.repo, origin = %origin_url, err = %e, "repo sync failed");
|
||||
let _ = db.mark_sync_failed(&item.id).await;
|
||||
crate::metrics::record_sync_processed("failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -693,284 +691,6 @@ async fn existing_promisor_state(repo: &str) -> PromisorProbe {
|
||||
}
|
||||
}
|
||||
|
||||
/// Byte-prefix check for the two node-managed blob namespaces exempt from
|
||||
/// the fail-closed commit walk (`visibility_pack.rs`): request markers
|
||||
/// (`refs/gitlawb/requests/`) and issue records (`refs/gitlawb/issues/`).
|
||||
/// Bytewise so non-UTF-8 refnames (which git accepts) are classified, not
|
||||
/// lossy-decoded into a false exempt match. Extracted so the predicate is
|
||||
/// unit-testable; [`prune_non_exempt_gitlawb_refs`] enforces it on import.
|
||||
fn gitlawb_ref_is_exempt(raw_refname: &[u8]) -> bool {
|
||||
raw_refname.starts_with(b"refs/gitlawb/requests/")
|
||||
|| raw_refname.starts_with(b"refs/gitlawb/issues/")
|
||||
}
|
||||
|
||||
/// Parse one `git show-ref` output line into its refname bytes: `<hex-sha>`
|
||||
/// (40-hex SHA-1 or 64-hex SHA-256, which a sha256 mirror inherits from its
|
||||
/// origin), one space, then the refname. Empty lines (trailing newline) yield
|
||||
/// `Ok(None)`; anything else-shaped is an anomaly the caller fails on rather
|
||||
/// than skips — silently passing an unclassifiable line could leave an evil
|
||||
/// ref imported. Extracted so malformed input is unit-testable without git.
|
||||
fn show_ref_refname(line: &[u8]) -> anyhow::Result<Option<&[u8]>> {
|
||||
if line.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(sp) = line.iter().position(|b| *b == b' ') else {
|
||||
return Err(anyhow::anyhow!("git show-ref emitted an unparsable line"));
|
||||
};
|
||||
let (sha, raw) = (&line[..sp], &line[sp + 1..]);
|
||||
let sha_ok = (sha.len() == 40 || sha.len() == 64) && sha.iter().all(|b| b.is_ascii_hexdigit());
|
||||
if !sha_ok || raw.is_empty() {
|
||||
return Err(anyhow::anyhow!("git show-ref emitted an unparsable line"));
|
||||
}
|
||||
Ok(Some(raw))
|
||||
}
|
||||
|
||||
/// Delete imported `refs/gitlawb/*` refs outside the two exempt subtrees.
|
||||
///
|
||||
/// Mirrors import whatever the origin advertises (`+refs/*:refs/*`), and
|
||||
/// `transfer.hideRefs` is a serve-side knob: a repo's own `git fetch` never
|
||||
/// consults local hideRefs, and origins hide only `requests/` anyway. So a
|
||||
/// planted `refs/gitlawb/<other>` blob ref — creatable on any pre-gate node —
|
||||
/// lands on every mirror and then trips `assert_all_refs_are_commits` closed,
|
||||
/// wedging serving on each one. Prune at import instead: after clone/fetch,
|
||||
/// list `refs/gitlawb` and delete every ref outside the exempt subtrees
|
||||
/// (negative refspecs cannot re-include them, so post-import deletion is the
|
||||
/// mechanism that works).
|
||||
///
|
||||
/// Callers map a transient prune error to [`PruneFailed`] so the batch
|
||||
/// defers the row (stays `pending`, retried next tick) instead of failing
|
||||
/// it: a failed row would leave a possibly-wedged mirror unretried until
|
||||
/// the next peer announcement. Deterministic refusals (identity mismatch,
|
||||
/// malformed enumeration) map to [`PruneInvalid`], which fails terminally
|
||||
/// since no retry can clear them.
|
||||
///
|
||||
/// Enumeration is `git show-ref` (no pattern), parsed as raw bytes: refnames
|
||||
/// cannot contain newlines, so one line per ref is lossless even for
|
||||
/// non-UTF-8 names, which a lossy decode would miss. The `<sha> SP <refname>`
|
||||
/// structure (not `for-each-ref --format=%(refname)`, which carries no object
|
||||
/// id) is what lets the parser width-check 40-hex SHA-1 vs 64-hex SHA-256
|
||||
/// and reject anything else-shaped instead of skipping it.
|
||||
///
|
||||
/// Fail-closed throughout: the exit code is classified (only `1` with empty
|
||||
/// output means "no refs"; anything else non-zero, e.g. 128 on a missing
|
||||
/// repo, is fatal, since a silent skip would leave an imported evil ref in
|
||||
/// place while the sync reports success), and any line that is not
|
||||
/// `<hex-sha> SP <refname>` — 40-hex SHA-1 or 64-hex SHA-256, which a sha256
|
||||
/// mirror inherits from its origin — fails the sync rather than being
|
||||
/// skipped into an unpruned wedge.
|
||||
/// Bounded git spawn for the prune below: `kill_on_drop` plus a hard timeout
|
||||
/// so a wedged child cannot stall a mirror import or sweep forever. The
|
||||
/// refname delete runs on exact bytes, so args arrive as `OsStr`, not `str`.
|
||||
/// Non-PruneInvalid failures (including a timeout) wrap as PruneFailed and
|
||||
/// defer for retry, matching the transient classification below.
|
||||
async fn prune_git(
|
||||
local_path: &Path,
|
||||
args: &[&std::ffi::OsStr],
|
||||
label: &str,
|
||||
) -> anyhow::Result<std::process::Output> {
|
||||
let fut = async {
|
||||
// allow-unbounded-git: bounded by the tokio::time::timeout wrapping
|
||||
// this future below, plus kill_on_drop.
|
||||
let mut cmd = tokio::process::Command::new("git");
|
||||
cmd.kill_on_drop(true);
|
||||
cmd.arg("-C").arg(local_path).args(args);
|
||||
cmd.output().await
|
||||
};
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), fut)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("{label} timed out"))?
|
||||
.map_err(|e| anyhow::anyhow!("{label} failed to spawn: {e}"))
|
||||
}
|
||||
|
||||
async fn prune_non_exempt_gitlawb_refs(local_path: &Path) -> anyhow::Result<()> {
|
||||
let local_str = local_path.to_str().unwrap_or(".");
|
||||
// Prove the path is itself a repo before trusting the enumeration:
|
||||
// `git -C` walks up to the nearest ancestor repo, so inside a non-repo
|
||||
// child (a partially-created mirror) `show-ref` would exit 0 with the
|
||||
// ANCESTOR's refs and `update-ref -d` would then run against that repo.
|
||||
// Require `rev-parse --absolute-git-dir` to resolve to the path itself
|
||||
// (mirrors are bare repos, where the git dir IS the path).
|
||||
let git_dir_out = prune_git(
|
||||
local_path,
|
||||
&[
|
||||
std::ffi::OsStr::new("rev-parse"),
|
||||
std::ffi::OsStr::new("--absolute-git-dir"),
|
||||
],
|
||||
"git rev-parse",
|
||||
)
|
||||
.await?;
|
||||
if !git_dir_out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&git_dir_out.stderr);
|
||||
return Err(anyhow::anyhow!(PruneInvalid(format!(
|
||||
"mirror path is not a git repo (rev-parse failed): {stderr}"
|
||||
))));
|
||||
}
|
||||
let actual_dir = PathBuf::from(
|
||||
String::from_utf8_lossy(&git_dir_out.stdout)
|
||||
.trim()
|
||||
.to_string(),
|
||||
);
|
||||
let canonical_path = local_path
|
||||
.canonicalize()
|
||||
.map_err(|e| anyhow::anyhow!(PruneInvalid(format!("mirror path does not resolve: {e}"))))?;
|
||||
let canonical_git_dir = actual_dir
|
||||
.canonicalize()
|
||||
.map_err(|e| anyhow::anyhow!(PruneInvalid(format!("git dir does not resolve: {e}"))))?;
|
||||
if canonical_git_dir != canonical_path {
|
||||
return Err(anyhow::anyhow!(PruneInvalid(format!(
|
||||
"mirror path {local_str} is not itself a repo (git dir resolves elsewhere); refusing to enumerate"
|
||||
))));
|
||||
}
|
||||
let out = prune_git(
|
||||
local_path,
|
||||
&[std::ffi::OsStr::new("show-ref")],
|
||||
"git show-ref",
|
||||
)
|
||||
.await?;
|
||||
// Classify on the exit code (the `existing_promisor_state` precedent):
|
||||
// 0 parses, 1 with empty output is an empty repo, anything else is a
|
||||
// fatal enumeration failure that must not read as "nothing to prune".
|
||||
match out.status.code() {
|
||||
Some(0) => {}
|
||||
Some(1) if out.stdout.is_empty() => return Ok(()),
|
||||
_ => {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
return Err(anyhow::anyhow!(
|
||||
"git show-ref failed with code {:?}: {stderr}",
|
||||
out.status.code()
|
||||
));
|
||||
}
|
||||
}
|
||||
for line in out.stdout.split(|b| *b == b'\n') {
|
||||
// Malformed lines are deterministic (same git, same output every
|
||||
// tick): fail terminally via PruneInvalid, not transiently.
|
||||
let Some(raw) =
|
||||
show_ref_refname(line).map_err(|e| anyhow::anyhow!(PruneInvalid(e.to_string())))?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !(raw == b"refs/gitlawb" || raw.starts_with(b"refs/gitlawb/"))
|
||||
|| gitlawb_ref_is_exempt(raw)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Delete by exact bytes (not the lossy display string below): a
|
||||
// non-UTF-8 refname would otherwise survive the prune.
|
||||
let refname_os = os_from_raw_refname(raw);
|
||||
let shown = refname_os.to_string_lossy().into_owned();
|
||||
let delete = prune_git(
|
||||
local_path,
|
||||
&[
|
||||
std::ffi::OsStr::new("update-ref"),
|
||||
std::ffi::OsStr::new("-d"),
|
||||
refname_os.as_os_str(),
|
||||
std::ffi::OsStr::new("--no-deref"),
|
||||
],
|
||||
"git update-ref -d",
|
||||
)
|
||||
.await?;
|
||||
if !delete.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&delete.stderr);
|
||||
return Err(anyhow::anyhow!(
|
||||
"git update-ref -d {shown} failed (mirror still carries a non-exempt refs/gitlawb ref): {stderr}"
|
||||
));
|
||||
}
|
||||
tracing::info!(
|
||||
gitlawb_ref = %shown,
|
||||
"mirror: pruned non-exempt refs/gitlawb ref on import"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Lossless bytes → OsString for `update-ref -d`: non-UTF-8 refnames must be
|
||||
/// deleted by exact bytes. Unix is the serving platform; elsewhere fall back
|
||||
/// to a lossy decode (best effort where exact bytes are unrepresentable).
|
||||
#[cfg(unix)]
|
||||
fn os_from_raw_refname(raw: &[u8]) -> std::ffi::OsString {
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
std::ffi::OsString::from_vec(raw.to_vec())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn os_from_raw_refname(raw: &[u8]) -> std::ffi::OsString {
|
||||
String::from_utf8_lossy(raw).into_owned().into()
|
||||
}
|
||||
|
||||
/// Marker for mirror-prune failures: the clone/fetch succeeded, so the
|
||||
/// mirror is on disk, but a non-exempt `refs/gitlawb/*` ref may still be
|
||||
/// imported. Local-git conditions are transient (a retry can succeed), while
|
||||
/// `mark_sync_failed` is terminal until the next peer announcement — and a
|
||||
/// failed row would leave a possibly-wedged mirror unretried in the
|
||||
/// meantime. The batch error arm downcasts to this and defers the row
|
||||
/// (stays `pending`, retried on the next 30s tick, where fetch+prune are
|
||||
/// idempotent) instead of failing it.
|
||||
#[derive(Debug)]
|
||||
struct PruneFailed(String);
|
||||
|
||||
impl std::fmt::Display for PruneFailed {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "mirror prune failed: {}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PruneFailed {}
|
||||
|
||||
/// Marker for deterministic mirror-prune refusals: the identity check (path
|
||||
/// is not itself a repo) and malformed `show-ref` output cannot clear on
|
||||
/// retry — re-running fetch+prune every tick would spin forever on a
|
||||
/// permanently bad path. Unlike [`PruneFailed`], these fail the row
|
||||
/// terminally so the condition surfaces instead of looping silently.
|
||||
#[derive(Debug)]
|
||||
struct PruneInvalid(String);
|
||||
|
||||
/// Preserve a deterministic prune refusal through the clone/fetch call
|
||||
/// sites: `prune_non_exempt_gitlawb_refs` already returns `PruneInvalid`
|
||||
/// for identity mismatches and malformed enumeration, and re-wrapping
|
||||
/// every error as `PruneFailed` would stringify that type away before
|
||||
/// `handle_sync_item_error` downcasts it — leaving a permanently invalid
|
||||
/// mirror deferred (fetch+prune every tick) instead of failed. Transient
|
||||
/// failures keep the `PruneFailed` deferral.
|
||||
fn wrap_prune_error(e: anyhow::Error) -> anyhow::Error {
|
||||
match e.downcast::<PruneInvalid>() {
|
||||
Ok(invalid) => anyhow::Error::new(invalid),
|
||||
Err(other) => anyhow::anyhow!(PruneFailed(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for PruneInvalid {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "mirror prune refused: {}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PruneInvalid {}
|
||||
|
||||
/// Classify a mirror sync failure for one queue row. Transient prune
|
||||
/// failures ([`PruneFailed`]) defer: the mirror is on disk but may still
|
||||
/// carry a non-exempt refs/gitlawb ref, a transient local-git condition a
|
||||
/// retry can clear — failing the row would leave it unretried until the
|
||||
/// next peer announcement, wedging serving in the meantime. Deterministic
|
||||
/// refusals ([`PruneInvalid`]) and everything else fail the row terminally:
|
||||
/// retrying a permanently bad path every tick would spin forever instead of
|
||||
/// surfacing. Extracted so the deferral (not just the prune itself) is
|
||||
/// unit-testable.
|
||||
async fn handle_sync_item_error(
|
||||
db: &Db,
|
||||
item: &crate::db::SyncQueueItem,
|
||||
origin_url: &str,
|
||||
err: anyhow::Error,
|
||||
) {
|
||||
if err.downcast_ref::<PruneFailed>().is_some() {
|
||||
warn!(repo = %item.repo, origin = %origin_url, err = %err, "mirror prune failed; leaving sync row pending for retry");
|
||||
crate::metrics::record_sync_processed("deferred");
|
||||
return;
|
||||
}
|
||||
warn!(repo = %item.repo, origin = %origin_url, err = %err, "repo sync failed");
|
||||
let _ = db.mark_sync_failed(&item.id).await;
|
||||
crate::metrics::record_sync_processed("failed");
|
||||
}
|
||||
|
||||
/// Mirror-clone a repo from a remote URL into a local bare repo.
|
||||
/// `Promisor` mode adds `--filter=blob:limit=10g`, which marks the repo a git
|
||||
/// promisor (so a pack with origin-omitted withheld blobs is accepted) while
|
||||
@@ -995,13 +715,6 @@ async fn clone_repo(remote_url: &str, local_path: &Path, mode: MirrorMode) -> an
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
return Err(anyhow::anyhow!("git clone --mirror failed: {stderr}"));
|
||||
}
|
||||
// A fresh mirror imports everything the origin advertises, including any
|
||||
// planted non-exempt `refs/gitlawb/*` ref; prune before serving from it.
|
||||
// Prune outcome flows through `wrap_prune_error`: transient failures
|
||||
// defer (stays pending), deterministic refusals fail terminally.
|
||||
prune_non_exempt_gitlawb_refs(local_path)
|
||||
.await
|
||||
.map_err(wrap_prune_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1022,11 +735,7 @@ async fn fetch_repo(local_path: &Path, remote_url: &str, mode: MirrorMode) -> an
|
||||
])
|
||||
.await?;
|
||||
|
||||
// The fetch imports whatever the origin advertises via the stored
|
||||
// `+refs/*:refs/*` refspec; prune non-exempt `refs/gitlawb/*` refs after
|
||||
// every fetch so one planted origin ref cannot wedge serving on the
|
||||
// mirror (see `prune_non_exempt_gitlawb_refs`).
|
||||
let fetch_result = match mode {
|
||||
match mode {
|
||||
MirrorMode::Promisor => {
|
||||
git_run(&["-C", local_str, "config", "remote.origin.promisor", "true"]).await?;
|
||||
git_run(&[
|
||||
@@ -1066,13 +775,7 @@ async fn fetch_repo(local_path: &Path, remote_url: &str, mode: MirrorMode) -> an
|
||||
git_run(&["-C", local_str, "fetch", "--prune", "origin"]).await
|
||||
}
|
||||
}
|
||||
};
|
||||
fetch_result?;
|
||||
// Prune after every fetch (see above); outcome flows through
|
||||
// `wrap_prune_error` like the clone path.
|
||||
prune_non_exempt_gitlawb_refs(local_path)
|
||||
.await
|
||||
.map_err(wrap_prune_error)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2405,463 +2108,4 @@ mod tests {
|
||||
assert!(object_count(&mirror) > 0, "mirror has no objects");
|
||||
assert_eq!(sync_status(&pool, "z6Mkfoo/hello").await, "done");
|
||||
}
|
||||
|
||||
/// The import-exemption predicate is bytewise over both node-managed
|
||||
/// namespaces and nothing else. Non-UTF-8 bytes never match (git accepts
|
||||
/// them, so a lossy decode could false-exempt). Narrowing either prefix
|
||||
/// turns these red.
|
||||
#[test]
|
||||
fn gitlawb_import_exemption_covers_both_namespaces_by_bytes() {
|
||||
for exempt in [
|
||||
&b"refs/gitlawb/requests/550e8400-e29b-41d4-a716-446655440000"[..],
|
||||
&b"refs/gitlawb/issues/abc123"[..],
|
||||
] {
|
||||
assert!(
|
||||
gitlawb_ref_is_exempt(exempt),
|
||||
"import must keep {}",
|
||||
String::from_utf8_lossy(exempt)
|
||||
);
|
||||
}
|
||||
for pruned in [
|
||||
&b"refs/gitlawb/evil"[..],
|
||||
&b"refs/gitlawb"[..],
|
||||
&b"refs/gitlawbfoo"[..],
|
||||
&b"refs/heads/main"[..],
|
||||
&b""[..],
|
||||
// Raw 0xFF byte: never an exempt prefix, always pruned.
|
||||
&b"refs/gitlawb/\xffvil"[..],
|
||||
] {
|
||||
assert!(
|
||||
!gitlawb_ref_is_exempt(pruned),
|
||||
"import must prune {}",
|
||||
String::from_utf8_lossy(pruned)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// End-to-end prune on a scratch bare repo: a planted non-exempt
|
||||
/// `refs/gitlawb/*` blob ref is deleted, exempt `requests/`/`issues/`
|
||||
/// blob refs and content refs survive. Removing the prune call from
|
||||
/// `clone_repo`/`fetch_repo` leaves the evil ref in place, which is what
|
||||
/// wedges serving on mirrors — this test plants exactly that state and
|
||||
/// proves the prune removes it.
|
||||
#[test]
|
||||
fn prune_removes_non_exempt_gitlawb_refs_and_keeps_exempt() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
rt.block_on(async {
|
||||
let td = TempDir::new().unwrap();
|
||||
let work = td.path().join("work");
|
||||
let bare = td.path().join("m.git");
|
||||
std::fs::create_dir_all(&work).unwrap();
|
||||
// A real commit for the content branch (branch refs must point
|
||||
// at commits; the evil/exempt refs below point at blobs).
|
||||
std::fs::write(work.join("f.txt"), b"x\n").unwrap();
|
||||
let run_work = |args: &[&str]| {
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&work)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git {args:?} failed"
|
||||
);
|
||||
};
|
||||
run_work(&["init", "-q"]);
|
||||
run_work(&["config", "user.email", "t@t"]);
|
||||
run_work(&["config", "user.name", "t"]);
|
||||
run_work(&["add", "f.txt"]);
|
||||
run_work(&["commit", "-qm", "init"]);
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
"-q",
|
||||
"--bare",
|
||||
work.to_str().unwrap(),
|
||||
bare.to_str().unwrap()
|
||||
])
|
||||
.current_dir(td.path())
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git clone --bare failed"
|
||||
);
|
||||
let run = |args: &[&str]| {
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git {args:?} failed"
|
||||
);
|
||||
};
|
||||
let run_out = |args: &[&str]| {
|
||||
let out = Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(out.status.success(), "git {args:?} failed");
|
||||
out.stdout
|
||||
};
|
||||
run(&["init", "-q", "--bare", "."]);
|
||||
std::fs::write(bare.join("blob-body"), b"marker\n").unwrap();
|
||||
let blob = String::from_utf8_lossy(&run_out(&["hash-object", "-w", "blob-body"]))
|
||||
.trim()
|
||||
.to_string();
|
||||
// The clone carries the content branch; plant the evil ref and
|
||||
// both exempt refs (all blob-pointing, like node metadata).
|
||||
run(&["update-ref", "refs/gitlawb/evil", &blob]);
|
||||
run(&["update-ref", "refs/gitlawb/requests/r1", &blob]);
|
||||
run(&["update-ref", "refs/gitlawb/issues/i1", &blob]);
|
||||
|
||||
prune_non_exempt_gitlawb_refs(&bare)
|
||||
.await
|
||||
.expect("prune succeeds");
|
||||
|
||||
let raw_refs = run_out(&["for-each-ref", "--format=%(refname)"]);
|
||||
let refs = String::from_utf8_lossy(&raw_refs);
|
||||
let set: std::collections::HashSet<&str> = refs
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
assert!(
|
||||
!set.contains("refs/gitlawb/evil"),
|
||||
"non-exempt blob ref must be pruned, still present: {set:?}"
|
||||
);
|
||||
for keep in ["refs/gitlawb/requests/r1", "refs/gitlawb/issues/i1"] {
|
||||
assert!(set.contains(keep), "{keep} must survive the prune: {set:?}");
|
||||
}
|
||||
assert!(
|
||||
set.iter().any(|r| r.starts_with("refs/heads/")),
|
||||
"content branch ref must survive the prune: {set:?}"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// `show-ref` line parser: 40-hex SHA-1 and 64-hex SHA-256 lines yield
|
||||
/// the refname, empty lines yield None, and anything else-shaped errors
|
||||
/// (the caller fails the sync rather than skipping into an unpruned
|
||||
/// wedge). Reverting to a 40-only width check turns the sha256 case red.
|
||||
#[test]
|
||||
fn show_ref_line_parser_accepts_both_hash_widths_and_rejects_junk() {
|
||||
// 40-hex SHA-1 line.
|
||||
let sha1_line = format!("{} refs/gitlawb/evil", "a".repeat(40));
|
||||
assert_eq!(
|
||||
show_ref_refname(sha1_line.as_bytes()).unwrap(),
|
||||
Some(b"refs/gitlawb/evil".as_slice())
|
||||
);
|
||||
// 64-hex SHA-256 line (a sha256 mirror inherits the width).
|
||||
let sha256_line = format!("{} refs/gitlawb/evil", "b".repeat(64));
|
||||
assert_eq!(
|
||||
show_ref_refname(sha256_line.as_bytes()).unwrap(),
|
||||
Some(b"refs/gitlawb/evil".as_slice())
|
||||
);
|
||||
// Non-UTF-8 refname passes through as exact bytes.
|
||||
let mut raw = vec![b'c'; 40];
|
||||
raw.extend_from_slice(b" refs/gitlawb/\xffvil");
|
||||
assert_eq!(
|
||||
show_ref_refname(&raw).unwrap(),
|
||||
Some(b"refs/gitlawb/\xffvil".as_slice())
|
||||
);
|
||||
// Trailing newline residue.
|
||||
assert_eq!(show_ref_refname(b"").unwrap(), None);
|
||||
// No space, short sha, non-hex sha, empty refname: all errors.
|
||||
let bads: Vec<Vec<u8>> = vec![
|
||||
b"nospaceshere".to_vec(),
|
||||
format!("{}refs/x", "a".repeat(40)).into_bytes(),
|
||||
[b'z'; 40]
|
||||
.iter()
|
||||
.chain(b" refs/x".iter())
|
||||
.copied()
|
||||
.collect(),
|
||||
format!("{} ", "a".repeat(40)).into_bytes(),
|
||||
format!("{} refs/x", "a".repeat(41)).into_bytes(),
|
||||
];
|
||||
for bad in &bads {
|
||||
assert!(
|
||||
show_ref_refname(bad).is_err(),
|
||||
"malformed show-ref line must error, not skip"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Pruning a path that is not a repo errors. Two shapes: a missing path
|
||||
/// fails `rev-parse` outright (exit 128), while a plain subdirectory of
|
||||
/// a repo would otherwise enumerate the ancestor's refs — covered by the
|
||||
/// identity test below. Either way the prune refuses instead of pruning
|
||||
/// the wrong repo or silently skipping an evil ref.
|
||||
#[test]
|
||||
fn prune_on_non_repo_path_errors() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
rt.block_on(async {
|
||||
let err =
|
||||
prune_non_exempt_gitlawb_refs(Path::new("/tmp/gitlawb-definitely-not-a-repo-xyz"))
|
||||
.await
|
||||
.expect_err("non-repo path must fail the prune, not skip it");
|
||||
assert!(
|
||||
err.to_string().contains("not a git repo")
|
||||
|| err.to_string().contains("does not resolve"),
|
||||
"unexpected error shape: {err:#}"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Identity check: pruning a plain subdirectory of a repo refuses instead
|
||||
/// of enumerating the ancestor. `git -C` on the child exits 0 from
|
||||
/// `show-ref` with the ANCESTOR's refs, so without the
|
||||
/// `rev-parse --absolute-git-dir` identity check the delete loop would
|
||||
/// enumerate and delete the wrong repository's refs. Deleting the check
|
||||
/// makes the prune succeed and removes the ancestor's evil ref — both
|
||||
/// assertions go red.
|
||||
#[test]
|
||||
fn prune_on_non_repo_child_of_repo_refuses_and_touches_nothing() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
rt.block_on(async {
|
||||
let td = TempDir::new().unwrap();
|
||||
let ancestor = td.path().join("ancestor.git");
|
||||
std::fs::create_dir_all(&ancestor).unwrap();
|
||||
let run_ancestor = |args: &[&str]| {
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&ancestor)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git {args:?} failed"
|
||||
);
|
||||
};
|
||||
run_ancestor(&["init", "-q", "--bare", "."]);
|
||||
// Blob target for the planted ref (mirrors the node-metadata shape).
|
||||
std::fs::write(ancestor.join("blob-body"), b"marker\n").unwrap();
|
||||
let out = Command::new("git")
|
||||
.args(["hash-object", "-w", "blob-body"])
|
||||
.current_dir(&ancestor)
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(out.status.success(), "git hash-object failed");
|
||||
let blob = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
run_ancestor(&["update-ref", "refs/gitlawb/evil", &blob]);
|
||||
|
||||
// A plain child directory: not itself a repo.
|
||||
let child = ancestor.join("not-a-repo");
|
||||
std::fs::create_dir_all(&child).unwrap();
|
||||
|
||||
let err = prune_non_exempt_gitlawb_refs(&child)
|
||||
.await
|
||||
.expect_err("pruning a non-repo child must refuse, not enumerate the ancestor");
|
||||
assert!(
|
||||
err.to_string().contains("not itself a repo"),
|
||||
"refusal must name the identity mismatch, got: {err:#}"
|
||||
);
|
||||
|
||||
// The ancestor's evil ref is untouched.
|
||||
let refs_out = Command::new("git")
|
||||
.args(["for-each-ref", "--format=%(refname)"])
|
||||
.current_dir(&ancestor)
|
||||
.output()
|
||||
.unwrap();
|
||||
let refs = String::from_utf8_lossy(&refs_out.stdout);
|
||||
assert!(
|
||||
refs.lines().any(|l| l.trim() == "refs/gitlawb/evil"),
|
||||
"ancestor evil ref must survive the refused prune"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// sha256 mirror: a 64-hex `show-ref` line parses and the planted ref is
|
||||
/// pruned. A 40-only parser skips every line and leaves the evil ref.
|
||||
#[test]
|
||||
fn prune_parses_sha256_show_ref_and_prunes() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
rt.block_on(async {
|
||||
let td = TempDir::new().unwrap();
|
||||
let bare = td.path().join("m256.git");
|
||||
std::fs::create_dir_all(&bare).unwrap();
|
||||
let run = |args: &[&str]| {
|
||||
assert!(
|
||||
Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success(),
|
||||
"git {args:?} failed"
|
||||
);
|
||||
};
|
||||
let run_out = |args: &[&str]| {
|
||||
let out = Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(&bare)
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(out.status.success(), "git {args:?} failed");
|
||||
out.stdout
|
||||
};
|
||||
run(&["init", "-q", "--bare", "--object-format=sha256", "."]);
|
||||
std::fs::write(bare.join("blob-body"), b"marker\n").unwrap();
|
||||
let blob = String::from_utf8_lossy(&run_out(&["hash-object", "-w", "blob-body"]))
|
||||
.trim()
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
blob.len(),
|
||||
64,
|
||||
"sanity: sha256 repo must mint 64-hex objects"
|
||||
);
|
||||
run(&["update-ref", "refs/gitlawb/evil", &blob]);
|
||||
run(&["update-ref", "refs/gitlawb/issues/i1", &blob]);
|
||||
|
||||
prune_non_exempt_gitlawb_refs(&bare)
|
||||
.await
|
||||
.expect("prune succeeds on a sha256 repo");
|
||||
|
||||
let raw_refs = run_out(&["for-each-ref", "--format=%(refname)"]);
|
||||
let refs = String::from_utf8_lossy(&raw_refs);
|
||||
let set: std::collections::HashSet<&str> = refs
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
assert!(
|
||||
!set.contains("refs/gitlawb/evil"),
|
||||
"evil ref must be pruned from sha256 mirror: {set:?}"
|
||||
);
|
||||
assert!(
|
||||
set.contains("refs/gitlawb/issues/i1"),
|
||||
"exempt ref must survive: {set:?}"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Prune failures defer the queue row (stays `pending` for the next
|
||||
/// tick); deterministic prune refusals and any other sync error fail it
|
||||
/// terminally. Deleting the `downcast_ref::<PruneFailed>` arm restores
|
||||
/// terminal failure and turns the first assertion red.
|
||||
/// Prune failures defer the queue row (stays `pending` for the next
|
||||
/// tick); deterministic prune refusals and any other sync error fail it
|
||||
/// terminally. Deleting the `downcast_ref::<PruneFailed>` arm restores
|
||||
/// terminal failure and turns the first assertion red.
|
||||
///
|
||||
/// The wrapper below is what the clone/fetch call sites run through:
|
||||
/// a `PruneInvalid` from the helper must still downcast as
|
||||
/// `PruneInvalid` afterwards (not stringified into `PruneFailed`),
|
||||
/// and anything else must arrive as `PruneFailed`.
|
||||
#[test]
|
||||
fn wrap_prune_error_preserves_deterministic_type() {
|
||||
let invalid = wrap_prune_error(anyhow::anyhow!(PruneInvalid("x".to_string())));
|
||||
assert!(
|
||||
invalid.downcast_ref::<PruneInvalid>().is_some(),
|
||||
"deterministic refusal must survive the call-site wrap"
|
||||
);
|
||||
assert!(
|
||||
invalid.downcast_ref::<PruneFailed>().is_none(),
|
||||
"deterministic refusal must not also read as transient"
|
||||
);
|
||||
let transient = wrap_prune_error(anyhow::anyhow!("boom"));
|
||||
assert!(
|
||||
transient.downcast_ref::<PruneFailed>().is_some(),
|
||||
"generic failure must arrive as transient"
|
||||
);
|
||||
assert!(
|
||||
transient.downcast_ref::<PruneInvalid>().is_none(),
|
||||
"generic failure must not read as deterministic"
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test]
|
||||
async fn prune_failure_defers_row_other_failures_fail_it(pool: PgPool) {
|
||||
let state = crate::test_support::test_state(pool).await;
|
||||
let item = |id: &str| crate::db::SyncQueueItem {
|
||||
id: id.to_string(),
|
||||
repo: "z6Mkfoo/hello".to_string(),
|
||||
node_did: "did:key:z6MkOrigin".to_string(),
|
||||
ref_name: "refs/heads/main".to_string(),
|
||||
new_sha: "0".repeat(40),
|
||||
cid: None,
|
||||
status: "pending".to_string(),
|
||||
enqueued_at: chrono::Utc::now().to_rfc3339(),
|
||||
};
|
||||
async fn status_of(pool: &sqlx::PgPool, id: &str) -> String {
|
||||
let row: (String,) = sqlx::query_as("SELECT status FROM sync_queue WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
row.0
|
||||
}
|
||||
// Seed three pending rows directly (enqueue_sync mints random ids).
|
||||
for id in ["sync-prune-defer", "sync-prune-invalid", "sync-other-fail"] {
|
||||
sqlx::query(
|
||||
"INSERT INTO sync_queue (id, repo, node_did, ref_name, new_sha, cid, status, enqueued_at)
|
||||
VALUES ($1, $2, $3, $4, $5, NULL, 'pending', $6)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind("z6Mkfoo/hello")
|
||||
.bind("did:key:z6MkOrigin")
|
||||
.bind("refs/heads/main")
|
||||
.bind("0".repeat(40))
|
||||
.bind(chrono::Utc::now().to_rfc3339())
|
||||
.execute(state.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
handle_sync_item_error(
|
||||
&state.db,
|
||||
&item("sync-prune-defer"),
|
||||
"http://origin.example",
|
||||
anyhow::anyhow!(PruneFailed("boom".to_string())),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
status_of(state.db.pool(), "sync-prune-defer").await,
|
||||
"pending",
|
||||
"prune failure must leave the row pending for retry"
|
||||
);
|
||||
|
||||
handle_sync_item_error(
|
||||
&state.db,
|
||||
&item("sync-other-fail"),
|
||||
"http://origin.example",
|
||||
anyhow::anyhow!("boom"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
status_of(state.db.pool(), "sync-other-fail").await,
|
||||
"failed",
|
||||
"non-prune failure must fail the row"
|
||||
);
|
||||
|
||||
// Deterministic refusal: no retry can clear it, so it fails
|
||||
// terminally instead of spinning every tick forever.
|
||||
handle_sync_item_error(
|
||||
&state.db,
|
||||
&item("sync-prune-invalid"),
|
||||
"http://origin.example",
|
||||
anyhow::anyhow!(PruneInvalid("not itself a repo".to_string())),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
status_of(state.db.pool(), "sync-prune-invalid").await,
|
||||
"failed",
|
||||
"deterministic prune refusal must fail the row, not defer forever"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,968 +0,0 @@
|
||||
//! Scheduled external-forge fetches for canonical INBOUND mirrors.
|
||||
//!
|
||||
//! This worker is deliberately separate from `sync`, which mirrors repositories
|
||||
//! between Gitlawb peers. External upstream URLs are owner-controlled data that
|
||||
//! eventually reach `git fetch`, so each cycle resolves DNS itself, validates
|
||||
//! every address, and pins the approved answers into Git/libcurl with
|
||||
//! `http.curloptResolve`. Redirects, proxies, credential helpers, submodule
|
||||
//! recursion, and non-HTTPS protocols are disabled at the command boundary.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::IpAddr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::{Db, InboundMirrorTarget, MirrorStatus, MirrorTransitionPhase};
|
||||
use crate::git::repo_store::RepoStore;
|
||||
use crate::state::{repo_identity_key, RepoWriteLeases};
|
||||
|
||||
const DNS_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const MAX_PINNED_ADDRESSES: usize = 32;
|
||||
const MIN_CURL_OPT_RESOLVE_GIT: (u64, u64) = (2, 37);
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct EgressPolicy;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct PinnedUpstream {
|
||||
url: String,
|
||||
curlopt_resolve: Option<String>,
|
||||
}
|
||||
|
||||
impl EgressPolicy {
|
||||
async fn resolve(&self, url: reqwest::Url) -> Result<PinnedUpstream> {
|
||||
let raw_host = url.host_str().context("mirror upstream URL has no host")?;
|
||||
let connect_host = raw_host
|
||||
.trim_start_matches('[')
|
||||
.trim_end_matches(']')
|
||||
.to_ascii_lowercase();
|
||||
let policy_host = normalize_policy_host(&connect_host);
|
||||
reject_always_local_hostname(&policy_host)?;
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.context("mirror upstream URL has no known port")?;
|
||||
|
||||
if let Ok(ip) = connect_host.parse::<IpAddr>() {
|
||||
self.validate_addresses(&policy_host, &[ip])?;
|
||||
return Ok(PinnedUpstream {
|
||||
url: url.to_string(),
|
||||
// An IP literal performs no DNS lookup, so there is no rebinding
|
||||
// window to pin. TLS still verifies the literal from the URL.
|
||||
curlopt_resolve: None,
|
||||
});
|
||||
}
|
||||
|
||||
let resolved = tokio::time::timeout(
|
||||
DNS_TIMEOUT,
|
||||
tokio::net::lookup_host((connect_host.as_str(), port)),
|
||||
)
|
||||
.await
|
||||
.context("mirror upstream DNS lookup timed out")?
|
||||
.context("resolving mirror upstream host")?;
|
||||
let addresses: BTreeSet<IpAddr> = resolved.map(|addr| addr.ip()).collect();
|
||||
if addresses.is_empty() {
|
||||
anyhow::bail!("mirror upstream host resolved to no addresses");
|
||||
}
|
||||
let addresses: Vec<IpAddr> = addresses.into_iter().collect();
|
||||
self.validate_addresses(&policy_host, &addresses)?;
|
||||
|
||||
let pinned = addresses
|
||||
.iter()
|
||||
.map(|ip| match ip {
|
||||
IpAddr::V4(v4) => v4.to_string(),
|
||||
IpAddr::V6(v6) => format!("[{v6}]"),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
Ok(PinnedUpstream {
|
||||
url: url.to_string(),
|
||||
curlopt_resolve: Some(format!("+{connect_host}:{port}:{pinned}")),
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_addresses(&self, host: &str, addresses: &[IpAddr]) -> Result<()> {
|
||||
if addresses.is_empty() || addresses.len() > MAX_PINNED_ADDRESSES {
|
||||
anyhow::bail!(
|
||||
"mirror upstream host must resolve to 1..={MAX_PINNED_ADDRESSES} addresses"
|
||||
);
|
||||
}
|
||||
for &ip in addresses {
|
||||
if !address_is_permitted(ip) {
|
||||
anyhow::bail!("mirror upstream host {host:?} resolved to disallowed address {ip}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_policy_host(host: &str) -> String {
|
||||
host.trim_end_matches('.').to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn reject_always_local_hostname(host: &str) -> Result<()> {
|
||||
if host.is_empty()
|
||||
|| host == "localhost"
|
||||
|| host.ends_with(".localhost")
|
||||
|| host.ends_with(".local")
|
||||
|| host.ends_with(".internal")
|
||||
{
|
||||
anyhow::bail!("mirror upstream host is local-only and cannot be fetched");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn address_is_permitted(ip: IpAddr) -> bool {
|
||||
crate::api::peers::mirror_upstream_ip_permitted(ip)
|
||||
}
|
||||
|
||||
fn parse_git_version(output: &str) -> Option<(u64, u64)> {
|
||||
let version = output.split_whitespace().find(|part| {
|
||||
part.as_bytes()
|
||||
.first()
|
||||
.is_some_and(|byte| byte.is_ascii_digit())
|
||||
})?;
|
||||
let mut parts = version.split('.');
|
||||
Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?))
|
||||
}
|
||||
|
||||
fn ensure_safe_git_runtime(git_bin: &str) -> Result<()> {
|
||||
// Bounded runner like every other git spawn on this path: --version does
|
||||
// no I/O of its own, but the child still takes a permit and the deadline.
|
||||
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
|
||||
git_bin,
|
||||
&["--version"],
|
||||
&std::env::temp_dir(),
|
||||
b"",
|
||||
Instant::now() + Duration::from_secs(10),
|
||||
)?;
|
||||
if !status.success() {
|
||||
anyhow::bail!(
|
||||
"{git_bin} --version failed: {}",
|
||||
String::from_utf8_lossy(&stderr)
|
||||
.chars()
|
||||
.take(256)
|
||||
.collect::<String>()
|
||||
);
|
||||
}
|
||||
let stdout = String::from_utf8_lossy(&stdout);
|
||||
let version = parse_git_version(&stdout).context("parsing git version")?;
|
||||
if version < MIN_CURL_OPT_RESOLVE_GIT {
|
||||
anyhow::bail!(
|
||||
"upstream mirroring requires Git 2.37+ for DNS pinning; found {}.{}",
|
||||
version.0,
|
||||
version.1
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct FetchCommand {
|
||||
config: Vec<(String, String)>,
|
||||
args: Vec<String>,
|
||||
}
|
||||
|
||||
fn fetch_command(upstream: &PinnedUpstream) -> FetchCommand {
|
||||
let http_scope = |name: &str| format!("http.{}.{name}", upstream.url);
|
||||
let credential_scope = |name: &str| format!("credential.{}.{name}", upstream.url);
|
||||
let mut config = vec![
|
||||
("http.proxy".to_string(), "".to_string()),
|
||||
("http.followRedirects".to_string(), "false".to_string()),
|
||||
// Reset any inherited multi-valued resolver entries before adding the
|
||||
// address set validated for this exact fetch.
|
||||
("http.curloptResolve".to_string(), "".to_string()),
|
||||
// Never forward operator or repository-scoped HTTP credentials to an
|
||||
// owner-selected forge. Empty values reset Git's multi-valued headers
|
||||
// and disable cookie state for this invocation.
|
||||
("http.extraHeader".to_string(), "".to_string()),
|
||||
("http.cookieFile".to_string(), "".to_string()),
|
||||
("http.saveCookies".to_string(), "false".to_string()),
|
||||
("http.sslVerify".to_string(), "true".to_string()),
|
||||
// URL-specific configuration outranks generic `http.*` values even
|
||||
// when the generic value is command-scoped. Repeat every security
|
||||
// control at the exact validated URL so a repository-local scoped
|
||||
// setting cannot re-enable redirects/proxies/headers/cookies or disable
|
||||
// TLS. Key/value environment entries avoid `git -c` splitting a valid
|
||||
// URL path that contains `=`.
|
||||
(http_scope("proxy"), "".to_string()),
|
||||
(http_scope("followRedirects"), "false".to_string()),
|
||||
(http_scope("curloptResolve"), "".to_string()),
|
||||
(http_scope("extraHeader"), "".to_string()),
|
||||
(http_scope("cookieFile"), "".to_string()),
|
||||
(http_scope("saveCookies"), "false".to_string()),
|
||||
(http_scope("sslVerify"), "true".to_string()),
|
||||
];
|
||||
if let Some(resolve) = &upstream.curlopt_resolve {
|
||||
config.extend([
|
||||
("http.curloptResolve".to_string(), resolve.clone()),
|
||||
(http_scope("curloptResolve"), resolve.clone()),
|
||||
]);
|
||||
}
|
||||
config.extend([
|
||||
("credential.helper".to_string(), "".to_string()),
|
||||
(credential_scope("helper"), "".to_string()),
|
||||
("credential.interactive".to_string(), "false".to_string()),
|
||||
("core.askPass".to_string(), "false".to_string()),
|
||||
("fetch.recurseSubmodules".to_string(), "false".to_string()),
|
||||
// Do not let a smart server or repository-local config introduce a
|
||||
// second, unvalidated pack/bundle URL outside the pinned upstream.
|
||||
("fetch.uriprotocols".to_string(), "".to_string()),
|
||||
("fetch.bundleURI".to_string(), "".to_string()),
|
||||
("protocol.allow".to_string(), "never".to_string()),
|
||||
("protocol.https.allow".to_string(), "always".to_string()),
|
||||
]);
|
||||
let args = vec![
|
||||
"fetch".to_string(),
|
||||
"--atomic".to_string(),
|
||||
"--force".to_string(),
|
||||
"--prune".to_string(),
|
||||
"--prune-tags".to_string(),
|
||||
"--no-auto-maintenance".to_string(),
|
||||
"--no-recurse-submodules".to_string(),
|
||||
"--no-write-fetch-head".to_string(),
|
||||
upstream.url.clone(),
|
||||
"+refs/heads/*:refs/heads/*".to_string(),
|
||||
"+refs/tags/*:refs/tags/*".to_string(),
|
||||
];
|
||||
FetchCommand { config, args }
|
||||
}
|
||||
|
||||
fn reject_local_url_rewrite(
|
||||
git_bin: &str,
|
||||
repo_path: &Path,
|
||||
upstream_url: &str,
|
||||
deadline: Instant,
|
||||
) -> Result<()> {
|
||||
let args = ["ls-remote", "--get-url", upstream_url];
|
||||
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
|
||||
git_bin, &args, repo_path, b"", deadline,
|
||||
)?;
|
||||
if !status.success() {
|
||||
let stderr = String::from_utf8_lossy(&stderr);
|
||||
anyhow::bail!(
|
||||
"checking mirror upstream URL rewrite failed: {}",
|
||||
stderr.chars().take(4_096).collect::<String>()
|
||||
);
|
||||
}
|
||||
let expanded = std::str::from_utf8(&stdout)
|
||||
.context("Git returned a non-UTF-8 expanded mirror upstream URL")?
|
||||
.trim_end_matches(['\r', '\n']);
|
||||
if expanded != upstream_url {
|
||||
anyhow::bail!("repository-local Git config attempted to rewrite the mirror upstream URL");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reject_sensitive_local_config(git_bin: &str, repo_path: &Path, deadline: Instant) -> Result<()> {
|
||||
let args = [
|
||||
"config",
|
||||
"--local",
|
||||
"--includes",
|
||||
"--get-regexp",
|
||||
r"^(http|credential|url)\.",
|
||||
];
|
||||
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
|
||||
git_bin, &args, repo_path, b"", deadline,
|
||||
)?;
|
||||
match status.code() {
|
||||
Some(1) if stdout.is_empty() => Ok(()),
|
||||
Some(0) => anyhow::bail!(
|
||||
"repository-local network or credential Git config is not allowed for upstream mirrors"
|
||||
),
|
||||
_ => {
|
||||
let stderr = String::from_utf8_lossy(&stderr);
|
||||
anyhow::bail!(
|
||||
"checking repository-local Git config failed: {}",
|
||||
stderr.chars().take(4_096).collect::<String>()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Budget for each preflight probe inside [`run_fetch`] (the URL-rewrite
|
||||
/// check and the sensitive-local-config scan). Both run local git children
|
||||
/// against the on-disk repo, so a short bound is generous; keeping them on
|
||||
/// their own deadlines means `upstream_mirror_fetch_timeout_secs` covers
|
||||
/// only the external pack transfer it documents.
|
||||
const PREFLIGHT_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
fn run_fetch(
|
||||
git_bin: &str,
|
||||
repo_path: &Path,
|
||||
upstream: &PinnedUpstream,
|
||||
timeout: Duration,
|
||||
) -> Result<()> {
|
||||
let command = fetch_command(upstream);
|
||||
let args: Vec<&str> = command.args.iter().map(String::as_str).collect();
|
||||
let config: Vec<(&str, &str)> = command
|
||||
.config
|
||||
.iter()
|
||||
.map(|(key, value)| (key.as_str(), value.as_str()))
|
||||
.collect();
|
||||
let rewrite_deadline = Instant::now()
|
||||
.checked_add(PREFLIGHT_PROBE_TIMEOUT)
|
||||
.context("mirror URL-rewrite probe timeout is not representable")?;
|
||||
reject_local_url_rewrite(git_bin, repo_path, &upstream.url, rewrite_deadline)?;
|
||||
let config_deadline = Instant::now()
|
||||
.checked_add(PREFLIGHT_PROBE_TIMEOUT)
|
||||
.context("mirror local-config probe timeout is not representable")?;
|
||||
reject_sensitive_local_config(git_bin, repo_path, config_deadline)?;
|
||||
let deadline = Instant::now()
|
||||
.checked_add(timeout)
|
||||
.context("upstream mirror fetch timeout is not representable")?;
|
||||
let (status, _stdout, stderr) =
|
||||
crate::git::visibility_pack::run_bounded_git_raw_isolated_https_with_config(
|
||||
git_bin, &args, repo_path, b"", deadline, &config,
|
||||
)?;
|
||||
if !status.success() {
|
||||
let stderr = String::from_utf8_lossy(&stderr);
|
||||
let stderr = stderr.chars().take(4_096).collect::<String>();
|
||||
anyhow::bail!("git upstream fetch failed: {stderr}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Worker {
|
||||
db: Arc<Db>,
|
||||
config: Arc<Config>,
|
||||
repo_store: RepoStore,
|
||||
repo_write_leases: RepoWriteLeases,
|
||||
git_write_semaphore: Arc<tokio::sync::Semaphore>,
|
||||
policy: EgressPolicy,
|
||||
git_bin: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum FetchOutcome {
|
||||
Updated,
|
||||
SkippedStale,
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
async fn fetch_target(&self, target: InboundMirrorTarget) -> Result<FetchOutcome> {
|
||||
let url = crate::db::validate_mirror_upstream_url(&target.upstream_url)?;
|
||||
let upstream = self.policy.resolve(url).await?;
|
||||
|
||||
let repo_identity = repo_identity_key(&target.owner_did, &target.name);
|
||||
let steal_after = Duration::from_secs(
|
||||
self.config
|
||||
.upstream_mirror_fetch_timeout_secs
|
||||
.saturating_mul(2)
|
||||
.saturating_add(60),
|
||||
);
|
||||
let _lease = self
|
||||
.repo_write_leases
|
||||
.acquire(&repo_identity, steal_after)
|
||||
.await
|
||||
.context("upstream mirror write-lease waiter cap reached")?;
|
||||
|
||||
let _write_permit = Arc::clone(&self.git_write_semaphore)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.context("upstream mirror write semaphore closed")?;
|
||||
let acquire_timeout = Duration::from_secs(self.config.git_acquire_timeout_secs);
|
||||
let guard = tokio::time::timeout(
|
||||
acquire_timeout,
|
||||
self.repo_store
|
||||
.acquire_write(&target.owner_did, &target.name),
|
||||
)
|
||||
.await
|
||||
.context("upstream mirror write-lock acquisition timed out")??;
|
||||
let repo_path = guard.path().to_path_buf();
|
||||
|
||||
// A row can leave INBOUND state after selection. Recheck only after
|
||||
// both the process-local lease and the cluster-wide advisory write lock
|
||||
// are held. Every future transition path must take these same locks
|
||||
// before changing authority, so no other node can commit a transition
|
||||
// between this check and the fetch.
|
||||
let current = match self.db.get_repo_mirror_state(&target.repo_id).await {
|
||||
Ok(Some(current)) => current,
|
||||
Ok(None) => {
|
||||
let _ = guard.release(false).await;
|
||||
anyhow::bail!("inbound mirror state disappeared before fetch");
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = guard.release(false).await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
// Both stable shapes the CHECK admits: never transitioned, or a
|
||||
// completed transition whose metadata is retained as audit. Only an
|
||||
// in-flight transition makes the target stale.
|
||||
let stable = (current.transition_id.is_none() && current.transition_phase.is_none())
|
||||
|| (current.transition_id.is_some()
|
||||
&& current.transition_phase == Some(MirrorTransitionPhase::Completed));
|
||||
if current.status != MirrorStatus::Inbound
|
||||
|| !stable
|
||||
|| current.upstream_url != target.upstream_url
|
||||
{
|
||||
let _ = guard.release(false).await;
|
||||
info!(repo_id = %target.repo_id, "upstream mirror state changed before fetch; skipping stale target");
|
||||
return Ok(FetchOutcome::SkippedStale);
|
||||
}
|
||||
|
||||
let git_bin = self.git_bin.clone();
|
||||
let fetch_timeout = Duration::from_secs(self.config.upstream_mirror_fetch_timeout_secs);
|
||||
let fetch = tokio::task::spawn_blocking(move || {
|
||||
run_fetch(&git_bin, &repo_path, &upstream, fetch_timeout)
|
||||
})
|
||||
.await
|
||||
.context("upstream mirror fetch task panicked")?;
|
||||
let success = fetch.is_ok();
|
||||
let outcome = guard.release(success).await;
|
||||
// A refused publish (fenced or unknowable upload) must not report
|
||||
// Updated; surface it as the failure it is before mapping the fetch.
|
||||
outcome.into_result()?;
|
||||
fetch.map(|()| FetchOutcome::Updated)
|
||||
}
|
||||
|
||||
async fn scan_once(&self) {
|
||||
let mut cursor: Option<String> = None;
|
||||
loop {
|
||||
let targets = match self
|
||||
.db
|
||||
.list_inbound_mirror_targets(
|
||||
cursor.as_deref(),
|
||||
self.config.upstream_mirror_page_size as i64,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(targets) => targets,
|
||||
Err(error) => {
|
||||
warn!(err = %error, "failed to list inbound mirror targets");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if targets.is_empty() {
|
||||
return;
|
||||
}
|
||||
let page_len = targets.len();
|
||||
cursor = targets.last().map(|target| target.repo_id.clone());
|
||||
for target in targets {
|
||||
let repo_id = target.repo_id.clone();
|
||||
let upstream_url = target.upstream_url.clone();
|
||||
match self.fetch_target(target).await {
|
||||
Ok(FetchOutcome::Updated) => {
|
||||
info!(repo_id, upstream = %upstream_url, "upstream mirror fetch completed")
|
||||
}
|
||||
Ok(FetchOutcome::SkippedStale) => {}
|
||||
Err(error) => {
|
||||
warn!(repo_id, upstream = %upstream_url, err = %error, "upstream mirror fetch failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
if page_len < self.config.upstream_mirror_page_size {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the opt-in runtime and spawn the scheduled worker. The worker runs
|
||||
/// one scan immediately, sleeps only after a complete scan, and exits between
|
||||
/// scans when graceful shutdown is signalled. An in-flight Git child is bounded
|
||||
/// and reaped before the worker observes shutdown.
|
||||
pub fn start(
|
||||
db: Arc<Db>,
|
||||
config: Arc<Config>,
|
||||
repo_store: RepoStore,
|
||||
repo_write_leases: RepoWriteLeases,
|
||||
git_write_semaphore: Arc<tokio::sync::Semaphore>,
|
||||
mut shutdown_rx: tokio::sync::watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
ensure_safe_git_runtime("git")?;
|
||||
let policy = EgressPolicy;
|
||||
let worker = Worker {
|
||||
db,
|
||||
config: Arc::clone(&config),
|
||||
repo_store,
|
||||
repo_write_leases,
|
||||
git_write_semaphore,
|
||||
policy,
|
||||
git_bin: "git".to_string(),
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
info!(
|
||||
interval_secs = config.upstream_mirror_interval_secs,
|
||||
"upstream mirror worker started"
|
||||
);
|
||||
loop {
|
||||
worker.scan_once().await;
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(config.upstream_mirror_interval_secs)) => {}
|
||||
changed = shutdown_rx.changed() => {
|
||||
if changed.is_err() || *shutdown_rx.borrow() {
|
||||
info!("upstream mirror worker stopped");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
|
||||
async fn test_worker(
|
||||
pool: sqlx::PgPool,
|
||||
git_bin: PathBuf,
|
||||
page_size: usize,
|
||||
) -> (Worker, Arc<Db>, tempfile::TempDir) {
|
||||
let db = Arc::new(Db::for_testing(pool.clone()));
|
||||
db.run_migrations().await.unwrap();
|
||||
let repos = tempfile::tempdir().unwrap();
|
||||
let mut config = Config::parse_from([
|
||||
"gitlawb-node".to_string(),
|
||||
"--upstream-mirror-enabled".to_string(),
|
||||
"--enforce-owner-push".to_string(),
|
||||
"--upstream-mirror-fetch-timeout-secs".to_string(),
|
||||
"5".to_string(),
|
||||
"--upstream-mirror-page-size".to_string(),
|
||||
page_size.to_string(),
|
||||
]);
|
||||
config.validate().unwrap();
|
||||
let config = Arc::new(config);
|
||||
let worker = Worker {
|
||||
db: Arc::clone(&db),
|
||||
config,
|
||||
repo_store: RepoStore::for_testing(repos.path().to_path_buf(), pool),
|
||||
repo_write_leases: RepoWriteLeases::new(8),
|
||||
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(4)),
|
||||
policy: EgressPolicy,
|
||||
git_bin: git_bin.to_string_lossy().into_owned(),
|
||||
};
|
||||
(worker, db, repos)
|
||||
}
|
||||
|
||||
async fn add_inbound_mirror(
|
||||
worker: &Worker,
|
||||
db: &Db,
|
||||
id: &str,
|
||||
name: &str,
|
||||
upstream_url: &str,
|
||||
) -> InboundMirrorTarget {
|
||||
let now = chrono::Utc::now();
|
||||
let record = crate::db::RepoRecord {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
owner_did: "did:key:z6MkMirrorWorker".to_string(),
|
||||
description: None,
|
||||
is_public: true,
|
||||
default_branch: "main".to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
disk_path: format!("/test/{id}.git"),
|
||||
forked_from: None,
|
||||
machine_id: None,
|
||||
};
|
||||
db.create_repo(&record).await.unwrap();
|
||||
worker
|
||||
.repo_store
|
||||
.init(&record.owner_did, &record.name)
|
||||
.await
|
||||
.unwrap();
|
||||
let state = db
|
||||
.configure_inbound_mirror(&record.id, upstream_url)
|
||||
.await
|
||||
.unwrap();
|
||||
InboundMirrorTarget {
|
||||
repo_id: record.id,
|
||||
owner_did: record.owner_did,
|
||||
name: record.name,
|
||||
upstream_url: state.upstream_url,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn fake_git(repos: &tempfile::TempDir) -> (PathBuf, PathBuf) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let git = repos.path().join("fake-git");
|
||||
let calls = repos.path().join("fetch-calls");
|
||||
let script = format!(
|
||||
r#"#!/bin/sh
|
||||
if [ "$1" = "ls-remote" ]; then
|
||||
printf '%s\n' "$3"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "config" ]; then
|
||||
# Match real Git when no sensitive repository-local keys are configured.
|
||||
exit 1
|
||||
fi
|
||||
case "$*" in
|
||||
*fail.git*) printf '%s\n' fail >> '{}'; exit 7 ;;
|
||||
*) printf '%s\n' ok >> '{}'; exit 0 ;;
|
||||
esac
|
||||
"#,
|
||||
calls.display(),
|
||||
calls.display()
|
||||
);
|
||||
std::fs::write(&git, script).unwrap();
|
||||
let mut permissions = std::fs::metadata(&git).unwrap().permissions();
|
||||
permissions.set_mode(0o700);
|
||||
std::fs::set_permissions(&git, permissions).unwrap();
|
||||
(git, calls)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn git_version_gate_requires_curlopt_resolve_support() {
|
||||
assert_eq!(parse_git_version("git version 2.37.0"), Some((2, 37)));
|
||||
assert_eq!(
|
||||
parse_git_version("git version 2.39.5 (Apple Git-154)"),
|
||||
Some((2, 39))
|
||||
);
|
||||
assert_eq!(parse_git_version("not git"), None);
|
||||
assert!((2, 36) < MIN_CURL_OPT_RESOLVE_GIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_resolved_address_must_be_public() {
|
||||
let policy = EgressPolicy;
|
||||
let public_v4: IpAddr = "8.8.8.8".parse().unwrap();
|
||||
let private_v4: IpAddr = "10.2.3.4".parse().unwrap();
|
||||
let private_v6: IpAddr = "fd00::20".parse().unwrap();
|
||||
assert!(policy
|
||||
.validate_addresses("public.example", &[public_v4])
|
||||
.is_ok());
|
||||
assert!(policy
|
||||
.validate_addresses("private.example", &[private_v4, private_v6])
|
||||
.is_err());
|
||||
assert!(policy
|
||||
.validate_addresses("mixed.example", &[public_v4, private_v4])
|
||||
.is_err());
|
||||
let too_many = (1..=MAX_PINNED_ADDRESSES + 1)
|
||||
.map(|last| IpAddr::V4(std::net::Ipv4Addr::new(8, 8, 0, last as u8)))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(policy
|
||||
.validate_addresses("oversized.example", &too_many)
|
||||
.is_err());
|
||||
|
||||
for never in [
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"192.0.2.1",
|
||||
"198.18.0.1",
|
||||
"198.51.100.1",
|
||||
"203.0.113.10",
|
||||
"::1",
|
||||
"::ffff:198.18.0.1",
|
||||
"2002:c612:1::",
|
||||
"64:ff9b::c612:1",
|
||||
"100::1",
|
||||
"100:0:0:1::1",
|
||||
"2001::1",
|
||||
"2001:2::1",
|
||||
"2001:db8::1",
|
||||
"fec0::1",
|
||||
"fe80::1",
|
||||
"3fff::1",
|
||||
"5f00::1",
|
||||
] {
|
||||
let ip = never.parse().unwrap();
|
||||
assert!(!address_is_permitted(ip), "{never} must stay denied");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_only_hostnames_are_always_rejected() {
|
||||
for invalid in [
|
||||
"",
|
||||
"localhost",
|
||||
"forge.localhost",
|
||||
"forge.local",
|
||||
"ghe.internal",
|
||||
] {
|
||||
assert!(
|
||||
reject_always_local_hostname(invalid).is_err(),
|
||||
"{invalid:?}"
|
||||
);
|
||||
}
|
||||
assert!(reject_always_local_hostname("github.example").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_command_pins_dns_and_only_updates_branches_and_tags() {
|
||||
let upstream = PinnedUpstream {
|
||||
url: "https://github.example/org/repo.git".to_string(),
|
||||
curlopt_resolve: Some("+github.example:443:8.8.8.8,[2606:4700:4700::1111]".to_string()),
|
||||
};
|
||||
let command = fetch_command(&upstream);
|
||||
let has_config = |key: &str, value: &str| {
|
||||
command
|
||||
.config
|
||||
.contains(&(key.to_string(), value.to_string()))
|
||||
};
|
||||
assert!(has_config("http.followRedirects", "false"));
|
||||
assert!(has_config("http.proxy", ""));
|
||||
assert!(has_config("http.extraHeader", ""));
|
||||
assert!(has_config("http.cookieFile", ""));
|
||||
assert!(has_config("http.saveCookies", "false"));
|
||||
assert!(has_config("http.sslVerify", "true"));
|
||||
assert!(has_config(
|
||||
"http.curloptResolve",
|
||||
"+github.example:443:8.8.8.8,[2606:4700:4700::1111]"
|
||||
));
|
||||
assert!(has_config(
|
||||
"http.https://github.example/org/repo.git.followRedirects",
|
||||
"false"
|
||||
));
|
||||
assert!(has_config(
|
||||
"http.https://github.example/org/repo.git.proxy",
|
||||
""
|
||||
));
|
||||
assert!(has_config(
|
||||
"http.https://github.example/org/repo.git.extraHeader",
|
||||
""
|
||||
));
|
||||
assert!(has_config(
|
||||
"http.https://github.example/org/repo.git.curloptResolve",
|
||||
"+github.example:443:8.8.8.8,[2606:4700:4700::1111]"
|
||||
));
|
||||
assert!(has_config(
|
||||
"credential.https://github.example/org/repo.git.helper",
|
||||
""
|
||||
));
|
||||
assert!(has_config("protocol.allow", "never"));
|
||||
assert!(has_config("protocol.https.allow", "always"));
|
||||
assert!(has_config("fetch.uriprotocols", ""));
|
||||
assert!(has_config("fetch.bundleURI", ""));
|
||||
assert!(!command
|
||||
.config
|
||||
.iter()
|
||||
.any(|(key, _)| key.ends_with("sslCert") || key.ends_with("sslKey")));
|
||||
assert!(command.args.contains(&"--atomic".to_string()));
|
||||
assert!(command
|
||||
.args
|
||||
.contains(&"+refs/heads/*:refs/heads/*".to_string()));
|
||||
assert!(command
|
||||
.args
|
||||
.contains(&"+refs/tags/*:refs/tags/*".to_string()));
|
||||
assert!(!command.args.iter().any(|arg| arg == "+refs/*:refs/*"));
|
||||
}
|
||||
|
||||
/// Spawn `git` inside a fixture repo with inherited repository-location
|
||||
/// and config environment scrubbed, so a test run inside an outer Git
|
||||
/// checkout or custom config can never leak into the fixture.
|
||||
fn fixture_git(dir: &std::path::Path) -> std::process::Command {
|
||||
let mut cmd = std::process::Command::new("git");
|
||||
cmd.current_dir(dir);
|
||||
for var in [
|
||||
"GIT_DIR",
|
||||
"GIT_WORK_TREE",
|
||||
"GIT_CONFIG",
|
||||
"GIT_CONFIG_GLOBAL",
|
||||
"GIT_CONFIG_SYSTEM",
|
||||
"GIT_CONFIG_NOSYSTEM",
|
||||
"GIT_CONFIG_COUNT",
|
||||
"GIT_EXEC_PATH",
|
||||
"GIT_SSH",
|
||||
"GIT_SSH_COMMAND",
|
||||
"GIT_ASKPASS",
|
||||
"SSH_ASKPASS",
|
||||
] {
|
||||
cmd.env_remove(var);
|
||||
}
|
||||
cmd
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_local_instead_of_rewrite_is_rejected() {
|
||||
let repo = tempfile::tempdir().unwrap();
|
||||
let init = fixture_git(repo.path())
|
||||
.args(["init", "--bare"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(init.status.success());
|
||||
let config = fixture_git(repo.path())
|
||||
.args([
|
||||
"config",
|
||||
"url.https://evil.example/.insteadOf",
|
||||
"https://github.example/",
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(config.status.success());
|
||||
|
||||
let error = reject_local_url_rewrite(
|
||||
"git",
|
||||
repo.path(),
|
||||
"https://github.example/org/repo.git",
|
||||
Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("attempted to rewrite"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_local_network_and_credential_config_is_rejected() {
|
||||
let repo = tempfile::tempdir().unwrap();
|
||||
let init = fixture_git(repo.path())
|
||||
.args(["init", "--bare"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(init.status.success());
|
||||
|
||||
reject_sensitive_local_config("git", repo.path(), Instant::now() + Duration::from_secs(5))
|
||||
.unwrap();
|
||||
|
||||
let config = fixture_git(repo.path())
|
||||
.args(["config", "http.sslCert", "/tmp/untrusted-client.pem"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(config.status.success());
|
||||
|
||||
let error = reject_sensitive_local_config(
|
||||
"git",
|
||||
repo.path(),
|
||||
Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("network or credential"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn hardened_real_git_reaches_https_transport_for_equals_path() {
|
||||
let repo = tempfile::tempdir().unwrap();
|
||||
let init = fixture_git(repo.path())
|
||||
.args(["init", "--bare"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(init.status.success());
|
||||
|
||||
// This is intentionally a loopback test seam around `run_fetch`, below
|
||||
// the egress-policy layer. Reaching the listener proves the generated
|
||||
// Git config neither dies on an empty client certificate nor splits the
|
||||
// valid `=` path at the command-config boundary. The listener closes
|
||||
// immediately, so the expected final result is a TLS transport error.
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let accepted = std::thread::spawn(move || {
|
||||
let deadline = Instant::now() + Duration::from_secs(3);
|
||||
while Instant::now() < deadline {
|
||||
match listener.accept() {
|
||||
Ok((_stream, _address)) => return true,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Err(error) => panic!("accepting real-Git transport probe: {error}"),
|
||||
}
|
||||
}
|
||||
false
|
||||
});
|
||||
let upstream = PinnedUpstream {
|
||||
url: format!("https://127.0.0.1:{port}/org/repo=mirror.git"),
|
||||
curlopt_resolve: None,
|
||||
};
|
||||
let error = run_fetch("git", repo.path(), &upstream, Duration::from_secs(5)).unwrap_err();
|
||||
assert!(
|
||||
accepted.join().unwrap(),
|
||||
"real Git must reach the HTTPS transport; got {error:#}"
|
||||
);
|
||||
let error = error.to_string();
|
||||
assert!(!error.contains("credential missing host field"));
|
||||
assert!(!error.contains("invalid key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn literal_addresses_are_checked_without_dns() {
|
||||
let policy = EgressPolicy;
|
||||
let allowed = policy
|
||||
.resolve(reqwest::Url::parse("https://8.8.8.8/org/repo.git").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(allowed.curlopt_resolve, None);
|
||||
|
||||
for denied in [
|
||||
"127.0.0.1",
|
||||
"10.2.3.4",
|
||||
"169.254.169.254",
|
||||
"192.0.2.1",
|
||||
"198.18.0.1",
|
||||
"203.0.113.10",
|
||||
"[::1]",
|
||||
"[100::1]",
|
||||
"[100:0:0:1::1]",
|
||||
"[2001::1]",
|
||||
"[2001:db8::1]",
|
||||
"[fec0::1]",
|
||||
"[fd00::20]",
|
||||
"[3fff::1]",
|
||||
"[5f00::1]",
|
||||
] {
|
||||
let url = reqwest::Url::parse(&format!("https://{denied}/org/repo.git")).unwrap();
|
||||
assert!(
|
||||
policy.resolve(url).await.is_err(),
|
||||
"{denied} must stay denied"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test]
|
||||
async fn stale_target_is_rechecked_under_the_write_locks(pool: sqlx::PgPool) {
|
||||
let missing_git = PathBuf::from("/definitely/missing/git");
|
||||
let (worker, db, _repos) = test_worker(pool, missing_git, 1).await;
|
||||
let target = add_inbound_mirror(
|
||||
&worker,
|
||||
&db,
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
"stale",
|
||||
"https://8.8.8.8/org/stale.git",
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"UPDATE repos
|
||||
SET mirror_status = 'outbound'
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(&target.repo_id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let outcome = worker.fetch_target(target).await.unwrap();
|
||||
assert_eq!(outcome, FetchOutcome::SkippedStale);
|
||||
assert!(worker.repo_write_leases.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[sqlx::test]
|
||||
async fn scan_pages_past_one_failure_and_fetches_the_next_repo(pool: sqlx::PgPool) {
|
||||
let bootstrap = tempfile::tempdir().unwrap();
|
||||
let (git, calls) = fake_git(&bootstrap);
|
||||
let (worker, db, _repos) = test_worker(pool, git, 1).await;
|
||||
add_inbound_mirror(
|
||||
&worker,
|
||||
&db,
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
"fails",
|
||||
"https://8.8.8.8/org/fail.git",
|
||||
)
|
||||
.await;
|
||||
add_inbound_mirror(
|
||||
&worker,
|
||||
&db,
|
||||
"00000000-0000-0000-0000-000000000002",
|
||||
"succeeds",
|
||||
"https://8.8.8.8/org/ok.git",
|
||||
)
|
||||
.await;
|
||||
|
||||
worker.scan_once().await;
|
||||
|
||||
let calls = std::fs::read_to_string(calls).unwrap();
|
||||
assert_eq!(calls.lines().collect::<Vec<_>>(), ["fail", "ok"]);
|
||||
assert!(worker.repo_write_leases.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -38,151 +38,26 @@ pub fn fire_event(
|
||||
event: &str,
|
||||
payload: serde_json::Value,
|
||||
) {
|
||||
fire_event_occurrence(db, http_client, repo_id, event, payload, None, None);
|
||||
let repo_id = repo_id.to_string();
|
||||
let event = event.to_string();
|
||||
tokio::spawn(async move {
|
||||
fire_event_async(db, http_client, &repo_id, &event, payload).await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Occurrence-keyed variant: at most one delivery per
|
||||
/// `(request_id, ref_name, hook)` is spawned; concurrent executors
|
||||
/// collapse via the `webhook_deliveries` ledger. Best-effort on crash
|
||||
/// (ledger is claimed before spawn), which the PR scope allows.
|
||||
///
|
||||
/// For durable outbox usage, call `claim_webhook_delivery_before_spawn` first
|
||||
/// to claim deliveries synchronously, then call
|
||||
/// `fire_event_occurrence_with_claimed_hooks` with the claimed hook IDs to
|
||||
/// spawn HTTP POST only for claimed hooks.
|
||||
pub fn fire_event_occurrence(
|
||||
async fn fire_event_async(
|
||||
db: Arc<Db>,
|
||||
http_client: Arc<reqwest::Client>,
|
||||
repo_id: &str,
|
||||
event: &str,
|
||||
payload: serde_json::Value,
|
||||
request_id: Option<&str>,
|
||||
ref_name: Option<&str>,
|
||||
) {
|
||||
let repo_id = repo_id.to_string();
|
||||
let event = event.to_string();
|
||||
let request_id = request_id.map(|s| s.to_string());
|
||||
let ref_name = ref_name.map(|s| s.to_string());
|
||||
tokio::spawn(async move {
|
||||
fire_event_async_occurrence(
|
||||
db,
|
||||
http_client,
|
||||
&repo_id,
|
||||
&event,
|
||||
payload,
|
||||
request_id.as_deref(),
|
||||
ref_name.as_deref(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Variant for durable outbox: only spawn HTTP POST for previously claimed hooks.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn fire_event_occurrence_with_claimed_hooks(
|
||||
db: Arc<Db>,
|
||||
http_client: Arc<reqwest::Client>,
|
||||
repo_id: &str,
|
||||
event: &str,
|
||||
payload: serde_json::Value,
|
||||
request_id: Option<&str>,
|
||||
ref_name: Option<&str>,
|
||||
claimed_hooks: Vec<crate::db::Webhook>,
|
||||
) {
|
||||
let repo_id = repo_id.to_string();
|
||||
let event = event.to_string();
|
||||
let request_id = request_id.map(|s| s.to_string());
|
||||
let ref_name = ref_name.map(|s| s.to_string());
|
||||
tokio::spawn(async move {
|
||||
fire_event_async_occurrence(
|
||||
db,
|
||||
http_client,
|
||||
&repo_id,
|
||||
&event,
|
||||
payload,
|
||||
request_id.as_deref(),
|
||||
ref_name.as_deref(),
|
||||
Some(claimed_hooks),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Claim webhook delivery before spawning HTTP POST.
|
||||
/// Returns `Some(hook ids)` successfully claimed, or `None` when the hook
|
||||
/// list itself could not be read (transient DB error, unknown membership).
|
||||
/// A `None` caller must fall back to best-effort send rather than dropping
|
||||
/// the delivery: children are still deleted downstream, so an empty claim
|
||||
/// set on DB error would lose the webhook permanently.
|
||||
/// This synchronous claim phase must be awaited before child deletion to prevent
|
||||
/// permanent webhook loss on crash between deletion and claim.
|
||||
pub async fn claim_webhook_delivery_before_spawn(
|
||||
db: Arc<Db>,
|
||||
repo_id: &str,
|
||||
event: &str,
|
||||
request_id: Option<&str>,
|
||||
ref_name: Option<&str>,
|
||||
) -> Option<Vec<crate::db::Webhook>> {
|
||||
let mut claimed_hooks = Vec::new();
|
||||
|
||||
// Only claim when we have occurrence context (request_id + ref_name)
|
||||
if let (Some(req), Some(r)) = (request_id, ref_name) {
|
||||
let hooks = match db.list_webhooks_for_event(repo_id, event).await {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::warn!(err = %e, "failed to list webhooks for event {event}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
for hook in hooks {
|
||||
let delivery_id = crate::db::deterministic_id(&["webhook", req, r, &hook.id]);
|
||||
match db
|
||||
.claim_webhook_delivery(&delivery_id, req, repo_id, event)
|
||||
.await
|
||||
{
|
||||
Ok(true) => claimed_hooks.push(hook),
|
||||
Ok(false) => continue, // Already claimed
|
||||
// Send fallback (legacy best-effort): a transient claim
|
||||
// error must not drop the delivery while children are
|
||||
// still deleted. The in-task claimer this replaced sent
|
||||
// anyway on claim error; keep that here.
|
||||
Err(e) => {
|
||||
tracing::warn!(err = %e, hook_id = %hook.id, "webhook claim failed; sending anyway");
|
||||
claimed_hooks.push(hook);
|
||||
}
|
||||
}
|
||||
let hooks = match db.list_webhooks_for_event(repo_id, event).await {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::warn!(err = %e, "failed to list webhooks for event {event}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Some(claimed_hooks)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn fire_event_async_occurrence(
|
||||
db: Arc<Db>,
|
||||
http_client: Arc<reqwest::Client>,
|
||||
repo_id: &str,
|
||||
event: &str,
|
||||
payload: serde_json::Value,
|
||||
request_id: Option<&str>,
|
||||
ref_name: Option<&str>,
|
||||
claimed_hooks: Option<Vec<crate::db::Webhook>>,
|
||||
) {
|
||||
// A caller that already claimed deliveries hands the claimed hook rows
|
||||
// through; the send path must not re-list, because a list failure here
|
||||
// would silently drop deliveries already claimed in the ledger.
|
||||
let already_claimed = claimed_hooks.is_some();
|
||||
let hooks = match claimed_hooks {
|
||||
Some(h) => h,
|
||||
None => match db.list_webhooks_for_event(repo_id, event).await {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::warn!(err = %e, "failed to list webhooks for event {event}");
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
if hooks.is_empty() {
|
||||
@@ -201,29 +76,8 @@ async fn fire_event_async_occurrence(
|
||||
let client = Arc::clone(&http_client);
|
||||
let event_name = event.to_string();
|
||||
let bytes = payload_bytes.clone();
|
||||
// Stable occurrence delivery key when request context exists;
|
||||
// random UUID otherwise (legacy callers without occurrence).
|
||||
let delivery_id = match (request_id, ref_name) {
|
||||
(Some(req), Some(r)) => crate::db::deterministic_id(&["webhook", req, r, &hook.id]),
|
||||
_ => uuid::Uuid::new_v4().to_string(),
|
||||
};
|
||||
// Claim before spawn so concurrent executors collapse to one
|
||||
// delivery per occurrence. Failures to record fall back to
|
||||
// sending (legacy best-effort) rather than dropping.
|
||||
// Skip claim if already done by caller (durable_outbox path).
|
||||
if !already_claimed && request_id.is_some() {
|
||||
match db
|
||||
.clone()
|
||||
.claim_webhook_delivery(&delivery_id, request_id.unwrap_or(""), repo_id, event)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {}
|
||||
Ok(false) => continue,
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
let delivery_id = uuid::Uuid::new_v4().to_string();
|
||||
|
||||
let db_for_mark = db.clone();
|
||||
tokio::spawn(async move {
|
||||
// Policy skip, not a delivery failure: the stored URL predates the
|
||||
// create-time gate. The "invalid_url" label is documented on
|
||||
@@ -238,7 +92,6 @@ async fn fire_event_async_occurrence(
|
||||
return;
|
||||
}
|
||||
|
||||
let delivery_for_mark = delivery_id.clone();
|
||||
let signature = hook.secret.as_deref().map(|s| sign_payload(s, &bytes));
|
||||
|
||||
let mut req = client
|
||||
@@ -260,10 +113,6 @@ async fn fire_event_async_occurrence(
|
||||
"http_error"
|
||||
};
|
||||
crate::metrics::record_webhook_delivery(result_label);
|
||||
// Any HTTP response proves the delivery fired; mark
|
||||
// sent so the ledger never claims an unsent delivery.
|
||||
// Network errors stay pending for stale-reclaim.
|
||||
let _ = db_for_mark.mark_webhook_sent(&delivery_for_mark).await;
|
||||
tracing::info!(
|
||||
url = %hook.url,
|
||||
event = %event_name,
|
||||
|
||||
@@ -416,8 +416,8 @@ fn f3_second_writer_leased_until_reap() {
|
||||
NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).",
|
||||
);
|
||||
let receive = repos_production
|
||||
.find("smart_http::receive_pack_raw")
|
||||
.expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack_raw");
|
||||
.find("smart_http::receive_pack(")
|
||||
.expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack");
|
||||
assert!(
|
||||
lease_acquire < with_lease && with_lease < receive,
|
||||
"F3 gate bypassed: the write lease must be acquired, then carried by the \
|
||||
@@ -509,8 +509,8 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() {
|
||||
/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing
|
||||
/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be
|
||||
/// what gates it, so the success check is explicit and this gate binds it: the spawn
|
||||
/// must sit inside `if disposition.spawn_tail`, and `release` must consume the same
|
||||
/// disposition (`disposition.release_ok`) so the two cannot drift apart.
|
||||
/// must sit inside `if push_succeeded`, and `release` must consume the same flag so
|
||||
/// the two cannot drift apart.
|
||||
///
|
||||
/// This is an ordering check rather than a cancellation-race test on purpose: it is
|
||||
/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which
|
||||
@@ -519,9 +519,7 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() {
|
||||
///
|
||||
/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below
|
||||
/// `guard.release(` and the ordering assertion fails; take it out of the
|
||||
/// `if disposition.spawn_tail` block and the failed-push assertion fails; gate
|
||||
/// `spawn_tail`/`release_ok` on the outcome commit and
|
||||
/// `post_git_disposition_replication_carve_out` fails with it.
|
||||
/// `if push_succeeded` block and the failed-push assertion fails.
|
||||
#[test]
|
||||
fn inv22_replication_tail_spawns_at_the_durability_boundary() {
|
||||
let repos = src("api/repos.rs");
|
||||
@@ -536,589 +534,41 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() {
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
let success_flag = production
|
||||
.find("post_git_disposition(")
|
||||
.expect("U5 gate missing: the tail's disposition must be bound before the gate");
|
||||
// One tail per push, on the proven-landed ref set. Either spawn
|
||||
// position (carve-out early, or durability-bound at release) must sit
|
||||
// inside a `disposition.spawn_tail` gate and feed `landed_refs` —
|
||||
// never the unfiltered `ref_updates`.
|
||||
let spawn_count = production
|
||||
.matches("tokio::spawn(post_receive_replication_tail(")
|
||||
.count();
|
||||
assert!(
|
||||
spawn_count >= 1,
|
||||
"U5 gate missing: the replication tail must be spawned by git_receive_pack"
|
||||
);
|
||||
for (i, (pos, _)) in production
|
||||
.match_indices("tokio::spawn(post_receive_replication_tail(")
|
||||
.enumerate()
|
||||
{
|
||||
let gate = production[..pos].rfind("if disposition.spawn_tail").expect(
|
||||
"U5 gate missing: every tail spawn must be gated on `if disposition.spawn_tail`",
|
||||
.find("let push_succeeded = receive_result.is_ok();")
|
||||
.expect(
|
||||
"U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()",
|
||||
);
|
||||
// The spawn must sit inside the gate's block: brace depth at the
|
||||
// spawn position must still be positive measured from the `if`.
|
||||
// An ungated spawn after the block closes fails this, as does a
|
||||
// stray `disposition.spawn_tail` mention in a comment.
|
||||
let mut depth = 0i32;
|
||||
for ch in production[gate..pos].chars() {
|
||||
match ch {
|
||||
'{' => depth += 1,
|
||||
'}' => depth -= 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
depth >= 1,
|
||||
"U5 gate bypassed: tail #{i} sits outside the `if disposition.spawn_tail` block"
|
||||
);
|
||||
assert!(
|
||||
!production[gate..pos].contains("post_receive_replication_tail"),
|
||||
"U5 gate bypassed: a second tail spawn sits between the gate and its spawn"
|
||||
);
|
||||
let call_end = production[pos..].find(");").unwrap() + pos;
|
||||
let call = &production[pos..call_end];
|
||||
assert!(
|
||||
call.contains("landed_refs"),
|
||||
"U5 gate bypassed: tail #{i} must carry the proven-landed ref set, not unfiltered ref_updates"
|
||||
);
|
||||
}
|
||||
let gate_open = production
|
||||
.find("if push_succeeded {")
|
||||
.expect("U5 gate missing: the tail spawn must be gated on the push having succeeded");
|
||||
let spawn = production
|
||||
.find("tokio::spawn(post_receive_replication_tail(")
|
||||
.expect("U5 gate missing: the replication tail must be spawned by git_receive_pack");
|
||||
let release = production
|
||||
.find(".release(disposition.release_ok)")
|
||||
.expect("U5 gate stale: release must consume the same disposition as the tail gate");
|
||||
.find(".release(push_succeeded)")
|
||||
.expect("U5 gate stale: release must consume the same success flag as the tail gate");
|
||||
let touch = production
|
||||
.find("state.db.touch_repo(")
|
||||
.expect("U5 gate stale: git_receive_pack no longer calls touch_repo");
|
||||
// #26 Split PR 1 step 3 — the webhook fan-out moved into
|
||||
// `durable_outbox::apply_request_effects`, which the live
|
||||
// handler calls inline (the recovery drain calls the same
|
||||
// function on the next startup). The gate now pins that the
|
||||
// handler is wired to the executor, not to a per-ref inline
|
||||
// webhook call.
|
||||
let effects_executor = production
|
||||
.find("apply_request_effects(&state, &request_id)")
|
||||
.expect("U5 gate stale: git_receive_pack no longer calls apply_request_effects inline");
|
||||
let webhook = production
|
||||
.find("webhooks::fire_event(")
|
||||
.expect("U5 gate stale: git_receive_pack no longer fires push webhooks");
|
||||
|
||||
assert!(
|
||||
success_flag < spawn,
|
||||
"U5 gate bypassed: the disposition must be bound before any tail spawn"
|
||||
success_flag < gate_open && gate_open < spawn,
|
||||
"U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \
|
||||
rejected push spawns a tail that pins and announces a half-applied repo"
|
||||
);
|
||||
// Still inside that block: no `}` may close it between the gate and the spawn.
|
||||
assert!(
|
||||
!production[gate_open + "if push_succeeded {".len()..spawn].contains('}'),
|
||||
"U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \
|
||||
rejected push now spawns a tail"
|
||||
);
|
||||
assert!(
|
||||
spawn < release && spawn < touch && spawn < effects_executor,
|
||||
spawn < release && spawn < touch && spawn < webhook,
|
||||
"U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \
|
||||
and the effect executor, so a disconnect in any of those windows cannot drop \
|
||||
and the webhook fan-out, so a disconnect in any of those windows cannot drop \
|
||||
this push's pins, recovery copy, and announcements"
|
||||
);
|
||||
}
|
||||
|
||||
/// #26 Split PR 1 step 3 — drain + handler share a single effect
|
||||
/// executor. The live handler (api/repos.rs) and the recovery
|
||||
/// drain (`durable_outbox::drain_receive_pack_requests_with`) both
|
||||
/// call `apply_request_effects`, so the per-ref effects fan-out
|
||||
/// lives in exactly one place. The v29 per-ref walk
|
||||
/// (`derive_one`, `drain_pending_ref_transitions_all`,
|
||||
/// `lookup_accepted_ordinal`) is dead code: any caller reintroduced
|
||||
/// would be the per-ref walk the step-3 PR removed. This gate
|
||||
/// fails if a call site slips back in or the new seam is bypassed.
|
||||
#[test]
|
||||
fn inv26_step3_live_and_drain_share_apply_request_effects() {
|
||||
let repos = src("api/repos.rs");
|
||||
let outbox = src("durable_outbox.rs");
|
||||
|
||||
// The live handler calls `apply_request_effects`. Split the
|
||||
// file at the test attribute so test code can't satisfy the
|
||||
// gate by itself.
|
||||
let production_repos = repos
|
||||
.split("\nmod tests {")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
assert!(
|
||||
production_repos.contains("apply_request_effects(&state, &request_id)"),
|
||||
"live handler must call `apply_request_effects(&state, &request_id)`; \
|
||||
reverting to a per-ref inline fan-out splits live and recovery"
|
||||
);
|
||||
|
||||
// The durable intent insert closes the pre-outbox crash window this
|
||||
// PR exists to fix. Wrapping it in `if false` must turn this gate
|
||||
// red; drain-only tests cannot prove the handler creates the rows.
|
||||
assert!(
|
||||
production_repos.contains("insert_receive_pack_request_with_children("),
|
||||
"live handler must persist durable intent via \
|
||||
`insert_receive_pack_request_with_children`; disabling it must fail this gate"
|
||||
);
|
||||
|
||||
// The drain's per-request seam calls the same executor. Test
|
||||
// code lives below `mod drain_tests`, so split there too.
|
||||
let production_outbox = outbox
|
||||
.split("\nmod drain_tests {")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
assert!(
|
||||
production_outbox.contains("drain_receive_pack_requests_with"),
|
||||
"drain seam `drain_receive_pack_requests_with` must exist; \
|
||||
removing it forces a per-ref walk back into the drain"
|
||||
);
|
||||
assert!(
|
||||
production_outbox.contains("apply_request_effects"),
|
||||
"durable_outbox production code must define `apply_request_effects`; \
|
||||
removing it splits the executor between live and recovery"
|
||||
);
|
||||
|
||||
// The drain's per-request walker is wired to the executor.
|
||||
// The closure body of `drain_receive_pack_requests_with` is the
|
||||
// only call site — if a future change calls `derive_one`
|
||||
// instead, this assertion fires.
|
||||
let drain_seam_open = production_outbox
|
||||
.find("pub async fn drain_receive_pack_requests_with<F, Fut>")
|
||||
.expect("drain seam must be defined in the production half");
|
||||
let drain_seam_close = production_outbox[drain_seam_open..]
|
||||
.find("\n}\n")
|
||||
.expect("drain seam body must close");
|
||||
let drain_seam_body = &production_outbox[drain_seam_open..drain_seam_open + drain_seam_close];
|
||||
assert!(
|
||||
drain_seam_body.contains("apply_request_effects"),
|
||||
"drain_receive_pack_requests_with must call apply_request_effects; \
|
||||
wiring it to a per-ref helper reintroduces the v29 walk"
|
||||
);
|
||||
|
||||
// The deleted per-ref drain must have zero call sites in the
|
||||
// production half. A regression that re-adds a caller would
|
||||
// bring back the per-ref fan-out.
|
||||
assert!(
|
||||
!production_outbox.contains("drain_pending_ref_transitions_all("),
|
||||
"deleted `drain_pending_ref_transitions_all` must have zero production call sites; \
|
||||
a per-ref walk is reintroduced"
|
||||
);
|
||||
assert!(
|
||||
!production_outbox.contains("derive_one("),
|
||||
"deleted `derive_one` must have zero production call sites; \
|
||||
the per-ref fan-out is reintroduced"
|
||||
);
|
||||
assert!(
|
||||
!production_outbox.contains("lookup_accepted_ordinal("),
|
||||
"deleted `lookup_accepted_ordinal` must have zero production call sites; \
|
||||
the per-ref ordinal lookup is reintroduced"
|
||||
);
|
||||
assert!(
|
||||
!production_repos.contains("derive_one("),
|
||||
"deleted `derive_one` must have zero live-handler call sites"
|
||||
);
|
||||
assert!(
|
||||
!production_repos.contains("drain_pending_ref_transitions"),
|
||||
"deleted per-ref drain functions must have zero live-handler call sites"
|
||||
);
|
||||
}
|
||||
|
||||
/// #26 Split PR 1 step 4 — the periodic queue-lifecycle purge is
|
||||
/// wired in `main.rs` and the contract is enforced by the `idx_receive_pack_requests_completed_at`
|
||||
/// partial index from v41.
|
||||
///
|
||||
/// Assertions:
|
||||
/// 1. `main.rs` calls `purge_request_queue` once at boot (well, on
|
||||
/// the spawn-task interval) and the spawn function exists.
|
||||
/// 2. The DB helpers `purge_completed_receive_pack_requests` and
|
||||
/// `purge_completed_pending_ref_transitions` exist in `db/mod.rs`.
|
||||
/// 3. The config knobs `queue_retention_days` and `queue_purge_batch` exist
|
||||
/// on `Config`.
|
||||
/// 4. `quarantined` is NOT in the purge WHERE clause (step 4 does not
|
||||
/// introduce the state, but the invariant holds for the future).
|
||||
#[test]
|
||||
fn inv26_step4_queue_lifecycle_purge_is_wired() {
|
||||
let main_src = src("main.rs");
|
||||
assert!(
|
||||
main_src.contains("spawn_queue_lifecycle_sweep"),
|
||||
"main.rs must spawn the periodic queue-lifecycle purge"
|
||||
);
|
||||
assert!(
|
||||
main_src.contains("purge_request_queue"),
|
||||
"main.rs must call purge_request_queue on the periodic sweep"
|
||||
);
|
||||
|
||||
let db_src = src("db/mod.rs");
|
||||
assert!(
|
||||
db_src.contains("purge_completed_receive_pack_requests"),
|
||||
"db/mod.rs must expose purge_completed_receive_pack_requests"
|
||||
);
|
||||
assert!(
|
||||
db_src.contains("purge_completed_pending_ref_transitions"),
|
||||
"db/mod.rs must expose purge_completed_pending_ref_transitions"
|
||||
);
|
||||
|
||||
let config_src = src("config.rs");
|
||||
assert!(
|
||||
config_src.contains("queue_retention_days"),
|
||||
"Config must expose queue_retention_days"
|
||||
);
|
||||
assert!(
|
||||
config_src.contains("queue_purge_batch"),
|
||||
"Config must expose queue_purge_batch"
|
||||
);
|
||||
}
|
||||
|
||||
/// #26 Split PR 1 step 5 — the marker gate and the retry-bound
|
||||
/// quarantine are wired end-to-end. A missing or hash-mismatched
|
||||
/// marker on disk quarantines the request; a Retry over the bound
|
||||
/// does too. This gate pins every load-bearing seam, against the
|
||||
/// production half of each file (the test modules name the same
|
||||
/// identifiers in their own harnesses).
|
||||
///
|
||||
/// Assertions:
|
||||
/// 1. `reconcile_prepared_page` calls `mark_request_quarantined`
|
||||
/// and `mark_children_rejected_for_quarantined_parent` on
|
||||
/// marker-gate failure.
|
||||
/// 2. The drain's `EffectsOutcome::Retry` arm checks
|
||||
/// `effects_max_attempts`.
|
||||
/// 3. `git::store::read_ref` exists in `git/store.rs`.
|
||||
/// 4. `db::mark_request_quarantined`,
|
||||
/// `db::mark_children_rejected_for_quarantined_parent`,
|
||||
/// `db::get_receive_pack_requests_by_ids`, and
|
||||
/// `db::request_state::QUARANTINED` all exist.
|
||||
/// 5. The handler writes the marker ref BEFORE calling
|
||||
/// `smart_http::receive_pack_raw` (the durability window).
|
||||
#[test]
|
||||
fn inv26_step5_marker_quarantine_and_bound_are_wired() {
|
||||
let outbox = src("durable_outbox.rs");
|
||||
let store = src("git/store.rs");
|
||||
let db = src("db/mod.rs");
|
||||
let repos = src("api/repos.rs");
|
||||
|
||||
// Split at the TEST MODULE for the production-only assertions.
|
||||
let production_outbox = outbox
|
||||
.split("\nmod drain_tests {")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
let production_repos = repos
|
||||
.split("\nmod tests {")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
// (1) The reconcile's marker gate quarantines via the DB helpers.
|
||||
assert!(
|
||||
production_outbox.contains("mark_request_quarantined"),
|
||||
"reconcile_prepared_page must call mark_request_quarantined on marker-gate failure"
|
||||
);
|
||||
assert!(
|
||||
production_outbox.contains("mark_children_rejected_for_quarantined_parent"),
|
||||
"reconcile_prepared_page must call mark_children_rejected_for_quarantined_parent \
|
||||
so quarantined parents cancel their children"
|
||||
);
|
||||
|
||||
// The gate reads the marker ref and compares against the parent's
|
||||
// `request_bytes_hash` via `git::store::read_ref` /
|
||||
// `git::store::marker_value_for`. Reverting either reintroduces
|
||||
// the DoS window the marker gate exists to close.
|
||||
assert!(
|
||||
production_outbox.contains("git::store::read_ref"),
|
||||
"reconcile_prepared_page must read the marker ref via git::store::read_ref"
|
||||
);
|
||||
assert!(
|
||||
production_outbox.contains("marker_value_for"),
|
||||
"reconcile_prepared_page must compute the expected marker value via \
|
||||
git::store::marker_value_for"
|
||||
);
|
||||
|
||||
// (2) The drain's `EffectsOutcome::Retry` arm checks the bound.
|
||||
assert!(
|
||||
production_outbox.contains("effects_max_attempts"),
|
||||
"drain_receive_pack_requests_with must consult effects_max_attempts on Retry"
|
||||
);
|
||||
|
||||
// (3) `git::store::read_ref` is the read seam the gate depends on.
|
||||
assert!(
|
||||
store.contains("pub fn read_ref("),
|
||||
"git::store::read_ref must exist; the marker gate reads through it"
|
||||
);
|
||||
assert!(
|
||||
store.contains("pub fn marker_value_for("),
|
||||
"git::store::marker_value_for must exist; the marker gate computes the expected \
|
||||
value with it (and the live handler writes the value via the same helper)"
|
||||
);
|
||||
|
||||
// (4) DB-side seams the gate depends on.
|
||||
assert!(
|
||||
db.contains("pub async fn mark_request_quarantined"),
|
||||
"Db::mark_request_quarantined must exist"
|
||||
);
|
||||
assert!(
|
||||
db.contains("pub async fn mark_children_rejected_for_quarantined_parent"),
|
||||
"Db::mark_children_rejected_for_quarantined_parent must exist"
|
||||
);
|
||||
assert!(
|
||||
db.contains("pub async fn get_receive_pack_requests_by_ids"),
|
||||
"Db::get_receive_pack_requests_by_ids must exist (avoids N+1 in the marker gate)"
|
||||
);
|
||||
assert!(
|
||||
db.contains("pub const QUARANTINED: &str = \"quarantined\""),
|
||||
"request_state::QUARANTINED must be defined"
|
||||
);
|
||||
|
||||
// (5) The handler writes the marker ref BEFORE the durability
|
||||
// boundary (smart_http::receive_pack_raw). Severing the
|
||||
// ordering re-opens the marker-gate DoS window for live pushes.
|
||||
let marker_write = production_repos
|
||||
.find("git::store::marker_value_for")
|
||||
.expect("U5 gate stale: the live handler no longer computes the marker value");
|
||||
let receive_raw = production_repos
|
||||
.find("smart_http::receive_pack_raw")
|
||||
.expect("U5 gate stale: git_receive_pack no longer calls smart_http::receive_pack_raw");
|
||||
assert!(
|
||||
marker_write < receive_raw,
|
||||
"U5 gate bypassed: the marker ref must be written BEFORE receive_pack_raw so the \
|
||||
reconcile's gate has evidence of the live push"
|
||||
);
|
||||
}
|
||||
|
||||
/// Issue #26 split 1 — namespace gate and pre-git refusal lifecycle.
|
||||
///
|
||||
/// The push gate (repos.rs) must deny:
|
||||
/// 1. Any ref-update line that fails UTF-8 decode (the raw body is passed verbatim
|
||||
/// to git receive-pack, and git accepts refnames with bytes >= 0x80; the gate
|
||||
/// never sees these refs, no durable child row exists, and under requests/issues/
|
||||
/// the lossy for-each-ref decode still matches the narrowed exemption).
|
||||
/// 2. The bare `refs/gitlawb` name (the `starts_with("refs/gitlawb/")` check misses it).
|
||||
///
|
||||
/// Every explicit pre-git refusal must call `refuse_receive_pack_before_git` to
|
||||
/// terminalize the intent aggregate in one transaction, or a dropped future between
|
||||
/// the intent insert and `receive_pack` strands a `received` + `prepared` aggregate
|
||||
/// that is never purged, never promotable, and poisons its tuple.
|
||||
///
|
||||
/// The visibility exemption (visibility_pack.rs) was narrowed to `requests/` and
|
||||
/// `issues/` only. The push gate is the premise that narrowing relies on (the comment
|
||||
/// at visibility_pack.rs:340), so tests must prove:
|
||||
/// - A `refs/gitlawb/<other>` ref pointing at a blob is denied on push.
|
||||
/// - A stray `refs/gitlawb/<other>` blob ref planted in the pack path fails closed.
|
||||
///
|
||||
/// Source gates below pin the wiring; behavioral proofs live next to the code:
|
||||
/// `parse_ref_updates_refuses_non_utf8_command_line` and
|
||||
/// `internal_namespace_gate_covers_bare_name_and_subtree` (api/repos.rs),
|
||||
/// `planted_non_exempt_gitlawb_blob_ref_fails_closed` (git/visibility_pack.rs),
|
||||
/// and the mirror import prune
|
||||
/// `prune_removes_non_exempt_gitlawb_refs_and_keeps_exempt` (sync.rs).
|
||||
#[test]
|
||||
fn issue_26_namespace_gate_and_refusal_wiring() {
|
||||
let repos = src("api/repos.rs");
|
||||
let vis = src("git/visibility_pack.rs");
|
||||
|
||||
// P1: fail the push on ref-update lines the handler cannot decode.
|
||||
// The gate at parse_ref_updates must refuse when any command line in the
|
||||
// ref-update section is undecodable UTF-8. The check is an Err arm on
|
||||
// `from_utf8(data)` that returns BadRequest, so removing it allows non-UTF-8
|
||||
// refs to bypass the namespace gate.
|
||||
assert!(
|
||||
repos.contains("ref-update line is not valid UTF-8; push refused"),
|
||||
"P1 gate missing: parse_ref_updates must refuse when any ref-update line is \
|
||||
non-UTF-8 (git accepts bytes >= 0x80; the gate never sees them, no durable \
|
||||
child exists, and the lossy for-each-ref decode still matches the exemption)"
|
||||
);
|
||||
|
||||
// P1 (cont): the bare `refs/gitlawb` name must also be caught. The
|
||||
// predicate is extracted as `ref_is_internal_namespace` so it is
|
||||
// unit-testable; pin the bare-name arm inside it and its use at the
|
||||
// push gate (a `starts_with("refs/gitlawb/")`-only check misses the
|
||||
// bare name, and it lands on repos with no internal refs yet).
|
||||
assert!(
|
||||
repos.contains("ref_name == \"refs/gitlawb\"")
|
||||
&& repos.contains(".find(|r| ref_is_internal_namespace(r))"),
|
||||
"P1 gate missing: ref_is_internal_namespace must match the bare \
|
||||
'refs/gitlawb' name and the push gate must consult it \
|
||||
(starts_with('refs/gitlawb/') misses it, and it lands on repos with \
|
||||
no internal refs yet)"
|
||||
);
|
||||
|
||||
// P1 (cont): every explicit pre-git refusal must call refuse_receive_pack_before_git
|
||||
// to terminalize the intent. Exactly 6 occurrences: the five inline call
|
||||
// sites (lease cap, per-source cap, write pool, acquire timeout, acquire
|
||||
// error) plus the drop-guard's own pre-git spawn. The post-git guard phase
|
||||
// calls mark_receive_pack_interrupted (a different name), so deleting any
|
||||
// one inline site drops the count to 5 and turns this red. The parse and
|
||||
// namespace refusals correctly carry no call: they return before the
|
||||
// intent insert, so there is no aggregate to terminalize.
|
||||
assert_eq!(
|
||||
repos.matches("refuse_receive_pack_before_git(").count(),
|
||||
6,
|
||||
"P1 wiring: exactly 6 refuse_receive_pack_before_git calls expected (5 \
|
||||
inline refusal sites + guard pre-git spawn); deleting any inline site \
|
||||
must turn this red"
|
||||
);
|
||||
// Pin the five inline sites by their reason literals, so a site that stops
|
||||
// terminalizing (or is renumbered away) fails by name, not just by count.
|
||||
for reason in [
|
||||
"repo write-lease waiter cap reached",
|
||||
"admission per-source cap reached",
|
||||
"git write pool saturated",
|
||||
"acquire_write timed out",
|
||||
"acquire_write failed",
|
||||
"handler dropped before git started",
|
||||
] {
|
||||
assert!(
|
||||
repos.contains(reason),
|
||||
"P1 wiring: refusal reason literal missing: {reason}"
|
||||
);
|
||||
}
|
||||
|
||||
// P1 (cont): the intent drop-guard must be phase-aware. It is armed before
|
||||
// the insert resolves (covering the cancellation window), flipped to
|
||||
// git-started immediately before the receive_pack await (a drop from there
|
||||
// is a mid-git disconnect, terminalized via mark_receive_pack_interrupted
|
||||
// because the Err arm never runs on a drop), and disarmed once the outcome
|
||||
// commit resolves. Assert the ordering, not just presence: the phase flip
|
||||
// must precede the receive_pack call.
|
||||
assert!(
|
||||
repos.contains("IntentDropGuard::new") && repos.contains("notify_git_started()"),
|
||||
"P1 gate missing: IntentDropGuard must exist and flip to git-started phase \
|
||||
before receive_pack (a dropped future between intent insert and git start \
|
||||
strands a received+prepared aggregate)"
|
||||
);
|
||||
let flip_pos = repos
|
||||
.find("notify_git_started()")
|
||||
.expect("notify_git_started call must exist");
|
||||
let git_pos = repos
|
||||
.find("receive_pack_raw_with_reflog(")
|
||||
.expect("receive_pack call must exist");
|
||||
assert!(
|
||||
flip_pos < git_pos,
|
||||
"P1 ordering: notify_git_started() must precede the receive_pack call, \
|
||||
or a mid-git drop is misclassified as a pre-git shed"
|
||||
);
|
||||
assert!(
|
||||
repos.contains("_intent_guard.disarm()"),
|
||||
"P1 gate missing: the guard must disarm once the outcome commit resolves, \
|
||||
or every push spawns a wasteful cleanup txn at scope end"
|
||||
);
|
||||
// A sync Drop cannot await: cleanup is spawned via the current runtime
|
||||
// handle with an off-runtime fallback. A bare tokio::spawn panics
|
||||
// mid-unwind with no runtime (off-runtime drop, process teardown).
|
||||
assert!(
|
||||
repos.contains("Handle::try_current()"),
|
||||
"P1 gate missing: guard Drop must spawn via Handle::try_current with an \
|
||||
off-runtime arm, not a bare tokio::spawn"
|
||||
);
|
||||
|
||||
// P2: the post-git drop arm must exist and be wired. The guard's
|
||||
// GIT_STARTED phase spawns `mark_receive_pack_interrupted` (parent to
|
||||
// `rejected_at_git`, children to `uncertain` — the Err arm never runs on
|
||||
// a drop). Deleting that spawn keeps every count above green, so pin the
|
||||
// method name; its behavior is proved by
|
||||
// `post_git_interruption_marks_uncertain_and_terminalizes_parent`.
|
||||
assert!(
|
||||
repos.contains("mark_receive_pack_interrupted("),
|
||||
"P2 wiring: the guard's post-git phase must spawn \
|
||||
mark_receive_pack_interrupted (a mid-git drop otherwise strands \
|
||||
received+prepared with no Err arm to mark them uncertain)"
|
||||
);
|
||||
// The guard must commit computed fates rather than a blanket uncertain
|
||||
// once the report is parsed, and only disarm on a resolved commit: pin
|
||||
// both the stash call and the `outcome_commit_ok` gate. Making disarm()
|
||||
// unconditional strands the commit-failure path and misfiles proven
|
||||
// rejections as uncertain.
|
||||
assert!(
|
||||
repos.contains("set_fates(ComputedFates"),
|
||||
"P2 wiring: computed fates must be stashed on the guard once parsed, \
|
||||
or a post-commit drop can only apply a context-free uncertain"
|
||||
);
|
||||
assert!(
|
||||
repos.contains("if outcome_commit_ok"),
|
||||
"P2 wiring: disarm must be gated on outcome_commit_ok; unconditional \
|
||||
disarm strands the persistent-commit-failure path"
|
||||
);
|
||||
// The live-handler `AwaitingSiblings` arm is unreachable through the
|
||||
// current outcome builder (documented at the arm itself) and therefore
|
||||
// has no behavioral test; pin its presence so removing it is conscious.
|
||||
// The drain consumer IS behaviorally driven (sibling-wait tests).
|
||||
assert_eq!(
|
||||
repos.matches("EffectsOutcome::AwaitingSiblings").count(),
|
||||
1,
|
||||
"live AwaitingSiblings arm must exist (defensive symmetry with the \
|
||||
drain consumer); deleting it must turn this red"
|
||||
);
|
||||
|
||||
// P2: pin the mirror prune call sites. The prune helper is called from
|
||||
// both `clone_repo` and `fetch_repo` (plus its own definition) — deleting
|
||||
// either call leaves the direct-helper test green while mirrors import
|
||||
// evil refs again. Count occurrences of the call with its argument.
|
||||
let sync_src = src("sync.rs");
|
||||
assert_eq!(
|
||||
sync_src
|
||||
.matches("prune_non_exempt_gitlawb_refs(local_path)")
|
||||
.count(),
|
||||
2,
|
||||
"P2 wiring: prune must be called from both clone_repo and fetch_repo; \
|
||||
deleting either call must turn this red"
|
||||
);
|
||||
// Both call sites must route through `wrap_prune_error`, which preserves
|
||||
// deterministic `PruneInvalid` refusals instead of stringifying every
|
||||
// error into transient `PruneFailed` (an every-tick deferral spin for
|
||||
// permanently invalid mirrors). Pin the exact call shape at both sites.
|
||||
assert_eq!(
|
||||
sync_src.matches(".map_err(wrap_prune_error)").count(),
|
||||
2,
|
||||
"P2 wiring: both prune call sites must use wrap_prune_error; inline \
|
||||
rewrap must turn this red"
|
||||
);
|
||||
|
||||
// P2: the visibility exemption narrowing removed a serving path. The exemption
|
||||
// is now scoped to `requests/` and `issues/` only (visibility_pack.rs:371), and
|
||||
// the comment at :340 states the push gate is the premise. Tests must prove the
|
||||
// gate works, which the P1 findings above covered (non-UTF-8 line, bare name,
|
||||
// refusal wiring). This check verifies the exemption narrowing landed.
|
||||
assert!(
|
||||
vis.contains(
|
||||
r#"!r.starts_with("refs/gitlawb/requests/") && !r.starts_with("refs/gitlawb/issues/")"#
|
||||
),
|
||||
"P2 gate missing: the visibility exemption must be narrowed to requests/ and \
|
||||
issues/ only (the push gate is the premise; any other refs/gitlawb/* ref is \
|
||||
denied on push and must fail closed in the pack path)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Claimed-webhook delivery must not re-list hooks: the durable path claims
|
||||
/// synchronously, hands the claimed hook rows to the sender, and a list
|
||||
/// failure in the send task would silently drop already-claimed deliveries.
|
||||
#[test]
|
||||
fn inv26_claimed_hook_delivery_never_relists() {
|
||||
let webhooks = src("webhooks.rs");
|
||||
let production = webhooks
|
||||
.split("\nmod tests {")
|
||||
.next()
|
||||
.expect("split always yields a first chunk");
|
||||
|
||||
// The claimed entry point exists and passes hook ROWS (not ids) so the
|
||||
// sender never needs the hook table again.
|
||||
let entry = production
|
||||
.find("fire_event_occurrence_with_claimed_hooks(")
|
||||
.expect("claimed-hooks sender missing");
|
||||
let claimed_rows = production
|
||||
.find("claimed_hooks: Vec<crate::db::Webhook>")
|
||||
.expect("claimed path must carry Webhook rows, not ids");
|
||||
let list_call = production
|
||||
.find("db.list_webhooks_for_event(repo_id, event)")
|
||||
.expect("unclaimed path still lists hooks");
|
||||
assert!(
|
||||
entry < claimed_rows && claimed_rows < list_call,
|
||||
"claimed sender must hand rows through before any list fallback"
|
||||
);
|
||||
|
||||
// Inside the async worker, the claimed branch must bypass the list.
|
||||
// Inside the async worker (which starts after the claim helper), the
|
||||
// claimed branch must be selected before the list fallback can run.
|
||||
let worker = production
|
||||
.find("async fn fire_event_async_occurrence(")
|
||||
.expect("occurrence worker missing");
|
||||
let already_claimed = production[worker..]
|
||||
.find("let already_claimed = claimed_hooks.is_some();")
|
||||
.map(|i| worker + i)
|
||||
.expect("claimed path must short-circuit before re-listing");
|
||||
let worker_list = production[worker..]
|
||||
.find("db.list_webhooks_for_event(repo_id, event)")
|
||||
.map(|i| worker + i)
|
||||
.expect("unclaimed path still lists hooks");
|
||||
assert!(
|
||||
already_claimed < worker_list,
|
||||
"the claimed branch must be selected before list_webhooks_for_event can run"
|
||||
);
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Map a pull request's changed paths to the CI surfaces they touch, so each
|
||||
# check in pr-checks.yml and codeql.yml runs only when its inputs changed and
|
||||
# Rust jobs test only the affected crates.
|
||||
#
|
||||
# Usage:
|
||||
# git diff -z --name-only --no-renames BASE HEAD | scripts/ci-surfaces.sh
|
||||
# scripts/ci-surfaces.sh --all # every surface, whole workspace
|
||||
#
|
||||
# Input is NUL-terminated paths (`git diff -z`). Without -z, git C-quotes any
|
||||
# path with non-ASCII, tab, quote or newline characters ("crates/gl/\303\251.rs"),
|
||||
# which would match no surface and skip every job; newline-delimited input
|
||||
# also cannot carry a path that contains a newline. Input that does not end in
|
||||
# NUL (e.g. a plain `git diff --name-only`) is rejected rather than misread.
|
||||
# scripts/test-ci-surfaces.sh pins the mapping.
|
||||
#
|
||||
# Prints GITHUB_OUTPUT lines (key=value). Each surface runs only for the paths
|
||||
# listed for it below, so a path no surface lists (docs, README, infra/, any
|
||||
# other workflow) runs no PR Checks job. Within the Rust surface, a change this
|
||||
# script cannot attribute to specific crates widens rather than narrows: a
|
||||
# global Rust input or an unknown crate directory runs the Rust jobs on the
|
||||
# whole workspace. The CI wiring itself (pr-checks.yml, codeql.yml, this
|
||||
# script) runs every surface.
|
||||
# Non-PR events (push to main, merge_group, workflow_dispatch, schedule) are
|
||||
# expected to pass --all, so main and the merge queue keep full coverage.
|
||||
#
|
||||
# Runnable from anywhere in the repo.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
|
||||
all=false
|
||||
if [ "${1:-}" = "--all" ]; then
|
||||
all=true
|
||||
elif [ "$#" -ne 0 ]; then
|
||||
echo "usage: $0 [--all] < NUL-terminated-changed-paths" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
changed=()
|
||||
if [ "$all" = false ]; then
|
||||
path=""
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -n "$path" ] && changed+=("$path")
|
||||
done
|
||||
# read leaves any unterminated tail in $path.
|
||||
if [ -n "$path" ]; then
|
||||
echo "ci-surfaces.sh: input is not NUL-terminated; pipe \`git diff -z --name-only\`" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Workspace crates, derived from crates/*/Cargo.toml so a new crate is picked
|
||||
# up without editing this script. dir -> package name, and dir -> the crate
|
||||
# dirs it depends on through `path = "../<dir>"` (normal, dev, or build deps).
|
||||
declare -A pkg_of=() deps_of=()
|
||||
for manifest in "$ROOT"/crates/*/Cargo.toml; do
|
||||
dir="$(basename "$(dirname "$manifest")")"
|
||||
pkg_of[$dir]="$(sed -n 's/^name *= *"\([^"]*\)".*/\1/p' "$manifest" | head -n1)"
|
||||
deps_of[$dir]="$(sed -n 's#.*path *= *"\.\./\([^"/]*\)".*#\1#p' "$manifest" | tr '\n' ' ')"
|
||||
done
|
||||
|
||||
# Files outside crates/ that a crate compiles in (include_str!/include_bytes!),
|
||||
# path -> crate dir. A change to one is a change to that crate, so its tests
|
||||
# run. Candidates (keep those that resolve outside crates/<dir>/):
|
||||
# git grep -nE 'include_(str|bytes)!\("(\.\./){2,}' crates
|
||||
declare -A embedded_in=(
|
||||
[bootstrap-peers.json]=gitlawb-node
|
||||
)
|
||||
|
||||
# Mark a crate dir as touched; a dir this checkout does not know (removed or
|
||||
# renamed crate) widens the Rust jobs to the whole workspace.
|
||||
touch_crate() {
|
||||
if [ -n "${pkg_of[$1]:-}" ]; then
|
||||
touched[$1]=1
|
||||
else
|
||||
full_rust=true
|
||||
fi
|
||||
}
|
||||
|
||||
full_rust=false
|
||||
rust=false
|
||||
docker=false
|
||||
audit=false
|
||||
core_deps=false
|
||||
release_versioning=false
|
||||
release_tag=false
|
||||
cq_actions=false
|
||||
cq_js=false
|
||||
cq_rust=false
|
||||
cq_swift=false
|
||||
declare -A touched=()
|
||||
|
||||
for path in "${changed[@]}"; do
|
||||
case "$path" in
|
||||
# The CI wiring itself: validate everything it gates.
|
||||
.github/workflows/pr-checks.yml | .github/workflows/codeql.yml | scripts/ci-surfaces.sh)
|
||||
all=true ;;
|
||||
# Workspace-wide Rust inputs.
|
||||
Cargo.toml | Cargo.lock | .cargo/* | rust-toolchain | rust-toolchain.toml \
|
||||
| rustfmt.toml | .rustfmt.toml | clippy.toml | .clippy.toml)
|
||||
full_rust=true ;;
|
||||
crates/*/*)
|
||||
dir="${path#crates/}"
|
||||
touch_crate "${dir%%/*}" ;;
|
||||
*)
|
||||
if [ -n "${embedded_in[$path]:-}" ]; then
|
||||
touch_crate "${embedded_in[$path]}"
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
Dockerfile | .dockerignore | bootstrap-peers.json | Cargo.toml | Cargo.lock \
|
||||
| .cargo/* | rust-toolchain | rust-toolchain.toml | crates/*/Cargo.toml)
|
||||
docker=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
Cargo.lock | .cargo/audit.toml) audit=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
Cargo.toml | Cargo.lock | crates/gitlawb-core/Cargo.toml \
|
||||
| ci/gitlawb-core-allowed-deps.txt | scripts/check-gitlawb-core-deps.sh)
|
||||
core_deps=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
Cargo.toml | crates/*/Cargo.toml | release-please-config.json \
|
||||
| .release-please-manifest.json | scripts/check-release-versioning.sh)
|
||||
release_versioning=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
scripts/resolve-release-tag.sh | scripts/test-resolve-release-tag.sh)
|
||||
release_tag=true ;;
|
||||
esac
|
||||
|
||||
case "$path" in
|
||||
.github/workflows/* | .github/actions/*) cq_actions=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
npm/* | *.js | *.mjs | *.cjs | *.jsx | *.ts | *.mts | *.cts | *.tsx) cq_js=true ;;
|
||||
esac
|
||||
case "$path" in
|
||||
macos-app/* | *.swift) cq_swift=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$all" = true ]; then
|
||||
full_rust=true
|
||||
docker=true
|
||||
audit=true
|
||||
core_deps=true
|
||||
release_versioning=true
|
||||
release_tag=true
|
||||
cq_actions=true
|
||||
cq_js=true
|
||||
cq_swift=true
|
||||
fi
|
||||
|
||||
# Affected crates: the touched ones plus everything that depends on them,
|
||||
# transitively, so a gitlawb-core change still tests every consumer.
|
||||
declare -A affected=()
|
||||
for dir in "${!touched[@]}"; do affected[$dir]=1; done
|
||||
grew=true
|
||||
while [ "$grew" = true ]; do
|
||||
grew=false
|
||||
for dir in "${!deps_of[@]}"; do
|
||||
[ -n "${affected[$dir]:-}" ] && continue
|
||||
for dep in ${deps_of[$dir]}; do
|
||||
if [ -n "${affected[$dep]:-}" ]; then
|
||||
affected[$dir]=1
|
||||
grew=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
if [ "$full_rust" = true ] || [ "${#affected[@]}" -gt 0 ]; then
|
||||
rust=true
|
||||
cq_rust=true
|
||||
fi
|
||||
|
||||
rust_packages=""
|
||||
windows_packages=""
|
||||
if [ "$full_rust" = true ]; then
|
||||
rust_packages="--workspace"
|
||||
windows_packages="-p gl -p git-remote-gitlawb"
|
||||
elif [ "$rust" = true ]; then
|
||||
for dir in $(printf '%s\n' "${!affected[@]}" | sort); do
|
||||
rust_packages+="${rust_packages:+ }-p ${pkg_of[$dir]}"
|
||||
case "${pkg_of[$dir]}" in
|
||||
gl | git-remote-gitlawb) windows_packages+="${windows_packages:+ }-p ${pkg_of[$dir]}" ;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
windows=false
|
||||
[ -n "$windows_packages" ] && windows=true
|
||||
|
||||
languages=()
|
||||
[ "$cq_actions" = true ] && languages+=('{"language":"actions","build-mode":"none","runner":"ubuntu-latest"}')
|
||||
[ "$cq_js" = true ] && languages+=('{"language":"javascript-typescript","build-mode":"none","runner":"ubuntu-latest"}')
|
||||
[ "$cq_rust" = true ] && languages+=('{"language":"rust","build-mode":"none","runner":"ubuntu-latest"}')
|
||||
[ "$cq_swift" = true ] && languages+=('{"language":"swift","build-mode":"autobuild","runner":"macos-latest"}')
|
||||
codeql_matrix="{\"include\":[$(IFS=,; echo "${languages[*]}")]}"
|
||||
codeql=false
|
||||
[ "${#languages[@]}" -gt 0 ] && codeql=true
|
||||
|
||||
cat <<EOF
|
||||
rust=$rust
|
||||
rust_packages=$rust_packages
|
||||
windows=$windows
|
||||
windows_packages=$windows_packages
|
||||
docker=$docker
|
||||
audit=$audit
|
||||
core_deps=$core_deps
|
||||
release_versioning=$release_versioning
|
||||
release_tag=$release_tag
|
||||
codeql=$codeql
|
||||
codeql_matrix=$codeql_matrix
|
||||
EOF
|
||||
@@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fixture test for scripts/ci-surfaces.sh, which alone decides which PR Checks
|
||||
# jobs and CodeQL languages run. Pins at least one must-not-skip case per
|
||||
# surface row, the crate fan-out and widening rules, and paths that git would
|
||||
# C-quote without -z. Run from pr-checks.yml before the mapper is used.
|
||||
set -euo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
mapper="$repo_root/scripts/ci-surfaces.sh"
|
||||
failures=0
|
||||
|
||||
# map PATH... : run the mapper on the paths as `git diff -z` would emit them.
|
||||
map() {
|
||||
if [ "$#" -gt 0 ]; then printf '%s\0' "$@"; fi | "$mapper"
|
||||
}
|
||||
|
||||
# check CASE OUTPUT KEY=VALUE... : each KEY must have exactly VALUE.
|
||||
check() {
|
||||
local name="$1" out="$2" want key
|
||||
shift 2
|
||||
for want in "$@"; do
|
||||
key="${want%%=*}"
|
||||
if ! grep -qxF -- "$want" <<<"$out"; then
|
||||
printf 'FAIL %s: want %s, got %s\n' "$name" "$want" \
|
||||
"$(grep -E "^${key}=" <<<"$out" || echo '<missing>')" >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# has_packages CASE OUTPUT PKG... : rust_packages must include each -p PKG.
|
||||
has_packages() {
|
||||
local name="$1" out="$2" pkgs pkg
|
||||
shift 2
|
||||
pkgs=" $(sed -n 's/^rust_packages=//p' <<<"$out") "
|
||||
for pkg in "$@"; do
|
||||
if [[ "$pkgs" != *" -p $pkg "* ]]; then
|
||||
printf 'FAIL %s: rust_packages lacks -p %s (got%s)\n' "$name" "$pkg" "$pkgs" >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
nothing=(rust=false rust_packages= windows=false windows_packages= docker=false
|
||||
audit=false core_deps=false release_versioning=false release_tag=false
|
||||
codeql=false 'codeql_matrix={"include":[]}')
|
||||
everything=(rust=true rust_packages=--workspace windows=true
|
||||
'windows_packages=-p gl -p git-remote-gitlawb' docker=true audit=true
|
||||
core_deps=true release_versioning=true release_tag=true codeql=true)
|
||||
all_languages='codeql_matrix={"include":[{"language":"actions","build-mode":"none","runner":"ubuntu-latest"},{"language":"javascript-typescript","build-mode":"none","runner":"ubuntu-latest"},{"language":"rust","build-mode":"none","runner":"ubuntu-latest"},{"language":"swift","build-mode":"autobuild","runner":"macos-latest"}]}'
|
||||
rust_only='codeql_matrix={"include":[{"language":"rust","build-mode":"none","runner":"ubuntu-latest"}]}'
|
||||
|
||||
# Docs-only and empty input run nothing.
|
||||
check "docs only" "$(map README.md docs/x.md infra/x.tf)" "${nothing[@]}"
|
||||
check "empty input" "$(map)" "${nothing[@]}"
|
||||
|
||||
# A crate change tests that crate and its reverse deps, nothing else.
|
||||
check "gl source" "$(map crates/gl/src/main.rs)" \
|
||||
rust=true 'rust_packages=-p gl' windows=true 'windows_packages=-p gl' \
|
||||
docker=false audit=false "$rust_only"
|
||||
out="$(map crates/gitlawb-core/src/lib.rs)"
|
||||
check "gitlawb-core source" "$out" rust=true windows=true \
|
||||
'windows_packages=-p git-remote-gitlawb -p gl' "$rust_only"
|
||||
has_packages "gitlawb-core fan-out" "$out" gitlawb-core icaptcha-client \
|
||||
gitlawb-attest git-remote-gitlawb gl gitlawb-node
|
||||
check "icaptcha-client source" "$(map crates/icaptcha-client/src/lib.rs)" \
|
||||
'rust_packages=-p gl -p icaptcha-client' 'windows_packages=-p gl'
|
||||
|
||||
# Anything the Rust surface cannot attribute to known crates widens.
|
||||
check "unknown crate dir" "$(map crates/no-such-crate/src/lib.rs)" \
|
||||
rust=true rust_packages=--workspace docker=false
|
||||
check "root Cargo.lock" "$(map Cargo.lock)" \
|
||||
rust=true rust_packages=--workspace docker=true audit=true core_deps=true \
|
||||
release_versioning=false
|
||||
check "toolchain file" "$(map rust-toolchain.toml)" rust=true rust_packages=--workspace
|
||||
|
||||
# Files compiled into a crate from outside crates/.
|
||||
check "bootstrap-peers.json" "$(map bootstrap-peers.json)" \
|
||||
rust=true 'rust_packages=-p gitlawb-node' windows=false docker=true
|
||||
|
||||
# One must-not-skip trigger per non-Rust row.
|
||||
check "Dockerfile" "$(map Dockerfile)" docker=true rust=false
|
||||
check "crate manifest" "$(map crates/gl/Cargo.toml)" \
|
||||
docker=true release_versioning=true 'rust_packages=-p gl'
|
||||
check "audit config" "$(map .cargo/audit.toml)" audit=true rust_packages=--workspace
|
||||
check "core allowlist" "$(map ci/gitlawb-core-allowed-deps.txt)" core_deps=true rust=false
|
||||
check "release-please config" "$(map release-please-config.json)" \
|
||||
release_versioning=true rust=false
|
||||
check "release tag resolver" "$(map scripts/resolve-release-tag.sh)" release_tag=true rust=false
|
||||
check "other workflow" "$(map .github/workflows/release.yml)" "${nothing[@]:0:9}" \
|
||||
codeql=true 'codeql_matrix={"include":[{"language":"actions","build-mode":"none","runner":"ubuntu-latest"}]}'
|
||||
check "npm" "$(map npm/package.json)" codeql=true \
|
||||
'codeql_matrix={"include":[{"language":"javascript-typescript","build-mode":"none","runner":"ubuntu-latest"}]}'
|
||||
check "swift" "$(map macos-app/Sources/App.swift)" rust=false codeql=true \
|
||||
'codeql_matrix={"include":[{"language":"swift","build-mode":"autobuild","runner":"macos-latest"}]}'
|
||||
|
||||
# Editing the CI wiring runs every surface, exactly like --all.
|
||||
for wiring in .github/workflows/pr-checks.yml .github/workflows/codeql.yml scripts/ci-surfaces.sh; do
|
||||
check "wiring $wiring" "$(map "$wiring")" "${everything[@]}" "$all_languages"
|
||||
done
|
||||
check "--all" "$("$mapper" --all </dev/null)" "${everything[@]}" "$all_languages"
|
||||
|
||||
# Names git C-quotes without -z (non-ASCII, tab, quote, newline) still map.
|
||||
for name in $'\xc3\xa9.rs' $'a\tb.rs' 'a"b.rs' $'a\nb.rs'; do
|
||||
check "quoted name $(printf %q "$name")" "$(map "crates/gl/src/$name")" \
|
||||
rust=true 'rust_packages=-p gl'
|
||||
done
|
||||
check "non-ASCII swift" "$(map $'macos-app/Sources/Caf\xc3\xa9.swift')" codeql=true \
|
||||
'codeql_matrix={"include":[{"language":"swift","build-mode":"autobuild","runner":"macos-latest"}]}'
|
||||
|
||||
# Newline-delimited input (git diff without -z) is rejected, not misread.
|
||||
if printf 'crates/gl/src/main.rs\n' | "$mapper" >/dev/null 2>&1; then
|
||||
echo "FAIL newline-delimited input: mapper accepted it" >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "$failures ci-surfaces mapping check(s) failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "ci-surfaces mapping: all checks passed"
|
||||
Reference in new issue
Block a user