Compare commits

...
Author SHA1 Message Date
beardthelion eaab0f64b8 Merge remote-tracking branch 'origin/main' into pr-347
# Conflicts:
#	crates/gitlawb-node/src/api/peers.rs
#	crates/gitlawb-node/src/config.rs
#	crates/gitlawb-node/src/db/mod.rs
2026-11-08 21:39:11 -06:00
KRATOS d34aac8c2a fix(node): close inbound mirror review blockers
Origin-Session: local-d6a143 | Codex | 24 prompts
2026-08-15 16:14:59 +05:30
KRATOS 111d1ccd6b fix(node): harden scheduled mirror fetches
Origin-Session: local-d6a143 | Codex | 24 prompts
2026-08-15 15:41:50 +05:30
KRATOS 5cad709659 feat(node): add scheduled inbound mirror worker
Origin-Session: local-d6a143 | Codex | 18 prompts
2026-08-15 15:15:39 +05:30
KRATOS a6d8459002 fix(node): close remaining mirror review findings
Origin-Session: local-d6a143 | Codex | 21 prompts
2026-08-15 09:47:18 +05:30
KRATOS 1b7639f0d7 fix(node): align peer URL validation message
Origin-Session: local-d6a143 | Codex | 20 prompts
2026-08-15 00:33:01 +05:30
KRATOS 5b9e06d501 fix(node): close mirror review blockers
Origin-Session: local-d6a143 | Codex | 20 prompts
2026-08-15 00:25:14 +05:30
KRATOS 92997b3695 fix(node): address mirror state review findings
Origin-Session: local-d6a143 | Codex | 16 prompts
2026-08-14 22:22:19 +05:30
KRATOS ea34d84c3a fix(node): reject null mirror transition state
Origin-Session: local-d6a143 | Codex | 14 prompts
2026-08-14 21:44:27 +05:30
KRATOS 5cc7e7a564 feat(node): persist upstream mirror transition state
Origin-Session: local-d6a143 | Codex | 12 prompts
2026-08-14 21:28:17 +05:30
7 changed files with 1366 additions and 16 deletions

No files matched your search

+13
View File
@@ -359,6 +359,19 @@ GITLAWB_TRUSTED_PROXY=
# Enable automatic background sync from known peers
GITLAWB_AUTO_SYNC=false
# ── External upstream mirroring ───────────────────────────────────────────
# Default-off scheduled fetch worker for repositories configured as INBOUND.
# Enabling requires GITLAWB_ENFORCE_OWNER_PUSH=true; the node refuses to start
# otherwise. The worker updates branches and tags atomically while preserving
# Gitlawb-owned review/internal refs.
GITLAWB_UPSTREAM_MIRROR_ENABLED=false
# Pause after each complete scan. Must be 5..=86400 seconds.
GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS=300
# Per-repository fetch deadline. Must be 1..=3153600000 seconds.
GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS=600
# Keyset page size; every page is processed before the next scan delay.
GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE=100
# ── iCaptcha proof-of-intelligence gate ───────────────────────────────────
# Optional gate on create_repo + register: require callers to present an
# iCaptcha proof (X-ICaptcha-Proof header) earned at icaptcha.gitlawb.com.
+58 -1
View File
@@ -84,7 +84,7 @@ pub(crate) const PUBLIC_HTTP_URL_REQUIREMENT: &str = "must be a public http(s) U
/// sits at a prefix-length-dependent offset (RFC 6052 §2.2) — so they return
/// `None`. Any caller that needs them blocked must reject the wider
/// `64:ff9b::/32` itself; `is_public_http_url` does this in its native-v6 arm.
fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
pub(crate) fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) {
return Some(v4);
}
@@ -104,6 +104,63 @@ fn embedded_ipv4(v6: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
None
}
/// Whether an IP literal is outside the local/private ranges that an
/// attacker-controlled outbound URL must never reach. Kept separate from URL
/// parsing so callers that resolve a DNS name themselves can apply the same
/// classification to every answer and then pin the approved addresses.
pub(crate) fn is_public_ip(ip: std::net::IpAddr) -> bool {
// Reject loopback/unspecified on the literal as given — catches `::1` and
// `::` before the IPv4-folding below (`::1`.to_ipv4() would otherwise map
// to a non-loopback `0.0.0.1`).
if ip.is_loopback() || ip.is_unspecified() {
return false;
}
// Fold any IPv6 literal that embeds an IPv4 address (mapped, compatible,
// 6to4, NAT64) down to that IPv4 so the v4 range checks catch
// loopback/private addresses smuggled in via an IPv6 encoding, then
// re-check loopback/unspecified.
let ip = match ip {
std::net::IpAddr::V6(v6) => embedded_ipv4(v6).map(std::net::IpAddr::V4).unwrap_or(ip),
v4 => v4,
};
if ip.is_loopback() || ip.is_unspecified() {
return false;
}
match ip {
std::net::IpAddr::V4(v4) => {
let o = v4.octets();
// RFC1918 private, link-local, CGNAT (100.64.0.0/10), or the
// RFC1122 "this host" block 0.0.0.0/8 (never a valid destination;
// 0.0.0.0 itself is already caught by the is_unspecified check).
if v4.is_private()
|| v4.is_link_local()
|| (o[0] == 100 && (o[1] & 0xc0) == 64)
|| o[0] == 0
{
return false;
}
}
std::net::IpAddr::V6(v6) => {
let s = v6.segments();
// fc00::/7 (unique-local) or fe80::/10 (link-local)
if (s[0] & 0xfe00) == 0xfc00 || (s[0] & 0xffc0) == 0xfe80 {
return false;
}
// Any NAT64 address (64:ff9b::/32) that is not the cleanly
// decodable well-known /96 — e.g. the RFC 8215 local-use
// 64:ff9b:1::/48 — carries a translated target whose embedded v4
// sits at a prefix-length-dependent offset (RFC 6052 §2.2).
// Rather than risk a wrong decode across every prefix length we
// reject the whole NAT64 space here. The well-known /96 was already
// folded to its v4 above and never reaches this arm.
if s[0] == 0x0064 && s[1] == 0xff9b {
return false;
}
}
}
true
}
/// Whether a peer `http_url` is a public http(s) endpoint safe to register.
/// Rejects non-http(s) schemes, loopback/unspecified/private/link-local IPs,
/// and `localhost` / `.localhost` / `.local` / `.internal` hostnames. Used at
+91
View File
@@ -152,6 +152,45 @@ pub struct Config {
#[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)]
pub auto_sync: bool,
/// Periodically fetch configured INBOUND repositories from their external
/// HTTPS upstream. Disabled by default: enabling this also requires
/// `GITLAWB_ENFORCE_OWNER_PUSH=true`, so an upstream-authoritative mirror
/// cannot accept writes from arbitrary signed identities between fetches.
#[arg(long, env = "GITLAWB_UPSTREAM_MIRROR_ENABLED", default_value_t = false)]
pub upstream_mirror_enabled: bool,
/// Pause between complete upstream-mirror scans. A scan is sequential and
/// paginated; the pause begins only after the prior scan has finished, so a
/// slow forge cannot create overlapping worker cycles.
#[arg(
long,
env = "GITLAWB_UPSTREAM_MIRROR_INTERVAL_SECS",
default_value_t = 300,
value_parser = clap::value_parser!(u64).range(5..=86_400)
)]
pub upstream_mirror_interval_secs: u64,
/// Maximum wall-clock time for one external `git fetch`. The child runs in
/// its own process group and is fully reaped on timeout.
#[arg(
long,
env = "GITLAWB_UPSTREAM_MIRROR_FETCH_TIMEOUT_SECS",
default_value_t = 600,
value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX)
)]
pub upstream_mirror_fetch_timeout_secs: u64,
/// Number of durable INBOUND rows read per database page. Every page is
/// processed before the scan sleeps; this bounds query memory without
/// permanently starving repositories beyond the first page.
#[arg(
long,
env = "GITLAWB_UPSTREAM_MIRROR_PAGE_SIZE",
default_value_t = 100,
value_parser = clap::builder::RangedU64ValueParser::<usize>::new().range(1..=1_000)
)]
pub upstream_mirror_page_size: usize,
/// Irys URL for Arweave permanent anchoring.
/// Leave empty to disable. Use https://devnet.irys.xyz for free devnet.
#[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")]
@@ -832,6 +871,13 @@ impl Config {
Self::DB_LOCK_POOL_NON_PUSH_HEADROOM
));
}
if self.upstream_mirror_enabled && !self.enforce_owner_push {
return Err(
"GITLAWB_UPSTREAM_MIRROR_ENABLED requires GITLAWB_ENFORCE_OWNER_PUSH=true: \
an upstream-authoritative mirror must not accept writes from arbitrary signed identities"
.to_string(),
);
}
let main_floor = Self::DB_POOL_APP_HEADROOM as u64;
if (self.db_max_connections as u64) < main_floor {
return Err(format!(
@@ -1031,6 +1077,51 @@ pub(crate) fn validate_web_url(raw: &str) -> Result<(), String> {
mod tests {
use super::*;
#[test]
fn upstream_mirror_worker_is_default_off_and_bounded() {
let mut config = Config::parse_from(["gitlawb-node"]);
assert!(!config.upstream_mirror_enabled);
assert_eq!(config.upstream_mirror_interval_secs, 300);
assert_eq!(config.upstream_mirror_fetch_timeout_secs, 600);
assert_eq!(config.upstream_mirror_page_size, 100);
config.validate().unwrap();
assert!(
Config::try_parse_from(["gitlawb-node", "--upstream-mirror-interval-secs", "4",])
.is_err()
);
assert!(
Config::try_parse_from(["gitlawb-node", "--upstream-mirror-page-size", "0",]).is_err()
);
}
#[test]
fn upstream_mirror_worker_requires_owner_push_enforcement() {
// enforce_owner_push defaults true on main, so the only unsafe shape
// is the explicit opt-out: mirror enabled plus enforcement refused.
let mut unsafe_config = Config::parse_from([
"gitlawb-node",
"--upstream-mirror-enabled",
"--enforce-owner-push",
"false",
]);
let error = unsafe_config.validate().unwrap_err();
assert!(error.contains("GITLAWB_ENFORCE_OWNER_PUSH=true"));
Config::parse_from([
"gitlawb-node",
"--upstream-mirror-enabled",
"--enforce-owner-push",
"true",
])
.validate()
.unwrap();
// The default posture alone also validates: enforcement is on.
Config::parse_from(["gitlawb-node", "--upstream-mirror-enabled"])
.validate()
.unwrap();
}
#[test]
fn lock_pool_size_defaults_to_40_and_rejects_zero() {
assert_eq!(
+116 -1
View File
@@ -117,6 +117,17 @@ pub struct RepoMirrorState {
pub updated_at: DateTime<Utc>,
}
/// One canonical repository eligible for a scheduled external-forge fetch.
/// The worker keyset-pages these rows by `repo_id`; transition metadata is not
/// exposed because the selecting query admits only stable INBOUND state.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InboundMirrorTarget {
pub repo_id: String,
pub owner_did: String,
pub name: String,
pub upstream_url: String,
}
#[cfg_attr(not(test), allow(dead_code))]
impl RepoMirrorState {
fn validate(&self) -> Result<()> {
@@ -136,7 +147,7 @@ impl RepoMirrorState {
}
#[cfg_attr(not(test), allow(dead_code))]
fn validate_mirror_upstream_url(raw: &str) -> Result<reqwest::Url> {
pub(crate) fn validate_mirror_upstream_url(raw: &str) -> Result<reqwest::Url> {
if raw.is_empty() || raw.len() > 2048 {
anyhow::bail!("mirror upstream URL must contain 1 to 2048 bytes");
}
@@ -1609,6 +1620,51 @@ impl Db {
row.map(row_to_repo_mirror_state).transpose()
}
/// Keyset-page stable INBOUND mirrors for the external-forge worker.
/// Transitioning/outbound rows are excluded at the database boundary so a
/// cycle cannot intentionally schedule them. The worker rechecks the state
/// after taking the repo write lock to close the selection-to-fetch window.
pub async fn list_inbound_mirror_targets(
&self,
after_repo_id: Option<&str>,
limit: i64,
) -> Result<Vec<InboundMirrorTarget>> {
let rows = sqlx::query(
"SELECT id, owner_did, name, upstream_url
FROM repos
WHERE mirror_status = 'inbound'
AND upstream_url IS NOT NULL
AND mirror_transition_id IS NULL
AND mirror_transition_phase IS NULL
AND ($1::text IS NULL OR id > $1)
ORDER BY id ASC
LIMIT $2",
)
.bind(after_repo_id)
.bind(limit.clamp(1, 1_000))
.fetch_all(&self.pool)
.await?;
rows.into_iter()
.map(|row| {
Ok(InboundMirrorTarget {
repo_id: row
.try_get("id")
.context("reading inbound mirror repository id")?,
owner_did: row
.try_get("owner_did")
.context("reading inbound mirror owner DID")?,
name: row
.try_get("name")
.context("reading inbound mirror repository name")?,
upstream_url: row
.try_get("upstream_url")
.context("reading inbound mirror upstream URL")?,
})
})
.collect()
}
/// Register a mirrored repo from a peer in the local DB so git smart HTTP can serve it.
/// Uses INSERT OR IGNORE (SQLite) / ON CONFLICT DO NOTHING (Postgres) so it's idempotent.
pub async fn upsert_mirror_repo(
@@ -4770,6 +4826,65 @@ mod mirror_state_tests {
assert_eq!(stored.updated_at, original_updated_at);
}
#[sqlx::test]
async fn inbound_worker_pages_all_and_only_stable_inbound_mirrors(pool: PgPool) {
let db = migrated_db(pool).await;
let ids = [
Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap(),
Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap(),
Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap(),
];
for (index, id) in ids.into_iter().enumerate() {
let mut existing = repo(id);
existing.name = format!("mirror-{index}");
db.create_repo(&existing).await.unwrap();
db.configure_inbound_mirror(
&existing.id,
&format!("https://github.example/org/mirror-{index}.git"),
)
.await
.unwrap();
}
// The middle row simulates a transition selected by the next B1 slice.
// It must not be scheduled while authority is changing.
sqlx::query(
"UPDATE repos
SET mirror_status = 'transitioning_inbound_to_outbound',
mirror_transition_id = $2,
mirror_transition_phase = 'queued'
WHERE id = $1",
)
.bind(ids[1].to_string())
.bind(Uuid::new_v4())
.execute(db.pool())
.await
.unwrap();
let first = db.list_inbound_mirror_targets(None, 1).await.unwrap();
assert_eq!(first.len(), 1);
assert_eq!(first[0].repo_id, ids[0].to_string());
assert_eq!(first[0].name, "mirror-0");
assert_eq!(first[0].owner_did, "did:key:z6MkMirrorOwner");
assert_eq!(
first[0].upstream_url,
"https://github.example/org/mirror-0.git"
);
let second = db
.list_inbound_mirror_targets(Some(&first[0].repo_id), 1)
.await
.unwrap();
assert_eq!(second.len(), 1);
assert_eq!(second[0].repo_id, ids[2].to_string());
let exhausted = db
.list_inbound_mirror_targets(Some(&second[0].repo_id), 1)
.await
.unwrap();
assert!(exhausted.is_empty());
}
#[sqlx::test]
async fn malformed_mirror_row_returns_an_error_instead_of_panicking(pool: PgPool) {
let row = sqlx::query(
+101 -14
View File
@@ -31,6 +31,53 @@ const WALK_TIMEOUT: Duration = Duration::from_secs(600);
#[cfg(unix)]
const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1);
/// Build a Git child command, optionally with an isolated environment for an
/// owner-selected HTTPS destination. The isolated form intentionally drops
/// ambient proxy, credential-helper, askpass, and Git config environment that
/// could otherwise forward node-operator secrets to the selected upstream.
fn git_child_command(
git_bin: &str,
args: &[&str],
repo_path: &Path,
isolated_https: bool,
isolated_config: &[(&str, &str)],
) -> std::process::Command {
let mut command = std::process::Command::new(git_bin);
command
.args(args)
.current_dir(repo_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if isolated_https {
let path = std::env::var_os("PATH");
command.env_clear();
if let Some(path) = path {
command.env("PATH", path);
}
command
.env("GIT_CONFIG_NOSYSTEM", "1")
.env(
"GIT_CONFIG_GLOBAL",
if cfg!(windows) { "NUL" } else { "/dev/null" },
)
.env("GIT_TERMINAL_PROMPT", "0")
.env("GIT_ASKPASS", "false")
.env("SSH_ASKPASS", "false")
.env("LANG", "C")
.env("LC_ALL", "C");
if !isolated_config.is_empty() {
command.env("GIT_CONFIG_COUNT", isolated_config.len().to_string());
for (index, (key, value)) in isolated_config.iter().enumerate() {
command
.env(format!("GIT_CONFIG_KEY_{index}"), key)
.env(format!("GIT_CONFIG_VALUE_{index}"), value);
}
}
}
command
}
/// Run one git child under a shared `deadline` with process-group teardown,
/// BLOCKING, and return its stdout. The child runs in its own process group; a
/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs,
@@ -69,24 +116,22 @@ fn child_terminated_without_reaping(pid: i32) -> bool {
}
#[cfg(unix)]
pub(crate) fn run_bounded_git_raw(
fn run_bounded_git_raw_impl(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
isolated_https: bool,
isolated_config: &[(&str, &str)],
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
use std::io::{Read, Write};
use std::os::unix::process::CommandExt;
use std::sync::mpsc::RecvTimeoutError;
let label = args.first().copied().unwrap_or("git");
let mut child = std::process::Command::new(git_bin)
.args(args)
.current_dir(repo_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
let mut command = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config);
let mut child = command
.process_group(0)
.spawn()
.with_context(|| format!("failed to spawn git {label}"))?;
@@ -196,6 +241,51 @@ pub(crate) fn run_bounded_git_raw(
Ok((status, out, err))
}
pub(crate) fn run_bounded_git_raw(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, false, &[])
}
/// Run a bounded Git child without inheriting ambient Git/proxy/credential
/// configuration. Reserved for owner-selected external HTTPS destinations;
/// ordinary local Git operations retain their existing environment.
pub(crate) fn run_bounded_git_raw_isolated_https(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
run_bounded_git_raw_impl(git_bin, args, repo_path, stdin_bytes, deadline, true, &[])
}
/// Isolated HTTPS runner with command-scope Git configuration supplied as
/// separate key/value environment entries. Unlike `git -c key=value`, this
/// preserves valid URL subsection keys whose path contains `=`.
pub(crate) fn run_bounded_git_raw_isolated_https_with_config(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
config: &[(&str, &str)],
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
run_bounded_git_raw_impl(
git_bin,
args,
repo_path,
stdin_bytes,
deadline,
true,
config,
)
}
/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin
/// wrapper the walk callers use. Probes that must distinguish exit classes —
/// `git cat-file` absence vs an object-store access failure — call
@@ -229,23 +319,20 @@ pub(crate) fn run_bounded_git(
/// the Unix version's signature and result semantics so every caller compiles on all
/// targets (#174).
#[cfg(not(unix))]
pub(crate) fn run_bounded_git_raw(
fn run_bounded_git_raw_impl(
git_bin: &str,
args: &[&str],
repo_path: &Path,
stdin_bytes: &[u8],
deadline: Instant,
isolated_https: bool,
isolated_config: &[(&str, &str)],
) -> Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>)> {
use std::io::{Read, Write};
use std::sync::mpsc::RecvTimeoutError;
let label = args.first().copied().unwrap_or("git");
let mut child = std::process::Command::new(git_bin)
.args(args)
.current_dir(repo_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
let mut child = git_child_command(git_bin, args, repo_path, isolated_https, isolated_config)
.spawn()
.with_context(|| format!("failed to spawn git {label}"))?;
+17
View File
@@ -23,6 +23,7 @@ mod state;
mod sync;
#[cfg(test)]
mod test_support;
mod upstream_mirror;
mod visibility;
mod webhooks;
@@ -666,6 +667,22 @@ async fn main() -> Result<()> {
info!("auto-sync worker started");
}
// External-forge mirroring is a separate, default-off worker. Startup
// validation already requires owner-only pushes when it is enabled; the
// worker additionally verifies the Git runtime can pin DNS answers before
// any URL reaches `git fetch`.
if config.upstream_mirror_enabled {
upstream_mirror::start(
Arc::clone(&state.db),
Arc::clone(&state.config),
state.repo_store.clone(),
state.repo_write_leases.clone(),
Arc::clone(&state.git_write_semaphore),
state.subscribe_shutdown(),
)
.context("starting upstream mirror worker")?;
}
// On-chain operator setup: verify stake + spawn heartbeat loop
if !state.config.contract_node_staking.is_empty()
&& !state.config.operator_private_key.is_empty()
+970
View File
@@ -0,0 +1,970 @@
//! Scheduled external-forge fetches for canonical INBOUND mirrors.
//!
//! This worker is deliberately separate from `sync`, which mirrors repositories
//! between Gitlawb peers. External upstream URLs are owner-controlled data that
//! eventually reach `git fetch`, so each cycle resolves DNS itself, validates
//! every address, and pins the approved answers into Git/libcurl with
//! `http.curloptResolve`. Redirects, proxies, credential helpers, submodule
//! recursion, and non-HTTPS protocols are disabled at the command boundary.
use anyhow::{Context, Result};
use std::collections::BTreeSet;
use std::net::IpAddr;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tracing::{info, warn};
use crate::config::Config;
use crate::db::{Db, InboundMirrorTarget, MirrorStatus};
use crate::git::repo_store::RepoStore;
use crate::state::{repo_identity_key, RepoWriteLeases};
const DNS_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_PINNED_ADDRESSES: usize = 32;
const MIN_CURL_OPT_RESOLVE_GIT: (u64, u64) = (2, 37);
#[derive(Debug, Clone, Copy)]
struct EgressPolicy;
#[derive(Debug, Clone, PartialEq, Eq)]
struct PinnedUpstream {
url: String,
curlopt_resolve: Option<String>,
}
impl EgressPolicy {
async fn resolve(&self, url: reqwest::Url) -> Result<PinnedUpstream> {
let raw_host = url.host_str().context("mirror upstream URL has no host")?;
let connect_host = raw_host
.trim_start_matches('[')
.trim_end_matches(']')
.to_ascii_lowercase();
let policy_host = normalize_policy_host(&connect_host);
reject_always_local_hostname(&policy_host)?;
let port = url
.port_or_known_default()
.context("mirror upstream URL has no known port")?;
if let Ok(ip) = connect_host.parse::<IpAddr>() {
self.validate_addresses(&policy_host, &[ip])?;
return Ok(PinnedUpstream {
url: url.to_string(),
// An IP literal performs no DNS lookup, so there is no rebinding
// window to pin. TLS still verifies the literal from the URL.
curlopt_resolve: None,
});
}
let resolved = tokio::time::timeout(
DNS_TIMEOUT,
tokio::net::lookup_host((connect_host.as_str(), port)),
)
.await
.context("mirror upstream DNS lookup timed out")?
.context("resolving mirror upstream host")?;
let addresses: BTreeSet<IpAddr> = resolved.map(|addr| addr.ip()).collect();
if addresses.is_empty() {
anyhow::bail!("mirror upstream host resolved to no addresses");
}
let addresses: Vec<IpAddr> = addresses.into_iter().collect();
self.validate_addresses(&policy_host, &addresses)?;
let pinned = addresses
.iter()
.map(|ip| match ip {
IpAddr::V4(v4) => v4.to_string(),
IpAddr::V6(v6) => format!("[{v6}]"),
})
.collect::<Vec<_>>()
.join(",");
Ok(PinnedUpstream {
url: url.to_string(),
curlopt_resolve: Some(format!("+{connect_host}:{port}:{pinned}")),
})
}
fn validate_addresses(&self, host: &str, addresses: &[IpAddr]) -> Result<()> {
if addresses.is_empty() || addresses.len() > MAX_PINNED_ADDRESSES {
anyhow::bail!(
"mirror upstream host must resolve to 1..={MAX_PINNED_ADDRESSES} addresses"
);
}
for &ip in addresses {
if !address_is_permitted(ip) {
anyhow::bail!("mirror upstream host {host:?} resolved to disallowed address {ip}");
}
}
Ok(())
}
}
fn normalize_policy_host(host: &str) -> String {
host.trim_end_matches('.').to_ascii_lowercase()
}
fn reject_always_local_hostname(host: &str) -> Result<()> {
if host.is_empty()
|| host == "localhost"
|| host.ends_with(".localhost")
|| host.ends_with(".local")
|| host.ends_with(".internal")
{
anyhow::bail!("mirror upstream host is local-only and cannot be fetched");
}
Ok(())
}
fn address_is_permitted(ip: IpAddr) -> bool {
if !crate::api::peers::is_public_ip(ip) {
return false;
}
let classified = match ip {
IpAddr::V6(v6) => crate::api::peers::embedded_ipv4(v6)
.map(IpAddr::V4)
.unwrap_or(ip),
_ => ip,
};
match classified {
IpAddr::V4(v4) => {
let [a, b, c, _] = v4.octets();
// IANA special-purpose ranges that are not globally reachable but
// may be routed inside labs, clouds, or operator networks. They are
// unsuitable for an owner-controlled outbound fetch even though
// `Ipv4Addr::is_private` does not classify them as RFC1918.
!(a >= 224
|| (a == 192 && b == 0 && c == 0)
|| (a == 192 && b == 0 && c == 2)
|| (a == 192 && b == 88 && c == 99)
|| (a == 198 && matches!(b, 18 | 19))
|| (a == 198 && b == 51 && c == 100)
|| (a == 203 && b == 0 && c == 113))
}
IpAddr::V6(v6) => {
let s = v6.segments();
// IANA special-purpose ranges that are non-global, locally
// meaningful, or unsafe to treat as ordinary forge endpoints.
// Reject the complete IETF protocol-assignment /23 rather than
// trying to maintain its narrow globally reachable exceptions;
// a public forge has no reason to depend on those anycast ranges.
!(v6.is_multicast()
|| (s[0] & 0xffc0) == 0xfec0
|| (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 0)
|| (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 1)
|| (s[0] == 0x2001 && (s[1] & 0xfe00) == 0)
|| (s[0] == 0x2001 && s[1] == 0x0db8)
|| (s[0] == 0x3fff && (s[1] & 0xf000) == 0)
|| s[0] == 0x5f00)
}
}
}
fn parse_git_version(output: &str) -> Option<(u64, u64)> {
let version = output.split_whitespace().find(|part| {
part.as_bytes()
.first()
.is_some_and(|byte| byte.is_ascii_digit())
})?;
let mut parts = version.split('.');
Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?))
}
fn ensure_safe_git_runtime(git_bin: &str) -> Result<()> {
// Bounded runner like every other git spawn on this path: --version does
// no I/O of its own, but the child still takes a permit and the deadline.
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
git_bin,
&["--version"],
&std::env::temp_dir(),
b"",
Instant::now() + Duration::from_secs(10),
)?;
if !status.success() {
anyhow::bail!(
"{git_bin} --version failed: {}",
String::from_utf8_lossy(&stderr)
.chars()
.take(256)
.collect::<String>()
);
}
let stdout = String::from_utf8_lossy(&stdout);
let version = parse_git_version(&stdout).context("parsing git version")?;
if version < MIN_CURL_OPT_RESOLVE_GIT {
anyhow::bail!(
"upstream mirroring requires Git 2.37+ for DNS pinning; found {}.{}",
version.0,
version.1
);
}
Ok(())
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct FetchCommand {
config: Vec<(String, String)>,
args: Vec<String>,
}
fn fetch_command(upstream: &PinnedUpstream) -> FetchCommand {
let http_scope = |name: &str| format!("http.{}.{name}", upstream.url);
let credential_scope = |name: &str| format!("credential.{}.{name}", upstream.url);
let mut config = vec![
("http.proxy".to_string(), "".to_string()),
("http.followRedirects".to_string(), "false".to_string()),
// Reset any inherited multi-valued resolver entries before adding the
// address set validated for this exact fetch.
("http.curloptResolve".to_string(), "".to_string()),
// Never forward operator or repository-scoped HTTP credentials to an
// owner-selected forge. Empty values reset Git's multi-valued headers
// and disable cookie state for this invocation.
("http.extraHeader".to_string(), "".to_string()),
("http.cookieFile".to_string(), "".to_string()),
("http.saveCookies".to_string(), "false".to_string()),
("http.sslVerify".to_string(), "true".to_string()),
// URL-specific configuration outranks generic `http.*` values even
// when the generic value is command-scoped. Repeat every security
// control at the exact validated URL so a repository-local scoped
// setting cannot re-enable redirects/proxies/headers/cookies or disable
// TLS. Key/value environment entries avoid `git -c` splitting a valid
// URL path that contains `=`.
(http_scope("proxy"), "".to_string()),
(http_scope("followRedirects"), "false".to_string()),
(http_scope("curloptResolve"), "".to_string()),
(http_scope("extraHeader"), "".to_string()),
(http_scope("cookieFile"), "".to_string()),
(http_scope("saveCookies"), "false".to_string()),
(http_scope("sslVerify"), "true".to_string()),
];
if let Some(resolve) = &upstream.curlopt_resolve {
config.extend([
("http.curloptResolve".to_string(), resolve.clone()),
(http_scope("curloptResolve"), resolve.clone()),
]);
}
config.extend([
("credential.helper".to_string(), "".to_string()),
(credential_scope("helper"), "".to_string()),
("credential.interactive".to_string(), "false".to_string()),
("core.askPass".to_string(), "false".to_string()),
("fetch.recurseSubmodules".to_string(), "false".to_string()),
// Do not let a smart server or repository-local config introduce a
// second, unvalidated pack/bundle URL outside the pinned upstream.
("fetch.uriprotocols".to_string(), "".to_string()),
("fetch.bundleURI".to_string(), "".to_string()),
("protocol.allow".to_string(), "never".to_string()),
("protocol.https.allow".to_string(), "always".to_string()),
]);
let args = vec![
"fetch".to_string(),
"--atomic".to_string(),
"--force".to_string(),
"--prune".to_string(),
"--prune-tags".to_string(),
"--no-auto-maintenance".to_string(),
"--no-recurse-submodules".to_string(),
"--no-write-fetch-head".to_string(),
upstream.url.clone(),
"+refs/heads/*:refs/heads/*".to_string(),
"+refs/tags/*:refs/tags/*".to_string(),
];
FetchCommand { config, args }
}
fn reject_local_url_rewrite(
git_bin: &str,
repo_path: &Path,
upstream_url: &str,
deadline: Instant,
) -> Result<()> {
let args = ["ls-remote", "--get-url", upstream_url];
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
git_bin, &args, repo_path, b"", deadline,
)?;
if !status.success() {
let stderr = String::from_utf8_lossy(&stderr);
anyhow::bail!(
"checking mirror upstream URL rewrite failed: {}",
stderr.chars().take(4_096).collect::<String>()
);
}
let expanded = std::str::from_utf8(&stdout)
.context("Git returned a non-UTF-8 expanded mirror upstream URL")?
.trim_end_matches(['\r', '\n']);
if expanded != upstream_url {
anyhow::bail!("repository-local Git config attempted to rewrite the mirror upstream URL");
}
Ok(())
}
fn reject_sensitive_local_config(git_bin: &str, repo_path: &Path, deadline: Instant) -> Result<()> {
let args = [
"config",
"--local",
"--includes",
"--get-regexp",
r"^(http|credential|url)\.",
];
let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw_isolated_https(
git_bin, &args, repo_path, b"", deadline,
)?;
match status.code() {
Some(1) if stdout.is_empty() => Ok(()),
Some(0) => anyhow::bail!(
"repository-local network or credential Git config is not allowed for upstream mirrors"
),
_ => {
let stderr = String::from_utf8_lossy(&stderr);
anyhow::bail!(
"checking repository-local Git config failed: {}",
stderr.chars().take(4_096).collect::<String>()
)
}
}
}
fn run_fetch(
git_bin: &str,
repo_path: &Path,
upstream: &PinnedUpstream,
timeout: Duration,
) -> Result<()> {
let command = fetch_command(upstream);
let args: Vec<&str> = command.args.iter().map(String::as_str).collect();
let config: Vec<(&str, &str)> = command
.config
.iter()
.map(|(key, value)| (key.as_str(), value.as_str()))
.collect();
let deadline = Instant::now()
.checked_add(timeout)
.context("upstream mirror fetch timeout is not representable")?;
reject_local_url_rewrite(git_bin, repo_path, &upstream.url, deadline)?;
reject_sensitive_local_config(git_bin, repo_path, deadline)?;
let (status, _stdout, stderr) =
crate::git::visibility_pack::run_bounded_git_raw_isolated_https_with_config(
git_bin, &args, repo_path, b"", deadline, &config,
)?;
if !status.success() {
let stderr = String::from_utf8_lossy(&stderr);
let stderr = stderr.chars().take(4_096).collect::<String>();
anyhow::bail!("git upstream fetch failed: {stderr}");
}
Ok(())
}
#[derive(Clone)]
struct Worker {
db: Arc<Db>,
config: Arc<Config>,
repo_store: RepoStore,
repo_write_leases: RepoWriteLeases,
git_write_semaphore: Arc<tokio::sync::Semaphore>,
policy: EgressPolicy,
git_bin: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FetchOutcome {
Updated,
SkippedStale,
}
impl Worker {
async fn fetch_target(&self, target: InboundMirrorTarget) -> Result<FetchOutcome> {
let url = crate::db::validate_mirror_upstream_url(&target.upstream_url)?;
let upstream = self.policy.resolve(url).await?;
let repo_identity = repo_identity_key(&target.owner_did, &target.name);
let steal_after = Duration::from_secs(
self.config
.upstream_mirror_fetch_timeout_secs
.saturating_mul(2)
.saturating_add(60),
);
let _lease = self
.repo_write_leases
.acquire(&repo_identity, steal_after)
.await
.context("upstream mirror write-lease waiter cap reached")?;
let _write_permit = Arc::clone(&self.git_write_semaphore)
.acquire_owned()
.await
.context("upstream mirror write semaphore closed")?;
let acquire_timeout = Duration::from_secs(self.config.git_acquire_timeout_secs);
let guard = tokio::time::timeout(
acquire_timeout,
self.repo_store
.acquire_write(&target.owner_did, &target.name),
)
.await
.context("upstream mirror write-lock acquisition timed out")??;
let repo_path = guard.path().to_path_buf();
// A row can leave INBOUND state after selection. Recheck only after
// both the process-local lease and the cluster-wide advisory write lock
// are held. Every future transition path must take these same locks
// before changing authority, so no other node can commit a transition
// between this check and the fetch.
let current = match self.db.get_repo_mirror_state(&target.repo_id).await {
Ok(Some(current)) => current,
Ok(None) => {
let _ = guard.release(false).await;
anyhow::bail!("inbound mirror state disappeared before fetch");
}
Err(error) => {
let _ = guard.release(false).await;
return Err(error);
}
};
if current.status != MirrorStatus::Inbound
|| current.transition_id.is_some()
|| current.transition_phase.is_some()
|| current.upstream_url != target.upstream_url
{
let _ = guard.release(false).await;
info!(repo_id = %target.repo_id, "upstream mirror state changed before fetch; skipping stale target");
return Ok(FetchOutcome::SkippedStale);
}
let git_bin = self.git_bin.clone();
let fetch_timeout = Duration::from_secs(self.config.upstream_mirror_fetch_timeout_secs);
let fetch = tokio::task::spawn_blocking(move || {
run_fetch(&git_bin, &repo_path, &upstream, fetch_timeout)
})
.await
.context("upstream mirror fetch task panicked")?;
let success = fetch.is_ok();
let outcome = guard.release(success).await;
// A refused publish (fenced or unknowable upload) must not report
// Updated; surface it as the failure it is before mapping the fetch.
outcome.into_result()?;
fetch.map(|()| FetchOutcome::Updated)
}
async fn scan_once(&self) {
let mut cursor: Option<String> = None;
loop {
let targets = match self
.db
.list_inbound_mirror_targets(
cursor.as_deref(),
self.config.upstream_mirror_page_size as i64,
)
.await
{
Ok(targets) => targets,
Err(error) => {
warn!(err = %error, "failed to list inbound mirror targets");
return;
}
};
if targets.is_empty() {
return;
}
let page_len = targets.len();
cursor = targets.last().map(|target| target.repo_id.clone());
for target in targets {
let repo_id = target.repo_id.clone();
let upstream_url = target.upstream_url.clone();
match self.fetch_target(target).await {
Ok(FetchOutcome::Updated) => {
info!(repo_id, upstream = %upstream_url, "upstream mirror fetch completed")
}
Ok(FetchOutcome::SkippedStale) => {}
Err(error) => {
warn!(repo_id, upstream = %upstream_url, err = %error, "upstream mirror fetch failed")
}
}
}
if page_len < self.config.upstream_mirror_page_size {
return;
}
}
}
}
/// Validate the opt-in runtime and spawn the scheduled worker. The worker runs
/// one scan immediately, sleeps only after a complete scan, and exits between
/// scans when graceful shutdown is signalled. An in-flight Git child is bounded
/// and reaped before the worker observes shutdown.
pub fn start(
db: Arc<Db>,
config: Arc<Config>,
repo_store: RepoStore,
repo_write_leases: RepoWriteLeases,
git_write_semaphore: Arc<tokio::sync::Semaphore>,
mut shutdown_rx: tokio::sync::watch::Receiver<bool>,
) -> Result<()> {
ensure_safe_git_runtime("git")?;
let policy = EgressPolicy;
let worker = Worker {
db,
config: Arc::clone(&config),
repo_store,
repo_write_leases,
git_write_semaphore,
policy,
git_bin: "git".to_string(),
};
tokio::spawn(async move {
info!(
interval_secs = config.upstream_mirror_interval_secs,
"upstream mirror worker started"
);
loop {
worker.scan_once().await;
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(config.upstream_mirror_interval_secs)) => {}
changed = shutdown_rx.changed() => {
if changed.is_err() || *shutdown_rx.borrow() {
info!("upstream mirror worker stopped");
return;
}
}
}
}
});
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
use std::path::PathBuf;
async fn test_worker(
pool: sqlx::PgPool,
git_bin: PathBuf,
page_size: usize,
) -> (Worker, Arc<Db>, tempfile::TempDir) {
let db = Arc::new(Db::for_testing(pool.clone()));
db.run_migrations().await.unwrap();
let repos = tempfile::tempdir().unwrap();
let mut config = Config::parse_from([
"gitlawb-node".to_string(),
"--upstream-mirror-enabled".to_string(),
"--enforce-owner-push".to_string(),
"--upstream-mirror-fetch-timeout-secs".to_string(),
"5".to_string(),
"--upstream-mirror-page-size".to_string(),
page_size.to_string(),
]);
config.validate().unwrap();
let config = Arc::new(config);
let worker = Worker {
db: Arc::clone(&db),
config,
repo_store: RepoStore::for_testing(repos.path().to_path_buf(), pool),
repo_write_leases: RepoWriteLeases::new(8),
git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(4)),
policy: EgressPolicy,
git_bin: git_bin.to_string_lossy().into_owned(),
};
(worker, db, repos)
}
async fn add_inbound_mirror(
worker: &Worker,
db: &Db,
id: &str,
name: &str,
upstream_url: &str,
) -> InboundMirrorTarget {
let now = chrono::Utc::now();
let record = crate::db::RepoRecord {
id: id.to_string(),
name: name.to_string(),
owner_did: "did:key:z6MkMirrorWorker".to_string(),
description: None,
is_public: true,
default_branch: "main".to_string(),
created_at: now,
updated_at: now,
disk_path: format!("/test/{id}.git"),
forked_from: None,
machine_id: None,
};
db.create_repo(&record).await.unwrap();
worker
.repo_store
.init(&record.owner_did, &record.name)
.await
.unwrap();
let state = db
.configure_inbound_mirror(&record.id, upstream_url)
.await
.unwrap();
InboundMirrorTarget {
repo_id: record.id,
owner_did: record.owner_did,
name: record.name,
upstream_url: state.upstream_url,
}
}
#[cfg(unix)]
fn fake_git(repos: &tempfile::TempDir) -> (PathBuf, PathBuf) {
use std::os::unix::fs::PermissionsExt;
let git = repos.path().join("fake-git");
let calls = repos.path().join("fetch-calls");
let script = format!(
r#"#!/bin/sh
if [ "$1" = "ls-remote" ]; then
printf '%s\n' "$3"
exit 0
fi
if [ "$1" = "config" ]; then
# Match real Git when no sensitive repository-local keys are configured.
exit 1
fi
case "$*" in
*fail.git*) printf '%s\n' fail >> '{}'; exit 7 ;;
*) printf '%s\n' ok >> '{}'; exit 0 ;;
esac
"#,
calls.display(),
calls.display()
);
std::fs::write(&git, script).unwrap();
let mut permissions = std::fs::metadata(&git).unwrap().permissions();
permissions.set_mode(0o700);
std::fs::set_permissions(&git, permissions).unwrap();
(git, calls)
}
#[test]
fn git_version_gate_requires_curlopt_resolve_support() {
assert_eq!(parse_git_version("git version 2.37.0"), Some((2, 37)));
assert_eq!(
parse_git_version("git version 2.39.5 (Apple Git-154)"),
Some((2, 39))
);
assert_eq!(parse_git_version("not git"), None);
assert!((2, 36) < MIN_CURL_OPT_RESOLVE_GIT);
}
#[test]
fn every_resolved_address_must_be_public() {
let policy = EgressPolicy;
let public_v4: IpAddr = "8.8.8.8".parse().unwrap();
let private_v4: IpAddr = "10.2.3.4".parse().unwrap();
let private_v6: IpAddr = "fd00::20".parse().unwrap();
assert!(policy
.validate_addresses("public.example", &[public_v4])
.is_ok());
assert!(policy
.validate_addresses("private.example", &[private_v4, private_v6])
.is_err());
assert!(policy
.validate_addresses("mixed.example", &[public_v4, private_v4])
.is_err());
let too_many = (1..=MAX_PINNED_ADDRESSES + 1)
.map(|last| IpAddr::V4(std::net::Ipv4Addr::new(8, 8, 0, last as u8)))
.collect::<Vec<_>>();
assert!(policy
.validate_addresses("oversized.example", &too_many)
.is_err());
for never in [
"127.0.0.1",
"169.254.169.254",
"192.0.2.1",
"198.18.0.1",
"198.51.100.1",
"203.0.113.10",
"::1",
"::ffff:198.18.0.1",
"2002:c612:1::",
"64:ff9b::c612:1",
"100::1",
"100:0:0:1::1",
"2001::1",
"2001:2::1",
"2001:db8::1",
"fec0::1",
"fe80::1",
"3fff::1",
"5f00::1",
] {
let ip = never.parse().unwrap();
assert!(!address_is_permitted(ip), "{never} must stay denied");
}
}
#[test]
fn local_only_hostnames_are_always_rejected() {
for invalid in [
"",
"localhost",
"forge.localhost",
"forge.local",
"ghe.internal",
] {
assert!(
reject_always_local_hostname(invalid).is_err(),
"{invalid:?}"
);
}
assert!(reject_always_local_hostname("github.example").is_ok());
}
#[test]
fn fetch_command_pins_dns_and_only_updates_branches_and_tags() {
let upstream = PinnedUpstream {
url: "https://github.example/org/repo.git".to_string(),
curlopt_resolve: Some("+github.example:443:8.8.8.8,[2606:4700:4700::1111]".to_string()),
};
let command = fetch_command(&upstream);
let has_config = |key: &str, value: &str| {
command
.config
.contains(&(key.to_string(), value.to_string()))
};
assert!(has_config("http.followRedirects", "false"));
assert!(has_config("http.proxy", ""));
assert!(has_config("http.extraHeader", ""));
assert!(has_config("http.cookieFile", ""));
assert!(has_config("http.saveCookies", "false"));
assert!(has_config("http.sslVerify", "true"));
assert!(has_config(
"http.curloptResolve",
"+github.example:443:8.8.8.8,[2606:4700:4700::1111]"
));
assert!(has_config(
"http.https://github.example/org/repo.git.followRedirects",
"false"
));
assert!(has_config(
"http.https://github.example/org/repo.git.proxy",
""
));
assert!(has_config(
"http.https://github.example/org/repo.git.extraHeader",
""
));
assert!(has_config(
"http.https://github.example/org/repo.git.curloptResolve",
"+github.example:443:8.8.8.8,[2606:4700:4700::1111]"
));
assert!(has_config(
"credential.https://github.example/org/repo.git.helper",
""
));
assert!(has_config("protocol.allow", "never"));
assert!(has_config("protocol.https.allow", "always"));
assert!(has_config("fetch.uriprotocols", ""));
assert!(has_config("fetch.bundleURI", ""));
assert!(!command
.config
.iter()
.any(|(key, _)| key.ends_with("sslCert") || key.ends_with("sslKey")));
assert!(command.args.contains(&"--atomic".to_string()));
assert!(command
.args
.contains(&"+refs/heads/*:refs/heads/*".to_string()));
assert!(command
.args
.contains(&"+refs/tags/*:refs/tags/*".to_string()));
assert!(!command.args.iter().any(|arg| arg == "+refs/*:refs/*"));
}
#[test]
fn repository_local_instead_of_rewrite_is_rejected() {
let repo = tempfile::tempdir().unwrap();
let init = std::process::Command::new("git")
.args(["init", "--bare"])
.current_dir(repo.path())
.output()
.unwrap();
assert!(init.status.success());
let config = std::process::Command::new("git")
.args([
"config",
"url.https://evil.example/.insteadOf",
"https://github.example/",
])
.current_dir(repo.path())
.output()
.unwrap();
assert!(config.status.success());
let error = reject_local_url_rewrite(
"git",
repo.path(),
"https://github.example/org/repo.git",
Instant::now() + Duration::from_secs(5),
)
.unwrap_err();
assert!(error.to_string().contains("attempted to rewrite"));
}
#[test]
fn repository_local_network_and_credential_config_is_rejected() {
let repo = tempfile::tempdir().unwrap();
let init = std::process::Command::new("git")
.args(["init", "--bare"])
.current_dir(repo.path())
.output()
.unwrap();
assert!(init.status.success());
reject_sensitive_local_config("git", repo.path(), Instant::now() + Duration::from_secs(5))
.unwrap();
let config = std::process::Command::new("git")
.args(["config", "http.sslCert", "/tmp/untrusted-client.pem"])
.current_dir(repo.path())
.output()
.unwrap();
assert!(config.status.success());
let error = reject_sensitive_local_config(
"git",
repo.path(),
Instant::now() + Duration::from_secs(5),
)
.unwrap_err();
assert!(error.to_string().contains("network or credential"));
}
#[cfg(unix)]
#[test]
fn hardened_real_git_reaches_https_transport_for_equals_path() {
let repo = tempfile::tempdir().unwrap();
let init = std::process::Command::new("git")
.args(["init", "--bare"])
.current_dir(repo.path())
.output()
.unwrap();
assert!(init.status.success());
// This is intentionally a loopback test seam around `run_fetch`, below
// the egress-policy layer. Reaching the listener proves the generated
// Git config neither dies on an empty client certificate nor splits the
// valid `=` path at the command-config boundary. The listener closes
// immediately, so the expected final result is a TLS transport error.
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
listener.set_nonblocking(true).unwrap();
let accepted = std::thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(3);
while Instant::now() < deadline {
match listener.accept() {
Ok((_stream, _address)) => return true,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(Duration::from_millis(10));
}
Err(error) => panic!("accepting real-Git transport probe: {error}"),
}
}
false
});
let upstream = PinnedUpstream {
url: format!("https://127.0.0.1:{port}/org/repo=mirror.git"),
curlopt_resolve: None,
};
let error = run_fetch("git", repo.path(), &upstream, Duration::from_secs(5)).unwrap_err();
assert!(
accepted.join().unwrap(),
"real Git must reach the HTTPS transport; got {error:#}"
);
let error = error.to_string();
assert!(!error.contains("credential missing host field"));
assert!(!error.contains("invalid key"));
}
#[tokio::test]
async fn literal_addresses_are_checked_without_dns() {
let policy = EgressPolicy;
let allowed = policy
.resolve(reqwest::Url::parse("https://8.8.8.8/org/repo.git").unwrap())
.await
.unwrap();
assert_eq!(allowed.curlopt_resolve, None);
for denied in [
"127.0.0.1",
"10.2.3.4",
"169.254.169.254",
"192.0.2.1",
"198.18.0.1",
"203.0.113.10",
"[::1]",
"[100::1]",
"[100:0:0:1::1]",
"[2001::1]",
"[2001:db8::1]",
"[fec0::1]",
"[fd00::20]",
"[3fff::1]",
"[5f00::1]",
] {
let url = reqwest::Url::parse(&format!("https://{denied}/org/repo.git")).unwrap();
assert!(
policy.resolve(url).await.is_err(),
"{denied} must stay denied"
);
}
}
#[sqlx::test]
async fn stale_target_is_rechecked_under_the_write_locks(pool: sqlx::PgPool) {
let missing_git = PathBuf::from("/definitely/missing/git");
let (worker, db, _repos) = test_worker(pool, missing_git, 1).await;
let target = add_inbound_mirror(
&worker,
&db,
"00000000-0000-0000-0000-000000000001",
"stale",
"https://8.8.8.8/org/stale.git",
)
.await;
sqlx::query(
"UPDATE repos
SET mirror_status = 'outbound'
WHERE id = $1",
)
.bind(&target.repo_id)
.execute(db.pool())
.await
.unwrap();
let outcome = worker.fetch_target(target).await.unwrap();
assert_eq!(outcome, FetchOutcome::SkippedStale);
assert!(worker.repo_write_leases.is_empty());
}
#[cfg(unix)]
#[sqlx::test]
async fn scan_pages_past_one_failure_and_fetches_the_next_repo(pool: sqlx::PgPool) {
let bootstrap = tempfile::tempdir().unwrap();
let (git, calls) = fake_git(&bootstrap);
let (worker, db, _repos) = test_worker(pool, git, 1).await;
add_inbound_mirror(
&worker,
&db,
"00000000-0000-0000-0000-000000000001",
"fails",
"https://8.8.8.8/org/fail.git",
)
.await;
add_inbound_mirror(
&worker,
&db,
"00000000-0000-0000-0000-000000000002",
"succeeds",
"https://8.8.8.8/org/ok.git",
)
.await;
worker.scan_once().await;
let calls = std::fs::read_to_string(calls).unwrap();
assert_eq!(calls.lines().collect::<Vec<_>>(), ["fail", "ok"]);
assert!(worker.repo_write_leases.is_empty());
}
}