Trim comments and tidy the age filter

core/Cargo.toml

@@ -44,6 +44,10 @@ path = "tests/denylist.rs"
 name = "osv"
 path = "tests/osv.rs"
 
+[[test]]
+name = "age"
+path = "tests/age.rs"
+
 [[test]]
 name = "mitm"
 path = "tests/mitm.rs"
@@ -63,7 +67,7 @@ regex = "1.13.1"
 semver = "1.0.28"
 serde = { version = "1.0.229", features = ["derive"] }
 serde_json = "1.0.151"
-time = { version = "0.3.47", features = ["formatting"] }
+time = { version = "0.3.47", features = ["formatting", "parsing"] }
 tokio = { version = "1.53.1", features = ["io-util", "rt", "sync", "time"] }
 tracing = "0.1.44"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }

core/src/config/mod.rs

@@ -13,6 +13,8 @@ pub struct FirewallConfig {
     pub port: u16,
     /// Allow a package when a filter could not check it.
     pub fail_open: bool,
+    /// 0 turns the age check off.
+    pub min_age_hours: u64,
     pub denylist: DenylistSource,
     pub telemetry: bool,
     /// Not describable in the file yet, so this is always the built-in table.
@@ -43,6 +45,7 @@ impl Default for FirewallConfig {
         FirewallConfig {
             port: DEFAULT_PORT,
             fail_open: true,
+            min_age_hours: 0,
             denylist: DenylistSource::default(),
             telemetry: true,
             registries: BUILT_IN.to_vec(),

core/src/firewall.rs

@@ -1,4 +1,5 @@
 use crate::config::FirewallConfig;
+use crate::policies::filters::age::Age;
 use crate::policies::filters::denylist::Denylist;
 use crate::policies::filters::osv::Osv;
 use crate::policies::filters::Filter;
@@ -27,7 +28,10 @@ impl Firewall {
         }
         let ecosystems = config.registries.iter().map(|r| r.osv_name);
         let denylist = Denylist::new(config.denylist.clone(), config.denylist.token(), ecosystems);
-        let filters: Vec<Box<dyn Filter>> = vec![Box::new(denylist), Box::new(Osv)];
+        let mut filters: Vec<Box<dyn Filter>> = vec![Box::new(denylist), Box::new(Osv)];
+        if config.min_age_hours > 0 {
+            filters.push(Box::new(Age::new(config.min_age_hours)));
+        }
         Firewall { config, patterns, filters }
     }
 

core/src/policies/filters/age.rs

@@ -0,0 +1,72 @@
+use super::{Filter, Verdict, VerdictFuture, AGENT};
+use crate::policies::registries::{Published, Registry};
+use crate::policies::Package;
+use rama::http::client::EasyHttpWebClient;
+use rama::http::header::USER_AGENT;
+use rama::http::service::client::HttpClientExt;
+use rama::http::{BodyExtractExt, StatusCode};
+use serde_json::Value;
+use time::format_description::well_known::Rfc3339;
+use time::{Duration, OffsetDateTime};
+
+pub struct Age {
+    min: Duration,
+}
+
+impl Age {
+    pub fn new(hours: u64) -> Self {
+        Age { min: Duration::hours(hours as i64) }
+    }
+}
+
+impl Filter for Age {
+    fn name(&self) -> &'static str {
+        "age"
+    }
+
+    fn check<'a>(&'a self, registry: &'static Registry, pkg: &'a Package) -> VerdictFuture<'a> {
+        Box::pin(async move {
+            let Some(version) = pkg.version.as_deref() else { return Verdict::Allow };
+            let Some(published) = &registry.published else { return Verdict::Allow };
+            let published_at = match published_at(published, &pkg.name, version).await {
+                Ok(at) => at,
+                Err(e) => return Verdict::Unknown(e),
+            };
+
+            let age = OffsetDateTime::now_utc() - published_at;
+            if age >= self.min {
+                return Verdict::Allow;
+            }
+            let (hours, min) = (age.whole_hours(), self.min.whole_hours());
+            Verdict::Deny(format!("was published {hours} hours ago, under the {min}-hour minimum"))
+        })
+    }
+}
+
+async fn published_at(
+    published: &Published,
+    name: &str,
+    version: &str,
+) -> Result<OffsetDateTime, String> {
+    let name = match published.escape {
+        Some(escape) => escape(name),
+        None => name.to_string(),
+    };
+    let url = published.url.replace("{name}", &name).replace("{version}", version);
+    let pointer = published.pointer.replace("{version}", version);
+
+    let client = EasyHttpWebClient::default();
+    let response = client.get(&url).header(USER_AGENT, AGENT).send().await;
+    let response = response.map_err(|e| e.to_string())?;
+    if response.status() != StatusCode::OK {
+        return Err(format!("{url} replied {}", response.status()));
+    }
+    let text = response.try_into_string().await.map_err(|e| e.to_string())?;
+    let body: Value =
+        serde_json::from_str(&text).map_err(|e| format!("{url} sent unreadable JSON: {e}"))?;
+
+    let Some(stamp) = body.pointer(&pointer).and_then(Value::as_str) else {
+        return Err(format!("{url} gave no publish time"));
+    };
+    OffsetDateTime::parse(stamp, &Rfc3339).map_err(|e| format!("{url} sent a bad time: {e}"))
+}

core/src/policies/filters/denylist.rs

@@ -1,4 +1,4 @@
-use super::{Filter, Verdict, VerdictFuture};
+use super::{Filter, Verdict, VerdictFuture, AGENT};
 use crate::config::DenylistSource;
 use crate::policies::registries::Registry;
 use crate::policies::Package;
@@ -14,9 +14,6 @@ use tokio::sync::OnceCell;
 use tokio::task::JoinHandle;
 use tracing::{info, warn};
 
-/// The GitHub API turns away requests without a user agent.
-const AGENT: &str = "strix-firewall";
-
 #[derive(Debug, PartialEq)]
 pub enum AffectedVersions {
     All,

core/src/policies/filters/mod.rs

@@ -1,3 +1,4 @@
+pub mod age;
 pub mod denylist;
 pub mod osv;
 
@@ -6,6 +7,9 @@ use crate::Firewall;
 use std::future::Future;
 use std::pin::Pin;
 
+/// Some registry APIs turn away requests without a user agent.
+const AGENT: &str = "strix-firewall";
+
 pub type VerdictFuture<'a> = Pin<Box<dyn Future<Output = Verdict> + Send + 'a>>;
 
 /// `Unknown` means the filter could not check the package.

core/src/policies/registries/cargo.rs

@@ -1,4 +1,4 @@
-use super::{Pattern, Registry};
+use super::{Pattern, Published, Registry};
 
 pub static CARGO_INDEX: Registry = Registry {
     prefix: "/cargo/index/",
@@ -16,6 +16,7 @@ pub static CARGO_INDEX: Registry = Registry {
         // Sparse index paths shard by name: le/ft/left-pad, 3/s/syn, 2/cc.
         Pattern { re: r"(?<name>[^/]+)$", rewrite: None },
     ],
+    published: None,
 };
 
 pub static CARGO_DL: Registry = Registry {
@@ -26,6 +27,11 @@ pub static CARGO_DL: Registry = Registry {
     index: false,
     normalize: None,
     patterns: &[Pattern { re: r"^(?<name>[^/]+)/(?<version>[^/]+)", rewrite: None }],
+    published: Some(Published {
+        url: "https://crates.io/api/v1/crates/{name}/{version}",
+        pointer: "/version/created_at",
+        escape: None,
+    }),
 };
 
 /// `cargo add` resolves versions here, named by config.json's "api" field.
@@ -37,4 +43,5 @@ pub static CRATES_API: Registry = Registry {
     index: false,
     normalize: None,
     patterns: &[],
+    published: None,
 };

core/src/policies/registries/gem.rs

@@ -1,4 +1,4 @@
-use super::{Pattern, Registry};
+use super::{Pattern, Published, Registry};
 
 pub static GEM: Registry = Registry {
     prefix: "/gem/",
@@ -16,4 +16,9 @@ pub static GEM: Registry = Registry {
         // The compact index, which names no version.
         Pattern { re: r"^info/(?<name>[^/]+)$", rewrite: None },
     ],
+    published: Some(Published {
+        url: "https://rubygems.org/api/v2/rubygems/{name}/versions/{version}.json",
+        pointer: "/created_at",
+        escape: None,
+    }),
 };

core/src/policies/registries/go.rs

@@ -1,4 +1,4 @@
-use super::{Pattern, Registry};
+use super::{Pattern, Published, Registry};
 
 pub static GO: Registry = Registry {
     prefix: "/go/",
@@ -13,6 +13,11 @@ pub static GO: Registry = Registry {
         // The version list and the latest lookup, which name no version.
         Pattern { re: r"^(?<name>.+)/@(?:v/list|latest)$", rewrite: None },
     ],
+    published: Some(Published {
+        url: "https://proxy.golang.org/{name}/@v/v{version}.info",
+        pointer: "/Time",
+        escape: Some(escape),
+    }),
 };
 
 /// The proxy protocol writes every capital as !<lowercase>; advisories do not.
@@ -27,3 +32,14 @@ fn unescape(path: &str) -> String {
     }
     out
 }
+
+fn escape(path: &str) -> String {
+    let mut out = String::with_capacity(path.len());
+    for c in path.chars() {
+        if c.is_ascii_uppercase() {
+            out.push('!');
+        }
+        out.push(c.to_ascii_lowercase());
+    }
+    out
+}

core/src/policies/registries/mod.rs

@@ -48,6 +48,15 @@ pub struct Registry {
     /// PyPI escapes a name into a filename and Go escapes its capitals; advisories do neither.
     pub normalize: Option<fn(&str) -> String>,
     pub patterns: &'static [Pattern],
+    pub published: Option<Published>,
+}
+
+/// A URL to fetch and a JSON pointer to the timestamp in its reply.
+#[derive(Debug)]
+pub struct Published {
+    pub url: &'static str,
+    pub pointer: &'static str,
+    pub escape: Option<fn(&str) -> String>,
 }
 
 impl Registry {

core/src/policies/registries/npm.rs

@@ -1,4 +1,4 @@
-use super::{Pattern, Registry};
+use super::{Pattern, Published, Registry};
 
 pub static NPM: Registry = Registry {
     prefix: "/npm/",
@@ -18,4 +18,9 @@ pub static NPM: Registry = Registry {
             rewrite: Some(("https://registry.npmjs.org/", "/npm/")),
         },
     ],
+    published: Some(Published {
+        url: "https://registry.npmjs.org/{name}",
+        pointer: "/time/{version}",
+        escape: None,
+    }),
 };

core/src/policies/registries/pypi.rs

@@ -1,4 +1,4 @@
-use super::{Pattern, Registry};
+use super::{Pattern, Published, Registry};
 
 const FILES: (&str, &str) = ("https://files.pythonhosted.org/", "/pypi-files/");
 
@@ -13,6 +13,7 @@ pub static PYPI: Registry = Registry {
         Pattern { re: r"^simple/(?<name>[^/]+)", rewrite: Some(FILES) },
         Pattern { re: "", rewrite: Some(FILES) },
     ],
+    published: None,
 };
 
 /// The directories are hash shards, but the filename names the distribution.
@@ -35,6 +36,11 @@ pub static PYPI_FILES: Registry = Registry {
             rewrite: None,
         },
     ],
+    published: Some(Published {
+        url: "https://pypi.org/pypi/{name}/{version}/json",
+        pointer: "/urls/0/upload_time_iso_8601",
+        escape: None,
+    }),
 };
 
 /// PEP 503: runs of - _ and . collapse to one -, lowercased, which also un-escapes a wheel.

core/tests/age.rs

@@ -0,0 +1,27 @@
+use firewall::policies::screen;
+use firewall::{Firewall, FirewallConfig};
+use rama::rt::blocking::Runtime;
+
+const OLD_DOWNLOADS: &[&str] = &[
+    "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz",
+    "https://files.pythonhosted.org/packages/aa/bb/requests-2.31.0-py3-none-any.whl",
+    "https://static.crates.io/crates/serde/1.0.200",
+    "https://rubygems.org/gems/rake-13.0.6.gem",
+    "https://proxy.golang.org/github.com/!burnt!sushi/toml/@v/v1.3.2.zip",
+];
+
+#[test]
+#[ignore = "queries every registry"]
+fn every_registry_reports_when_a_version_was_published() {
+    let runtime = Runtime::try_new().expect("runtime");
+    runtime.block_on(async {
+        let firewall =
+            |hours| Firewall::new(FirewallConfig { min_age_hours: hours, ..Default::default() });
+        let (lenient, strict) = (firewall(24), firewall(24 * 365 * 100));
+        for url in OLD_DOWNLOADS {
+            assert_eq!(screen(url, &lenient).await, None, "{url}");
+            let blocked = screen(url, &strict).await.unwrap_or_default();
+            assert!(blocked.contains("hour minimum"), "{url}: {blocked:?}");
+        }
+    });
+}