Merge origin/main into fix/tnt-bubble-particles

crates/horizon-telemetry-server/Cargo.lock

@@ -253,6 +253,7 @@ name = "horizon-telemetry-server"
 version = "0.1.0"
 dependencies = [
  "axum",
+ "flate2",
  "horizon-telemetry",
  "http-body-util",
  "rusqlite",

crates/horizon-telemetry-server/Cargo.toml

@@ -19,6 +19,8 @@ rusqlite = { version = "0.37", features = ["bundled"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 ruzstd = "0.9"
+# The export answers gzip (pure Rust; ureq already pulls it in).
+flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 # Opens the hub's bug reports as GitHub issues (blocking, run off the async workers; rustls).
 ureq = { version = "2.12", features = ["json"] }
 

crates/horizon-telemetry-server/README.md

@@ -15,6 +15,8 @@ It runs at **https://telemetry.playhorizon.dev** (`DEFAULT_ENDPOINT` in `crates/
 | `DELETE /v1/installs/<id>` | Deletes everything that install sent. Its share of the daily totals is rebuilt at once. |
 | `GET /v1/aggregate?days=30` | The totals the website reads. Cached 10 minutes, with `Access-Control-Allow-Origin` set to the website. |
 | `GET /v1/benchmarks/summary` | Players' benchmark uploads grouped by GPU, CPU, render distance and game, for the website's Benchmarks page (below). Cached 10 minutes, same `Access-Control-Allow-Origin`. |
+| `GET /v1/export/benchmarks?after=<id>&limit=100` | Owner-only: the raw benchmark reports, oldest first, `{"reports": [{"id", "day", "version", "report"}], "next": <id>\|null}` (pass `next` as `after`). No install ID. Needs `Authorization: Bearer <TELEMETRY_EXPORT_TOKEN>`: 401 without it, 404 while the server has no token, gzip if the client accepts it, never cached. |
+| `GET /v1/export/hardware?days=30&offset=0&limit=200` | Owner-only, same token: the play sessions of the last `days` per machine (CPU, threads, physical cores, RAM, GPU, drive, OS…), busiest first, with installs, sessions, hours, mean p50/p99 and the frame buckets. No minimum number of installs; no install ID. `next_offset` pages. |
 | `POST /v1/live` | A game's "I'm playing" ping (JSON, at most 512 bytes). Returns 204. Returns 400 for a bad ping, 413 for one too large, 429 over the address's limit, 503 when it holds `LIVE_MAX_SESSIONS` already. |
 | `GET /v1/live` | How many play now, exactly: `playing`, `lan`, `versions` (the 12 most played, the rest under `other`), `platforms`. Made again at most every 15 seconds, with the same `Access-Control-Allow-Origin`. |
 | `GET /v1/live/history?range=24h\|7d\|30d\|all&series=players\|official` | The live count (`players`, the default) or the official server's player count (`official`) over time for playhorizon.dev/stats: `series` of `[start, average, peak]`, the range's `peak` and the `all_time` one with their minute, the `latest` minute, the average per platform ([docs/telemetry.md](../../docs/telemetry.md)). Made again at most every minute (`Cache-Control: max-age=60`), on the live count's rate limit. 400 for another range or series. |
@@ -91,6 +93,7 @@ All are environment variables:
 - `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90)
 - `HORIZON_TELEMETRY_GITHUB_TOKEN`: a fine-grained token with Issues read and write on the repository only (unset: `POST /hub/report` answers 503 and the form points to GitHub and Discord)
 - `HORIZON_TELEMETRY_GITHUB_REPO` (`horizon-rust/horizon`)
+- `TELEMETRY_EXPORT_TOKEN` (or `HORIZON_TELEMETRY_EXPORT_TOKEN`): the bearer token of `/v1/export/*` (unset: those answer 404). A long random secret; requests with a wrong one count against the address's rate limit.
 
 ## Hosting
 

crates/horizon-telemetry-server/src/export.rs

@@ -0,0 +1,281 @@
+//! The protected, read-only export for the people who tune performance: the raw benchmark
+//! reports and the per-hardware totals, with no minimum number of installs (the public
+//! `/v1/aggregate` and `/v1/benchmarks/summary` hide small groups).
+//!
+//! - `GET /v1/export/benchmarks?after=<id>&limit=100`: `{"reports": [{"id", "day", "version",
+//!   "report"}], "next": <id>|null}`, oldest first. Pass `next` as `after` for the next page.
+//! - `GET /v1/export/hardware?days=30&offset=0&limit=200`: `{"machines": [...], "next_offset"}`:
+//!   one entry per distinct machine description (CPU threads and cores, RAM, GPU, drive, OS…),
+//!   with its installs, sessions, hours and frame-time numbers, busiest first.
+//!
+//! Both need `Authorization: Bearer <TELEMETRY_EXPORT_TOKEN>`, are off (404) while the server has
+//! no such token, answer gzip to a client that accepts it, and are never cached. Install IDs
+//! never leave the database: a report has no owner here, and a machine only a count of them.
+use std::collections::{HashMap, HashSet};
+use std::io::Write;
+use std::sync::Arc;
+
+use axum::extract::{Query, Request, State};
+use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use horizon_telemetry::payload::BUCKET_EDGES_MS;
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+use crate::store::{Extra, Store, unpack};
+use crate::{App, peer, status};
+
+/// Equal, in a time that depends on the token's length only, not on where they differ.
+pub(crate) fn same(sent: &[u8], token: &[u8]) -> bool {
+    let mut diff = sent.len() ^ token.len();
+    for (i, t) in token.iter().enumerate() {
+        diff |= usize::from(sent.get(i).copied().unwrap_or(0) ^ t);
+    }
+    diff == 0
+}
+
+/// The answer to send instead when the request's `Bearer` token doesn't check out: 404 while the
+/// export is off, 401 for a missing or wrong token, 429 once an address has failed too often.
+/// None: go ahead.
+fn refuse(
+    app: &App,
+    headers: &HeaderMap,
+    req_peer: Option<std::net::SocketAddr>,
+) -> Option<Response> {
+    let Some(token) = app.config.export_token.as_deref() else {
+        return Some(StatusCode::NOT_FOUND.into_response());
+    };
+    let sent = headers
+        .get(header::AUTHORIZATION)
+        .and_then(|v| v.to_str().ok())
+        .and_then(|v| v.strip_prefix("Bearer "))
+        .unwrap_or("");
+    if same(sent.as_bytes(), token.as_bytes()) {
+        return None;
+    }
+    // Only failures take from the address's bucket, so guessing is slow and using the token isn't.
+    if !app.allow(app.client(headers, req_peer)) {
+        return Some(status(StatusCode::TOO_MANY_REQUESTS, "too many requests"));
+    }
+    Some(StatusCode::UNAUTHORIZED.into_response())
+}
+
+/// `value` as JSON, gzipped when the client accepts it.
+fn answer(headers: &HeaderMap, value: Value) -> Response {
+    let json = value.to_string().into_bytes();
+    let gzip = headers
+        .get(header::ACCEPT_ENCODING)
+        .and_then(|v| v.to_str().ok())
+        .is_some_and(|v| {
+            v.split(',').any(|c| {
+                c.trim()
+                    .split(';')
+                    .next()
+                    .is_some_and(|c| c.eq_ignore_ascii_case("gzip"))
+            })
+        });
+    let mut res = if gzip {
+        let mut z = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        let packed = z.write_all(&json).and_then(|()| z.finish());
+        match packed {
+            Ok(packed) => {
+                let mut res = packed.into_response();
+                res.headers_mut()
+                    .insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
+                res
+            }
+            Err(_) => json.into_response(),
+        }
+    } else {
+        json.into_response()
+    };
+    let h = res.headers_mut();
+    h.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("application/json"),
+    );
+    h.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+    h.insert(header::VARY, HeaderValue::from_static("Accept-Encoding"));
+    res
+}
+
+#[derive(Deserialize)]
+pub struct Page {
+    after: Option<i64>,
+    limit: Option<u32>,
+}
+
+/// `GET /v1/export/benchmarks`: the raw benchmark reports, `limit` (1-500, default 100) a page.
+pub async fn benchmarks(
+    State(app): State<Arc<App>>,
+    Query(page): Query<Page>,
+    req: Request,
+) -> Response {
+    if let Some(res) = refuse(&app, req.headers(), peer(&req)) {
+        return res;
+    }
+    let limit = page.limit.unwrap_or(100).clamp(1, 500);
+    let listed = app
+        .store()
+        .export_benchmarks(page.after.unwrap_or(0), limit);
+    match listed {
+        Ok(reports) => {
+            let next = (reports.len() as u32 == limit)
+                .then(|| reports.last().map(|r| r["id"].clone()))
+                .flatten();
+            answer(req.headers(), json!({ "reports": reports, "next": next }))
+        }
+        Err(e) => {
+            eprintln!("horizon-telemetry-server: could not export the benchmarks: {e}");
+            status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
+        }
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Machines {
+    days: Option<u32>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+}
+
+/// `GET /v1/export/hardware`: play sessions of the last `days` (1-366, default 30) per machine
+/// description, `limit` (1-1000, default 200) machines a page from `offset`.
+pub async fn hardware(
+    State(app): State<Arc<App>>,
+    Query(q): Query<Machines>,
+    req: Request,
+) -> Response {
+    if let Some(res) = refuse(&app, req.headers(), peer(&req)) {
+        return res;
+    }
+    let days = q.days.unwrap_or(30).clamp(1, 366);
+    let (offset, limit) = (q.offset.unwrap_or(0), q.limit.unwrap_or(200).clamp(1, 1000));
+    let to = (app.today)();
+    let made = app.store().export_machines(to + 1 - days.min(to + 1), to);
+    match made {
+        Ok(all) => {
+            let total = all.len();
+            let machines: Vec<Value> = all.into_iter().skip(offset).take(limit).collect();
+            let next = (offset + machines.len() < total).then_some(offset + machines.len());
+            answer(
+                req.headers(),
+                json!({
+                    "from_day": to + 1 - days.min(to + 1), "to_day": to, "total_machines": total,
+                    "bucket_edges_ms": BUCKET_EDGES_MS.map(|e| (f64::from(e) * 10.0).round() / 10.0),
+                    "machines": machines, "next_offset": next,
+                }),
+            )
+        }
+        Err(e) => {
+            eprintln!("horizon-telemetry-server: could not export the machines: {e}");
+            status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
+        }
+    }
+}
+
+impl Store {
+    /// Benchmark reports with an id above `after`, oldest first.
+    pub fn export_benchmarks(&self, after: i64, limit: u32) -> rusqlite::Result<Vec<Value>> {
+        let mut q = self.db.prepare(
+            "SELECT rowid, day, version, body FROM reports WHERE kind = 3 AND rowid > ?1
+             ORDER BY rowid LIMIT ?2",
+        )?;
+        q.query_map(rusqlite::params![after, limit], |r| {
+            Ok(json!({
+                "id": r.get::<_, i64>(0)?, "day": r.get::<_, u32>(1)?, "version": r.get::<_, String>(2)?,
+                "report": crate::store::inflate(&r.get::<_, Vec<u8>>(3)?),
+            }))
+        })?
+        .collect()
+    }
+
+    /// Play sessions in `from..=to` per machine description, the busiest first. A machine is its
+    /// CPU, memory, GPU, drive and system: two sessions differing only in the screen or in
+    /// battery against mains count as one. No group is left out for being small.
+    pub fn export_machines(&self, from: u32, to: u32) -> rusqlite::Result<Vec<Value>> {
+        #[derive(Default)]
+        struct Machine {
+            installs: HashSet<Vec<u8>>,
+            sessions: u64,
+            seconds: u64,
+            p50_sum: f64,
+            p99_sum: f64,
+            buckets: Vec<u64>,
+            peak_ram_mb: u32,
+            description: Value,
+        }
+        let mut machines: HashMap<String, Machine> = HashMap::new();
+        let mut q = self.db.prepare(
+            "SELECT perf.install, perf.seconds, perf.frames, perf.peak_ram_mb, systems.os, systems.os_version,
+             systems.arch, systems.cpu, systems.threads, systems.ram_gb, systems.gpu, systems.driver,
+             systems.backend, systems.extra
+             FROM perf JOIN systems ON systems.id = perf.system WHERE perf.day BETWEEN ?1 AND ?2",
+        )?;
+        let mut rows = q.query([from, to])?;
+        while let Some(r) = rows.next()? {
+            let Some(frames) = unpack(&r.get::<_, Vec<u8>>(2)?) else {
+                continue;
+            };
+            let extra: Extra = serde_json::from_str(&r.get::<_, String>(13)?).unwrap_or_default();
+            let description = json!({
+                "os": r.get::<_, String>(4)?, "os_version": r.get::<_, String>(5)?,
+                "arch": r.get::<_, String>(6)?, "cpu": r.get::<_, String>(7)?,
+                "threads": r.get::<_, u16>(8)?, "cores": extra.cores,
+                "cpu_base_mhz": extra.cpu_base_mhz, "cpu_boost_mhz": extra.cpu_boost_mhz,
+                "ram_gb": r.get::<_, u16>(9)?,
+                "gpu": r.get::<_, String>(10)?, "gpu_vendor": extra.gpu_vendor, "gpu_kind": extra.gpu_kind,
+                "vram_mb": extra.vram_mb, "driver": r.get::<_, String>(11)?,
+                "backend": r.get::<_, String>(12)?, "storage": extra.storage, "form": extra.form,
+            });
+            let m = machines.entry(description.to_string()).or_default();
+            if m.buckets.is_empty() {
+                m.buckets = vec![0; frames.buckets.len()];
+                m.description = description;
+            }
+            m.installs.insert(r.get(0)?);
+            m.sessions += 1;
+            m.seconds += r.get::<_, u64>(1)?;
+            m.peak_ram_mb = m.peak_ram_mb.max(r.get(3)?);
+            m.p50_sum += f64::from(frames.p50_ms);
+            m.p99_sum += f64::from(frames.p99_ms);
+            for (sum, n) in m.buckets.iter_mut().zip(&frames.buckets) {
+                *sum += u64::from(*n);
+            }
+        }
+        let mut list: Vec<(String, Machine)> = machines.into_iter().collect();
+        // Busiest first; the description breaks ties so pages are stable.
+        list.sort_by(|a, b| b.1.sessions.cmp(&a.1.sessions).then(a.0.cmp(&b.0)));
+        let avg = |sum: f64, n: u64| (sum / n as f64 * 10.0).round() / 10.0;
+        Ok(list
+            .into_iter()
+            .map(|(_, m)| {
+                let mut out = m.description;
+                out["installs"] = m.installs.len().into();
+                out["sessions"] = m.sessions.into();
+                out["hours"] = ((m.seconds as f64 / 360.0).round() / 10.0).into();
+                out["p50_ms"] = avg(m.p50_sum, m.sessions).into();
+                out["p99_ms"] = avg(m.p99_sum, m.sessions).into();
+                out["peak_ram_mb"] = m.peak_ram_mb.into();
+                out["frame_buckets"] = m.buckets.into();
+                out
+            })
+            .collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::same;
+
+    #[test]
+    fn tokens_are_compared_whole() {
+        assert!(same(b"abc", b"abc"));
+        assert!(!same(b"abd", b"abc"));
+        assert!(!same(b"ab", b"abc"));
+        assert!(!same(b"abcd", b"abc"));
+        assert!(!same(b"", b"abc"));
+        // A token with a trailing NUL byte isn't the token without it.
+        assert!(!same(b"abc\0", b"abc"));
+    }
+}

crates/horizon-telemetry-server/src/lib.rs

@@ -5,6 +5,8 @@
 //! - `GET /v1/aggregate?days=30`: the anonymous totals the website reads, cached.
 //! - `GET /v1/benchmarks/summary`: players' benchmark results grouped by GPU and CPU, groups of at
 //!   least 3 installs only (`summary`), cached; the website's Benchmarks page reads it.
+//! - `GET /v1/export/benchmarks` and `/v1/export/hardware`: raw benchmark reports and per-machine
+//!   totals for the perf work, behind a bearer token (`export`); off without one.
 //! - `POST /v1/live`: a game's anonymous "I'm playing" ping (`live`, in memory only).
 //! - `GET /v1/live`: how many play now, exactly, cached briefly.
 //! - `GET /v1/live/history?range=24h|7d|30d|all&series=players|official`: that count, or the
@@ -32,6 +34,7 @@ use axum::routing::{delete, get, post};
 use horizon_telemetry::batch;
 
 pub mod classes;
+pub mod export;
 pub mod github;
 pub mod history;
 pub mod hub;
@@ -81,6 +84,8 @@ pub struct Config {
     pub hub_reports_per_day: u32,
     /// Days reports are kept (opened as issues long before).
     pub hub_keep_days: u32,
+    /// The bearer token the protected export (`export`) wants; unset, it answers 404.
+    pub export_token: Option<String>,
     /// Opens the website's bug reports as issues; unset, `POST /hub/report` answers 503.
     pub github_token: Option<String>,
     /// Where those issues go, `owner/name`.
@@ -106,6 +111,7 @@ impl Default for Config {
             hub_posts_per_hour: 20.0,
             hub_reports_per_day: 30,
             hub_keep_days: 90,
+            export_token: None,
             github_token: None,
             github_repo: "horizon-rust/horizon".into(),
         }
@@ -118,7 +124,8 @@ impl Config {
     /// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS`, `…_LIVE_MAX_SESSIONS`, `…_LIVE_MINUTE_DAYS`,
     /// `…_LIVE_HOUR_DAYS`, `…_OFFICIAL_SERVER` (`off` for none), `…_HUB_PER_HOUR`,
     /// `…_HUB_POSTS_PER_HOUR`, `…_HUB_REPORTS_PER_DAY`, `…_HUB_KEEP_DAYS`, `…_GITHUB_TOKEN` and
-    /// `…_GITHUB_REPO`. An empty token counts as unset.
+    /// `…_GITHUB_REPO`, plus `TELEMETRY_EXPORT_TOKEN` (or `HORIZON_TELEMETRY_EXPORT_TOKEN`) for the
+    /// protected export. An empty token counts as unset.
     pub fn from_env() -> Self {
         let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
         let secret = |k: &str| {
@@ -174,6 +181,11 @@ impl Config {
             hub_keep_days: var("HUB_KEEP_DAYS")
                 .and_then(|v| v.parse().ok())
                 .unwrap_or(d.hub_keep_days),
+            export_token: std::env::var("TELEMETRY_EXPORT_TOKEN")
+                .ok()
+                .map(|v| v.trim().to_owned())
+                .filter(|v| !v.is_empty())
+                .or_else(|| secret("EXPORT_TOKEN")),
             github_token: secret("GITHUB_TOKEN"),
             github_repo: var("GITHUB_REPO").unwrap_or(d.github_repo),
         }
@@ -718,6 +730,8 @@ pub fn router(app: Arc<App>) -> Router {
         .route("/v1/installs/{id}", delete(forget))
         .route("/v1/aggregate", get(aggregate))
         .route("/v1/benchmarks/summary", get(benchmarks))
+        .route("/v1/export/benchmarks", get(export::benchmarks))
+        .route("/v1/export/hardware", get(export::hardware))
         .route("/hub/vote", post(hub::vote).options(hub::preflight))
         .route("/hub/votes", get(hub::votes))
         .route("/hub/report", post(intake::report).options(hub::preflight))

crates/horizon-telemetry-server/tests/export.rs

@@ -0,0 +1,313 @@
+//! The protected export through its HTTP interface: off without a token, 401 without the right
+//! one, raw benchmark reports paginated, per-machine totals with the CPU core counts, gzip.
+use std::io::Read;
+use std::sync::Arc;
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use horizon_telemetry::Item;
+use horizon_telemetry::batch::{self, Batch};
+use horizon_telemetry::payload::{Payload, Perf, Settings, Source, System, Timings};
+use horizon_telemetry_server::store::Store;
+use horizon_telemetry_server::{App, Config, router};
+use http_body_util::BodyExt;
+use serde_json::{Value, json};
+use tower::ServiceExt;
+
+const DAY: u32 = 20_741;
+const TOKEN: &str = "a-long-random-export-token";
+
+fn install(n: u32) -> String {
+    format!("{n:032x}")
+}
+
+fn app(token: Option<&str>) -> Arc<App> {
+    App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            requests_per_hour: 100_000.0,
+            export_token: token.map(str::to_owned),
+            ..Config::default()
+        },
+        Box::new(|| DAY),
+    )
+}
+
+fn perf(cpu: &str, threads: u16, cores: u16, storage: &str) -> Item {
+    let mut t = Timings::new();
+    (0..3000).for_each(|i| t.record(6.0 + (i % 50) as f32 / 10.0));
+    let mut item = Item::new(
+        Source::Game,
+        "v0.2.97",
+        Payload::Perf(Perf {
+            system: System {
+                os: "windows".into(),
+                cpu: cpu.into(),
+                threads,
+                cores,
+                ram_gb: 32,
+                gpu: "NVIDIA GeForce RTX 4070".into(),
+                gpu_vendor: "nvidia".into(),
+                gpu_kind: "discrete".into(),
+                backend: "Dx12".into(),
+                storage: storage.into(),
+                ..System::default()
+            },
+            settings: Settings {
+                render_distance: 12,
+                ..Settings::default()
+            },
+            seconds: 3600,
+            frames: t.summary(),
+            peak_ram_mb: 2500,
+            ..Perf::default()
+        }),
+    );
+    item.day = DAY;
+    item
+}
+
+fn benchmark(n: u32) -> Item {
+    let mut item = Item::new(
+        Source::Game,
+        "v0.2.97",
+        Payload::Benchmark(json!({
+            "format": "horizon-benchmark-report", "version": 1, "n": n,
+            "hardware": { "os": "windows", "cpu_threads": 32, "cpu_cores": 16, "storage": "nvme" },
+        })),
+    );
+    item.day = DAY;
+    item
+}
+
+async fn send(app: &Arc<App>, req: Request<Body>) -> (StatusCode, Vec<u8>, Option<String>) {
+    let res = router(app.clone()).oneshot(req).await.unwrap();
+    let code = res.status();
+    let encoding = res
+        .headers()
+        .get("content-encoding")
+        .map(|v| v.to_str().unwrap().to_owned());
+    (
+        code,
+        res.into_body().collect().await.unwrap().to_bytes().to_vec(),
+        encoding,
+    )
+}
+
+async fn upload(app: &Arc<App>, id: u32, items: Vec<Item>) {
+    let body = batch::encode(&Batch::new(&install(id), items));
+    let req = Request::post("/v1/batch")
+        .header("content-encoding", "zstd")
+        .body(Body::from(body))
+        .unwrap();
+    assert_eq!(send(app, req).await.0, StatusCode::NO_CONTENT);
+}
+
+async fn get(app: &Arc<App>, uri: &str, token: Option<&str>) -> (StatusCode, Value) {
+    let mut req = Request::get(uri);
+    if let Some(t) = token {
+        req = req.header("authorization", format!("Bearer {t}"));
+    }
+    let (code, body, encoding) = send(app, req.body(Body::empty()).unwrap()).await;
+    assert_eq!(encoding, None, "plain unless gzip is asked for");
+    (code, serde_json::from_slice(&body).unwrap_or(Value::Null))
+}
+
+#[tokio::test]
+async fn it_is_off_without_a_token_and_refuses_a_wrong_one() {
+    for uri in ["/v1/export/benchmarks", "/v1/export/hardware"] {
+        let off = app(None);
+        assert_eq!(get(&off, uri, None).await.0, StatusCode::NOT_FOUND);
+        assert_eq!(get(&off, uri, Some(TOKEN)).await.0, StatusCode::NOT_FOUND);
+
+        let on = app(Some(TOKEN));
+        assert_eq!(get(&on, uri, None).await.0, StatusCode::UNAUTHORIZED);
+        for wrong in [
+            "",
+            "a-long-random-export-toke",
+            "a-long-random-export-tokenX",
+            "nope",
+        ] {
+            assert_eq!(
+                get(&on, uri, Some(wrong)).await.0,
+                StatusCode::UNAUTHORIZED,
+                "{wrong:?}"
+            );
+        }
+        // The wrong scheme, even with the right secret.
+        let req = Request::get(uri)
+            .header("authorization", format!("Basic {TOKEN}"))
+            .body(Body::empty())
+            .unwrap();
+        assert_eq!(send(&on, req).await.0, StatusCode::UNAUTHORIZED);
+        assert_eq!(get(&on, uri, Some(TOKEN)).await.0, StatusCode::OK);
+    }
+}
+
+#[tokio::test]
+async fn guessing_is_rate_limited_but_using_the_token_is_not() {
+    let app = App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            requests_per_hour: 10.0,
+            export_token: Some(TOKEN.into()),
+            ..Config::default()
+        },
+        Box::new(|| DAY),
+    );
+    let mut codes = Vec::new();
+    for _ in 0..5 {
+        codes.push(get(&app, "/v1/export/hardware", Some("guess")).await.0);
+    }
+    assert_eq!(codes[0], StatusCode::UNAUTHORIZED);
+    assert_eq!(codes[4], StatusCode::TOO_MANY_REQUESTS);
+    for _ in 0..20 {
+        assert_eq!(
+            get(&app, "/v1/export/hardware", Some(TOKEN)).await.0,
+            StatusCode::OK
+        );
+    }
+}
+
+#[tokio::test]
+async fn benchmark_reports_come_out_raw_and_in_pages() {
+    let app = app(Some(TOKEN));
+    for n in 0..5 {
+        upload(&app, n + 1, vec![benchmark(n)]).await;
+    }
+    // A play session is not a benchmark.
+    upload(&app, 9, vec![perf("AMD Ryzen 9 7950X", 32, 16, "nvme")]).await;
+
+    let (code, first) = get(&app, "/v1/export/benchmarks?limit=2", Some(TOKEN)).await;
+    assert_eq!(code, StatusCode::OK);
+    let reports = first["reports"].as_array().unwrap();
+    assert_eq!(reports.len(), 2);
+    assert_eq!(reports[0]["report"]["hardware"]["cpu_threads"], 32);
+    assert_eq!(reports[0]["report"]["hardware"]["cpu_cores"], 16);
+    assert_eq!(
+        (reports[0]["day"].as_u64(), reports[0]["version"].as_str()),
+        (Some(DAY.into()), Some("v0.2.97"))
+    );
+
+    let mut seen = vec![
+        reports[0]["report"]["n"].clone(),
+        reports[1]["report"]["n"].clone(),
+    ];
+    let mut next = first["next"].clone();
+    let mut pages = 1;
+    while let Some(after) = next.as_i64() {
+        let (_, page) = get(
+            &app,
+            &format!("/v1/export/benchmarks?limit=2&after={after}"),
+            Some(TOKEN),
+        )
+        .await;
+        seen.extend(
+            page["reports"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|r| r["report"]["n"].clone()),
+        );
+        next = page["next"].clone();
+        pages += 1;
+    }
+    assert_eq!(seen, [0, 1, 2, 3, 4].map(Value::from));
+    assert_eq!(pages, 3);
+
+    // Whose report it was never leaves the database.
+    let (_, all) = get(&app, "/v1/export/benchmarks", Some(TOKEN)).await;
+    assert_eq!(all["next"], Value::Null);
+    assert!(!all.to_string().contains(&install(1)));
+    assert!(!all.to_string().contains("install"));
+}
+
+#[tokio::test]
+async fn machines_come_with_their_core_counts_and_no_minimum() {
+    let app = app(Some(TOKEN));
+    // One install on a 16-core / 32-thread machine, two on a 4-core / 8-thread one.
+    upload(&app, 1, vec![perf("AMD Ryzen 9 7950X", 32, 16, "nvme")]).await;
+    upload(&app, 2, vec![perf("Intel Core i5-8250U", 8, 4, "ssd")]).await;
+    upload(&app, 3, vec![perf("Intel Core i5-8250U", 8, 4, "ssd")]).await;
+
+    let (code, out) = get(&app, "/v1/export/hardware?days=30", Some(TOKEN)).await;
+    assert_eq!(code, StatusCode::OK);
+    assert_eq!(out["total_machines"], 2);
+    let machines = out["machines"].as_array().unwrap();
+    // Busiest first.
+    assert_eq!(machines[0]["cpu"], "Intel Core i5-8250U");
+    assert_eq!(
+        (
+            machines[0]["threads"].as_u64(),
+            machines[0]["cores"].as_u64()
+        ),
+        (Some(8), Some(4))
+    );
+    assert_eq!(
+        (
+            machines[0]["installs"].as_u64(),
+            machines[0]["sessions"].as_u64()
+        ),
+        (Some(2), Some(2))
+    );
+    assert_eq!(machines[0]["storage"], "ssd");
+    assert_eq!(machines[0]["hours"], 2.0);
+    assert!(machines[0]["p99_ms"].as_f64().unwrap() > 0.0);
+    assert_eq!(
+        machines[0]["frame_buckets"].as_array().unwrap().len(),
+        out["bucket_edges_ms"].as_array().unwrap().len() + 1
+    );
+    assert_eq!(machines[1]["cpu"], "AMD Ryzen 9 7950X");
+    assert_eq!(
+        (
+            machines[1]["threads"].as_u64(),
+            machines[1]["cores"].as_u64()
+        ),
+        (Some(32), Some(16))
+    );
+    assert_eq!(machines[1]["installs"], 1);
+    assert!(!out.to_string().contains(&install(1)));
+
+    // Pages by offset.
+    let (_, page) = get(&app, "/v1/export/hardware?limit=1", Some(TOKEN)).await;
+    assert_eq!(page["machines"].as_array().unwrap().len(), 1);
+    assert_eq!(page["next_offset"], 1);
+    let (_, page) = get(&app, "/v1/export/hardware?limit=1&offset=1", Some(TOKEN)).await;
+    assert_eq!(page["machines"][0]["cpu"], "AMD Ryzen 9 7950X");
+    assert_eq!(page["next_offset"], Value::Null);
+
+    // Outside the window: nothing.
+    let old = App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            export_token: Some(TOKEN.into()),
+            ..Config::default()
+        },
+        Box::new(|| DAY + 90),
+    );
+    let (_, none) = get(&old, "/v1/export/hardware?days=30", Some(TOKEN)).await;
+    assert_eq!(none["total_machines"], 0);
+}
+
+#[tokio::test]
+async fn gzip_when_asked() {
+    let app = app(Some(TOKEN));
+    upload(&app, 1, vec![benchmark(7)]).await;
+    let req = Request::get("/v1/export/benchmarks")
+        .header("authorization", format!("Bearer {TOKEN}"))
+        .header("accept-encoding", "br, gzip;q=0.8")
+        .body(Body::empty())
+        .unwrap();
+    let res = router(app.clone()).oneshot(req).await.unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(res.headers()["content-encoding"], "gzip");
+    assert_eq!(res.headers()["content-type"], "application/json");
+    assert_eq!(res.headers()["cache-control"], "no-store");
+    let body = res.into_body().collect().await.unwrap().to_bytes();
+    let mut json = String::new();
+    flate2::read::GzDecoder::new(&body[..])
+        .read_to_string(&mut json)
+        .unwrap();
+    let out: Value = serde_json::from_str(&json).unwrap();
+    assert_eq!(out["reports"][0]["report"]["n"], 7);
+}

docs/telemetry.md

@@ -108,6 +108,7 @@ The launcher reads the count when it opens and every minute after, and shows "N
 - `DELETE /v1/installs/<install_id>`: deletes everything sent under that ID.
 - `GET /v1/aggregate`: the anonymous totals the website's results page reads. Its `breakdowns` split the play sessions of the period by hardware class (below), each class with its installs, sessions, hours and mean p50 and p99 frame times.
 - `GET /v1/benchmarks/summary`: players' benchmark uploads grouped by GPU, CPU, render distance and game for playhorizon.dev/benchmarks. A group shows only with at least 3 different installs, and only its medians and counts: no install ID, day, OS version, driver, exact memory or screen. Uploads are never published one by one. Each group's `breakdowns` count its installs per hardware class.
+- `GET /v1/export/benchmarks` and `GET /v1/export/hardware`: for the maintainers' performance work only, behind a bearer token the public cannot use (404 while the server has none). The raw benchmark reports and the play sessions per machine description, including CPU threads and physical cores, with no minimum group size. Nothing identifying: no install ID, and only what a player already agreed to send (this page's lists).
 - `POST /v1/live` and `GET /v1/live`: the "I'm playing" ping and the live count.
 - `GET /v1/live/history?range=24h|7d|30d|all&series=players|official`: the live count, or the official server's, over time (above).