Merge remote-tracking branch 'origin/main' into fix/blast-wind-remaining

crates/horizon-client-render/src/gfx.rs

@@ -1857,7 +1857,10 @@ impl Renderer {
                     device_lost: renderer.device_lost.clone(),
                 }
             }
-            None => open_device(&window, offscreen, api, layers).await?,
+            None => {
+                let _step = crate::spans::startup::step("renderer.open_device");
+                open_device(&window, offscreen, api, layers).await?
+            }
         };
         // Computed once: the sprite arrays, the shaders' constants and the
         // shader pack atlas all cut the sprites by it.
@@ -1937,6 +1940,8 @@ impl Renderer {
         if previous.is_none() {
             startup_step(format_args!("creating the pipelines"));
         }
+        let pipelines_step = crate::spans::startup::step("renderer.pipelines_textures");
+        let terrain_step = crate::spans::startup::step("renderer.terrain_pipelines");
         let gpu_timer = timing::GpuTimer::new(&device, &queue);
         let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
             label: Some("Horizon terrain"),
@@ -2283,6 +2288,7 @@ impl Renderer {
             false,
             Some(alpha_blend),
         );
+        drop(terrain_step);
         let globals = device.create_buffer(&wgpu::BufferDescriptor {
             label: Some("Horizon globals"),
             size: std::mem::size_of::<Globals>() as u64,
@@ -2594,6 +2600,7 @@ impl Renderer {
             &glint_sampler,
         )?;
         hud.atlas_texture = Some(atlas.clone());
+        let entity_step = crate::spans::startup::step("renderer.entity_pipelines");
         let entity_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
             label: Some("Horizon entities"),
             source: wgpu::ShaderSource::Wgsl(entity_shader().into()),
@@ -3165,6 +3172,7 @@ impl Renderer {
             multiview_mask: None,
             cache: None,
         });
+        drop(entity_step);
         // WorldBorderRenderer: misc/forcefield.png as a MipmappedTexture
         // (repeat, linear minification with mipmaps, nearest magnification).
         let forcefield_size = forcefield.1;
@@ -3542,6 +3550,7 @@ impl Renderer {
         );
         let panorama =
             panorama::Panorama::new(&device, config.format.remove_srgb_suffix(), panorama_faces);
+        drop(pipelines_step);
         if previous.is_none() {
             startup_step(format_args!(
                 "pipelines ready (multi-draw {}, occlusion culling {}, GPU timer {})",

crates/horizon-client-render/src/spans.rs

@@ -92,6 +92,161 @@ pub fn describe(spans: &[Span]) -> String {
     list.join(", ")
 }
 
+/// The start's timeline: `HORIZON_STARTUP_TRACE=FILE` writes every named step of
+/// the start (resource tables, the renderer, the title, a world's open, a
+/// server's join, a dimension change) on whichever thread ran it, one Chrome
+/// trace event per line (`tools/startup_trace.py` reads it; wrapped in `[...]`,
+/// Perfetto opens it). Times are microseconds since the process started. Off,
+/// a step costs one relaxed load.
+pub mod startup {
+    use std::{
+        io::Write as _,
+        sync::{
+            Mutex, OnceLock,
+            atomic::{AtomicBool, AtomicU32, Ordering},
+        },
+        time::Instant,
+    };
+
+    static ON: AtomicBool = AtomicBool::new(false);
+    static SINK: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();
+    static ORIGIN: OnceLock<Instant> = OnceLock::new();
+    static NEXT_THREAD: AtomicU32 = AtomicU32::new(1);
+    thread_local! {
+        /// This thread's trace id, given (with its name) on its first event.
+        static THREAD: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
+    }
+
+    /// Opens the trace named by `HORIZON_STARTUP_TRACE`, timed from `origin`
+    /// (the process's start). Call once, first thing.
+    pub fn init(origin: Instant) {
+        ORIGIN.get_or_init(|| origin);
+        let sink = SINK.get_or_init(|| {
+            let path = std::env::var_os("HORIZON_STARTUP_TRACE")?;
+            std::fs::File::create(&path).ok().map(Mutex::new)
+        });
+        ON.store(sink.is_some(), Ordering::Relaxed);
+    }
+
+    /// Whether a trace is being written.
+    #[inline]
+    pub fn on() -> bool {
+        ON.load(Ordering::Relaxed)
+    }
+
+    fn micros(at: Instant) -> u128 {
+        ORIGIN.get().map_or(0, |origin| {
+            at.saturating_duration_since(*origin).as_micros()
+        })
+    }
+
+    fn write(event: std::fmt::Arguments) {
+        let Some(Some(sink)) = SINK.get() else {
+            return;
+        };
+        let tid = THREAD.with(|id| {
+            if id.get() == 0 {
+                id.set(NEXT_THREAD.fetch_add(1, Ordering::Relaxed));
+                let name = std::thread::current()
+                    .name()
+                    .unwrap_or("unnamed")
+                    .replace(['"', '\\'], "_");
+                let mut file = sink.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+                let _ = writeln!(
+                    file,
+                    r#"{{"ph":"M","name":"thread_name","pid":1,"tid":{},"args":{{"name":"{name}"}}}}"#,
+                    id.get()
+                );
+            }
+            id.get()
+        });
+        let mut file = sink
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let _ = writeln!(file, r#"{{"pid":1,"tid":{tid},{event}}}"#);
+    }
+
+    /// A step running until it is dropped.
+    #[must_use = "the step ends when this is dropped"]
+    pub struct Step {
+        name: &'static str,
+        started: Instant,
+    }
+    impl Drop for Step {
+        fn drop(&mut self) {
+            let end = Instant::now();
+            write(format_args!(
+                r#""ph":"X","name":"{}","ts":{},"dur":{}"#,
+                self.name,
+                micros(self.started),
+                end.saturating_duration_since(self.started).as_micros()
+            ));
+        }
+    }
+
+    /// Starts a step named `name` (a literal: no quotes or backslashes).
+    #[inline]
+    pub fn step(name: &'static str) -> Option<Step> {
+        on().then(|| Step {
+            name,
+            started: Instant::now(),
+        })
+    }
+
+    /// Runs `f` as a step named `name`.
+    #[inline]
+    pub fn time<T>(name: &'static str, f: impl FnOnce() -> T) -> T {
+        let _step = step(name);
+        f()
+    }
+
+    /// A moment of the start: the title drawn, the world in view.
+    #[inline]
+    pub fn mark(name: &'static str) {
+        if on() {
+            write(format_args!(
+                r#""ph":"i","s":"g","name":"{name}","ts":{}"#,
+                micros(Instant::now())
+            ));
+        }
+    }
+
+    /// The first time `name` happens, as a mark. `once` is the caller's flag.
+    #[inline]
+    pub fn mark_once(once: &AtomicBool, name: &'static str) {
+        if on() && !once.swap(true, Ordering::Relaxed) {
+            mark(name);
+        }
+    }
+
+    /// Set while a world opens, a server is joined or the dimension changes,
+    /// until the view first fills ([`view_filled`]).
+    static FILLING: AtomicBool = AtomicBool::new(false);
+
+    /// A world starts loading: the next full view is marked.
+    pub fn arm(name: &'static str) {
+        if on() {
+            mark(name);
+            FILLING.store(true, Ordering::Relaxed);
+        }
+    }
+
+    /// Called each frame in a world: marks `world.view_full` the first frame
+    /// after [`arm`] where nothing waits to load, generate, light, mesh or
+    /// upload and the resident columns cover perf_bench's share of the view
+    /// (0.8 pi r^2: corners are never loaded).
+    #[inline]
+    pub fn view_filled(resident: usize, pending: usize, radius: u32) {
+        if FILLING.load(Ordering::Relaxed)
+            && pending == 0
+            && resident as f64 >= 0.8 * std::f64::consts::PI * f64::from(radius * radius)
+            && FILLING.swap(false, Ordering::Relaxed)
+        {
+            mark("world.view_full");
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::Span;

crates/horizon-client-render/src/telemetry.rs

@@ -241,6 +241,7 @@ impl Session {
     pub fn title_shown(&mut self) {
         if !self.title_measured {
             self.title_measured = true;
+            crate::spans::startup::mark("title.shown");
             let ui_ms = UI_READY_MS.load(Ordering::Relaxed);
             self.to_title_ms = if ui_ms != u32::MAX {
                 Some(ui_ms)
@@ -252,6 +253,9 @@ impl Session {
 
     /// A world starts loading.
     pub fn world_opening(&mut self) {
+        if self.opening.is_none() {
+            crate::spans::startup::arm("world.opening");
+        }
         self.opening.get_or_insert_with(Instant::now);
     }
 
@@ -262,10 +266,11 @@ impl Session {
 
     /// A frame drawn while playing (not in menus or loading).
     pub fn frame(&mut self, ms: f32) {
-        if let Some(opening) = self.opening.take()
-            && self.to_world_ms.is_none()
-        {
-            self.to_world_ms = Some(opening.elapsed().as_millis() as u32);
+        if let Some(opening) = self.opening.take() {
+            crate::spans::startup::mark("world.first_frame");
+            if self.to_world_ms.is_none() {
+                self.to_world_ms = Some(opening.elapsed().as_millis() as u32);
+            }
         }
         self.frames.record(ms);
         self.played_ms += ms;

crates/horizon-client/src/entry.rs

@@ -5,6 +5,7 @@ use crate::{crash, game, logging, ram_limit};
 pub fn main() {
     horizon_threads::configure_allocator();
     horizon_client::telemetry::mark_start();
+    horizon_client::spans::startup::init(std::time::Instant::now());
     let args: Vec<String> = std::env::args().skip(1).collect();
     let game_args = args.iter().take_while(|a| *a != "--dedicated");
     let help = game_args

crates/horizon-client/src/game.rs

@@ -328,6 +328,7 @@ mod world_loading;
 mod zombie_reinforcements;
 mod zombie_villagers;
 use event_loop::EventLoopCtl;
+use horizon_client::spans::startup;
 use horizon_client::{
     Result,
     assets::{Assets, RenderAssets},
@@ -16386,6 +16387,7 @@ impl ApplicationHandler for Game<'_> {
             attributes, "horizon", "horizon",
         );
         // `--offscreen` renders into a texture: its window would only show white.
+        let window_step = startup::step("window.create");
         let window = match event_loop.create_window(
             attributes
                 .with_inner_size(winit::dpi::LogicalSize::new(size[0], size[1]))
@@ -16399,6 +16401,7 @@ impl ApplicationHandler for Game<'_> {
                 return;
             }
         };
+        drop(window_step);
         horizon_client::telemetry::note_display(&window);
         match self.new_renderer(window.clone()) {
             Ok(r) => {
@@ -17845,6 +17848,15 @@ impl Game<'_> {
                             self.perf.title_shown();
                         } else if self.screen == Screen::Playing && self.player.is_some() {
                             self.perf.frame(elapsed.as_secs_f32() * 1000.0);
+                            if startup::on() {
+                                startup::view_filled(
+                                    self.world.chunks.len(),
+                                    self.streaming.pending()
+                                        + self.pending_uploads.len()
+                                        + self.remote_backlog(),
+                                    self.streaming.radius.max(0) as u32,
+                                );
+                            }
                             self.world_frames += 1;
                         }
                         self.frames += 1;
@@ -19059,7 +19071,7 @@ fn load_core(
     parallel: bool,
     progress: &Progress,
 ) -> std::result::Result<std::result::Result<CoreTables, String>, String> {
-    let assets = match Assets::load(root) {
+    let assets = match startup::time("core.assets", || Assets::load(root)) {
         Ok(assets) => assets,
         Err(e) => return Ok(Err(e.to_string())),
     };
@@ -19071,18 +19083,22 @@ fn load_core(
             let render = Step::spawn(scope, parallel, move || {
                 // A server reads no block sprites' pixels, which only the
                 // renderer draws (32 MB).
-                let render = if headless {
-                    assets.reload_for_server(root)
-                } else {
-                    assets.reload(root)
-                }
+                let render = startup::time("core.block_models_sprites", || {
+                    if headless {
+                        assets.reload_for_server(root)
+                    } else {
+                        assets.reload(root)
+                    }
+                })
                 .map_err(|e| e.to_string())?;
                 let library = render.model_library.clone();
-                let item_renderer = if headless {
-                    horizon_client::item_render::ItemRenderer::load_for_server(root, library)
-                } else {
-                    horizon_client::item_render::ItemRenderer::load_with_library(root, library)
-                }
+                let item_renderer = startup::time("core.item_models", || {
+                    if headless {
+                        horizon_client::item_render::ItemRenderer::load_for_server(root, library)
+                    } else {
+                        horizon_client::item_render::ItemRenderer::load_with_library(root, library)
+                    }
+                })
                 .map_err(|e| e.to_string());
                 Ok((render, item_renderer))
             });
@@ -19092,16 +19108,21 @@ fn load_core(
                 if headless {
                     return Ok(horizon_world::block_render::FaceVisibility::empty());
                 }
-                models::load_visibility(root, registry).map_err(|e| e.to_string())
+                startup::time("core.visibility", || {
+                    models::load_visibility(root, registry)
+                })
+                .map_err(|e| e.to_string())
             });
             let biomes = Step::spawn(scope, parallel, || {
-                Biomes::load(root).map_err(|e| e.to_string())
+                startup::time("core.biomes", || Biomes::load(root)).map_err(|e| e.to_string())
             });
             let environment = Step::spawn(scope, parallel, || {
-                Environment::load(root).map_err(|e| e.to_string())
+                startup::time("core.environment", || Environment::load(root))
+                    .map_err(|e| e.to_string())
             });
             let collision = Step::spawn(scope, parallel, || {
-                collisions::load(root, registry).map_err(|e| e.to_string())
+                startup::time("core.collision", || collisions::load(root, registry))
+                    .map_err(|e| e.to_string())
             });
             let gameplay = gameplay_tables(root, assets, parallel, progress);
             (
@@ -19139,6 +19160,7 @@ fn gameplay_tables(
     parallel: bool,
     progress: &Progress,
 ) -> std::result::Result<GameplayTables, String> {
+    let facts_step = startup::step("core.block_facts");
     let behaviour_json: serde_json::Value = serde_json::from_reader(std::io::BufReader::new(
         std::fs::File::open(root.join("block-behaviours.json")).map_err(|e| e.to_string())?,
     ))
@@ -19152,13 +19174,19 @@ fn gameplay_tables(
             .iter()
             .any(|s| !s.name.starts_with("minecraft:")),
     );
+    drop(facts_step);
     progress.set(2);
     let (light, motion) = std::thread::scope(|scope| {
         let light = Step::spawn(scope, parallel, || {
-            LightTables::load(root, assets.registry, &facts).map_err(|e| e.to_string())
+            startup::time("core.light", || {
+                LightTables::load(root, assets.registry, &facts)
+            })
+            .map_err(|e| e.to_string())
         });
-        let motion =
-            MotionResources::load_with_facts(root, assets, &facts).map_err(|e| e.to_string());
+        let motion = startup::time("core.motion", || {
+            MotionResources::load_with_facts(root, assets, &facts)
+        })
+        .map_err(|e| e.to_string());
         (light.join(), motion)
     });
     Ok(GameplayTables {
@@ -19195,6 +19223,7 @@ fn load_resources(
     // A dedicated server builds only the font (UiResources::server), as the
     // integrated server's game does: the full atlas and panorama are 144 MB.
     let load_ui = |root: &std::path::Path| {
+        let _step = startup::step("ui.load");
         if headless {
             UiResources::server(root)
         } else {
@@ -19222,7 +19251,7 @@ fn load_resources(
     let (core, loaded_gui) = loop {
         let (ui, core) = std::thread::scope(|scope| {
             let core = Step::spawn(scope, parallel, || {
-                load_core(&root, headless, parallel, progress)
+                startup::time("core", || load_core(&root, headless, parallel, progress))
             });
             let ui = load_ui(&root).map(|ui| offer(ui, &root));
             (ui, core.join())
@@ -19271,7 +19300,10 @@ fn load_resources(
     if let Some(folder) = launch_world {
         world_loading::preload_launch_world(folder, &root, assets.registry, biomes.clone());
     }
-    let tables = Arc::new(MeshTables::new(&assets, &render, &facts, visibility)?);
+    let tables = Arc::new(startup::time("pack.mesh_tables", || {
+        MeshTables::new(&assets, &render, &facts, visibility)
+    })?);
+    let pack_step = startup::step("pack.assemble");
     let pack = PackResources::assemble(
         &root,
         assets.clone(),
@@ -19281,6 +19313,7 @@ fn load_resources(
         biomes.clone(),
         Arc::new(light),
     );
+    drop(pack_step);
     log::info!("load pack tables: {:.2}s", started.elapsed().as_secs_f32());
     progress.set(4);
     if !menu {
@@ -19313,8 +19346,14 @@ fn load_resources(
     // Finish fallible CPU preparation before starting the tail, so an error
     // cannot detach a tail worker that races the next pack-loader generation.
     let prepared_at = Instant::now();
-    let game_cpu = menu.then(|| GameCpuResources::load(&root)).transpose()?;
-    let renderer_cpu = menu.then(|| prepare_renderer_textures(&pack));
+    let game_cpu = menu
+        .then(|| startup::time("handoff.game_cpu", || GameCpuResources::load(&root)))
+        .transpose()?;
+    let renderer_cpu = menu.then(|| {
+        startup::time("handoff.renderer_textures", || {
+            prepare_renderer_textures(&pack)
+        })
+    });
     if menu {
         log::info!(
             "Horizon: handoff CPU resources prepared on worker in {} ms",
@@ -19332,6 +19371,7 @@ fn load_resources(
         let handle = std::thread::Builder::new()
             .name("world-rest-loader".into())
             .spawn(move || {
+                let _step = startup::step("rest");
                 let rest = load_world_rest(
                     &root,
                     &resources,
@@ -19412,16 +19452,22 @@ fn load_world_rest(
     // Independent tables load together, within the worker budget the settings
     // allow. Only the recipes and the stations read another table: the items.
     let load_items = || -> Result<_> {
-        let items = Items::load(root, registry, facts, overlay)?;
+        let items = startup::time("rest.items", || Items::load(root, registry, facts, overlay))?;
         log::info!("load items: {:.2}s", started.elapsed().as_secs_f32());
         progress.set(7);
-        let recipes = Recipes::load(root, &items, overlay)?;
-        let stations = horizon_client::stations::StationData::load(root, &items, overlay);
+        let recipes = startup::time("rest.recipes", || Recipes::load(root, &items, overlay))?;
+        let stations = startup::time("rest.stations", || {
+            horizon_client::stations::StationData::load(root, &items, overlay)
+        });
         Ok((items, recipes, stations))
     };
-    let load_behaviours =
-        || Behaviours::load_with_gameplay(registry, root, behaviour_json, gameplay_projection);
+    let load_behaviours = || {
+        startup::time("rest.behaviours", || {
+            Behaviours::load_with_gameplay(registry, root, behaviour_json, gameplay_projection)
+        })
+    };
     let load_block_entity_render = || {
+        let _step = startup::step("rest.block_entity_render");
         if !draws {
             return Ok(Default::default());
         }
@@ -19430,6 +19476,7 @@ fn load_world_rest(
         )
     };
     let load_animate = || -> Result<_> {
+        let _step = startup::step("rest.animate_fog");
         let animate = if draws {
             horizon_client::animate_tick::AnimateTables::load(root, registry, biomes)?
         } else {
@@ -19438,6 +19485,7 @@ fn load_world_rest(
         Ok((animate, horizon_client::fog::FogTables::load(root, biomes)))
     };
     let load_mobs = || -> Result<_> {
+        let _step = startup::step("rest.mobs");
         let mob_types: HashMap<_, _> = MobTypes::load(root)?
             .types
             .into_iter()
@@ -19451,6 +19499,7 @@ fn load_world_rest(
         Ok((mob_types, mob_data))
     };
     let load_transformers = || -> Result<_> {
+        let _step = startup::step("rest.transformers_models");
         let transformers = horizon_client::block_transform::Transformers::load(root, registry)?;
         let models = (
             perspective::player_type(root),
@@ -19506,6 +19555,7 @@ fn load_world_rest(
         threads
     );
     progress.set(8);
+    let _tail = startup::step("rest.tail");
     Ok(WorldRest {
         stations,
         items,
@@ -20012,6 +20062,7 @@ impl Loading {
             attributes, "horizon", "horizon",
         );
         // `--offscreen` renders into a texture: its window would only show white.
+        let window_step = startup::step("window.create");
         let window = match event_loop.create_window(
             attributes
                 .with_inner_size(winit::dpi::LogicalSize::new(size[0], size[1]))
@@ -20025,6 +20076,7 @@ impl Loading {
                 return;
             }
         };
+        drop(window_step);
         horizon_client::telemetry::note_display(&window);
         // The renderer takes seconds to build and winit's window sits white
         // until the first frame presents, so paint the boot screen on it and
@@ -20049,6 +20101,7 @@ impl Loading {
             stop
         });
         let offscreen = self.options.as_ref().is_some_and(|o| o.offscreen);
+        let boot_step = startup::step("renderer.boot");
         match loading_renderer(
             window.clone(),
             offscreen,
@@ -20068,6 +20121,7 @@ impl Loading {
                 return;
             }
         }
+        drop(boot_step);
         // The wgpu frames take the window over: stop the boot paint thread
         // before the first one presents.
         #[cfg(target_os = "windows")]
@@ -20106,6 +20160,7 @@ impl Loading {
             renderer.release_surface();
         }
         let bundle = self.gui.as_ref().expect("the menu bundle was just set");
+        let _step = startup::step("renderer.title_artwork");
         match loading_renderer(
             window,
             offscreen,
@@ -20174,6 +20229,7 @@ impl Loading {
         }
         if !self.shown {
             self.shown = true;
+            startup::mark("loading_screen.shown");
             log::info!(
                 "Horizon: loading screen shown after {} ms",
                 self.started.elapsed().as_millis()
@@ -20194,17 +20250,19 @@ impl Loading {
             renderer.release_surface();
         }
         let renderer_at = Instant::now();
-        let renderer = build_renderer(
-            window.clone(),
-            options.offscreen,
-            &loaded.pack,
-            &gui,
-            self.renderer.as_ref(),
-            loaded.renderer_cpu,
-            self.settings.horizon.api,
-            self.settings.horizon.translucent_batching,
-            shader_packs::picked_pack_draws_flat(&self.settings),
-        )?;
+        let renderer = startup::time("renderer.world", || {
+            build_renderer(
+                window.clone(),
+                options.offscreen,
+                &loaded.pack,
+                &gui,
+                self.renderer.as_ref(),
+                loaded.renderer_cpu,
+                self.settings.horizon.api,
+                self.settings.horizon.translucent_batching,
+                shader_packs::picked_pack_draws_flat(&self.settings),
+            )
+        })?;
         log::info!(
             "Horizon: world renderer ready after {} ms",
             self.started.elapsed().as_millis()
@@ -20221,6 +20279,7 @@ impl Loading {
         // like the single copy an integrated server would share.
         let res: &'static Arc<Resources> = Box::leak(Box::new(loaded.resources));
         let game_at = Instant::now();
+        let game_step = startup::step("handoff.start_game");
         let Some(mut game) = start_game(
             res,
             loaded.pack,
@@ -20237,6 +20296,7 @@ impl Loading {
         else {
             return Ok(None);
         };
+        drop(game_step);
         let game_ms = game_at.elapsed().as_millis();
         game.window = Some(window);
         game.renderer = Some(renderer);

crates/horizon-client/src/game/integrated.rs

@@ -17,6 +17,7 @@ use super::{
     world_loading::{WorldLoader, WorldRequest},
 };
 use horizon_client::level_loading::{STATUS_RADIUS, Stage};
+use horizon_client::spans::startup;
 use horizon_client::{
     Result,
     audio::Audio,
@@ -685,9 +686,11 @@ impl Game<'_> {
         self.native_server_about_to_start(false);
         self.native_lifecycle_open_started();
         // `load_world`, with the world `serve` started preparing.
-        self.join_world_rest()?;
+        startup::time("server.wait_rest", || self.join_world_rest())?;
         self.perf.world_opening();
+        let open_step = startup::step("server.open_world");
         self.finish_open_world(opening.world?.finish()?)?;
+        drop(open_step);
         if let Some(sync) = opening.sync {
             sync.adopt(&self.pack_files());
         }
@@ -751,7 +754,9 @@ impl Game<'_> {
             guest_commands: false,
             force_game_mode: false,
         };
-        self.lan_host(settings, net, config, None)?;
+        startup::time("server.lan_host", || {
+            self.lan_host(settings, net, config, None)
+        })?;
         let commands = self.allow_commands();
         let pipe = {
             let host = self
@@ -781,7 +786,7 @@ impl Game<'_> {
         view.start(Stage::LoadInitialChunks, 0);
         view.finish(Stage::LoadInitialChunks);
         let _ = ready.send(Ok((pipe, self.level.name.clone())));
-        self.settle_spawn()
+        startup::time("server.settle_spawn", || self.settle_spawn())
     }
 
     /// `MinecraftServer.stopServer`: the players are saved and sent away,
@@ -1141,7 +1146,9 @@ fn serve(
         world,
         sync: preload_sync_data(&options.resources),
     };
-    let mut server = match server_game(res, pack, options, settings) {
+    let mut server = match startup::time("server.game_new", || {
+        server_game(res, pack, options, settings)
+    }) {
         Ok(server) => server,
         Err(e) => {
             // The world's session lock is let go before the failure is told.

crates/horizon-client/src/game/remote.rs

@@ -2972,6 +2972,9 @@ impl Game<'_> {
                     &p.spawn.dimension,
                     self.dim.id(),
                 );
+                if dimension_changed {
+                    horizon_client::spans::startup::arm("dimension.change");
+                }
                 self.enter_remote_dimension(&p.spawn, true, reason);
                 if dimension_changed {
                     // LevelExtractor.setLevel: a new level drops the highlights.

crates/horizon-telemetry-server/Cargo.lock

@@ -253,6 +253,7 @@ name = "horizon-telemetry-server"
 version = "0.1.0"
 dependencies = [
  "axum",
+ "flate2",
  "horizon-telemetry",
  "http-body-util",
  "rusqlite",

crates/horizon-telemetry-server/Cargo.toml

@@ -19,6 +19,8 @@ rusqlite = { version = "0.37", features = ["bundled"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 ruzstd = "0.9"
+# The export answers gzip (pure Rust; ureq already pulls it in).
+flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 # Opens the hub's bug reports as GitHub issues (blocking, run off the async workers; rustls).
 ureq = { version = "2.12", features = ["json"] }
 

crates/horizon-telemetry-server/README.md

@@ -15,6 +15,8 @@ It runs at **https://telemetry.playhorizon.dev** (`DEFAULT_ENDPOINT` in `crates/
 | `DELETE /v1/installs/<id>` | Deletes everything that install sent. Its share of the daily totals is rebuilt at once. |
 | `GET /v1/aggregate?days=30` | The totals the website reads. Cached 10 minutes, with `Access-Control-Allow-Origin` set to the website. |
 | `GET /v1/benchmarks/summary` | Players' benchmark uploads grouped by GPU, CPU, render distance and game, for the website's Benchmarks page (below). Cached 10 minutes, same `Access-Control-Allow-Origin`. |
+| `GET /v1/export/benchmarks?after=<id>&limit=100` | Owner-only: the raw benchmark reports, oldest first, `{"reports": [{"id", "day", "version", "report"}], "next": <id>\|null}` (pass `next` as `after`). No install ID. Needs `Authorization: Bearer <TELEMETRY_EXPORT_TOKEN>`: 401 without it, 404 while the server has no token, gzip if the client accepts it, never cached. |
+| `GET /v1/export/hardware?days=30&offset=0&limit=200` | Owner-only, same token: the play sessions of the last `days` per machine (CPU, threads, physical cores, RAM, GPU, drive, OS…), busiest first, with installs, sessions, hours, mean p50/p99 and the frame buckets. No minimum number of installs; no install ID. `next_offset` pages. |
 | `POST /v1/live` | A game's "I'm playing" ping (JSON, at most 512 bytes). Returns 204. Returns 400 for a bad ping, 413 for one too large, 429 over the address's limit, 503 when it holds `LIVE_MAX_SESSIONS` already. |
 | `GET /v1/live` | How many play now, exactly: `playing`, `lan`, `versions` (the 12 most played, the rest under `other`), `platforms`. Made again at most every 15 seconds, with the same `Access-Control-Allow-Origin`. |
 | `GET /v1/live/history?range=24h\|7d\|30d\|all&series=players\|official` | The live count (`players`, the default) or the official server's player count (`official`) over time for playhorizon.dev/stats: `series` of `[start, average, peak]`, the range's `peak` and the `all_time` one with their minute, the `latest` minute, the average per platform ([docs/telemetry.md](../../docs/telemetry.md)). Made again at most every minute (`Cache-Control: max-age=60`), on the live count's rate limit. 400 for another range or series. |
@@ -91,6 +93,7 @@ All are environment variables:
 - `HORIZON_TELEMETRY_HUB_KEEP_DAYS` (90)
 - `HORIZON_TELEMETRY_GITHUB_TOKEN`: a fine-grained token with Issues read and write on the repository only (unset: `POST /hub/report` answers 503 and the form points to GitHub and Discord)
 - `HORIZON_TELEMETRY_GITHUB_REPO` (`horizon-rust/horizon`)
+- `TELEMETRY_EXPORT_TOKEN` (or `HORIZON_TELEMETRY_EXPORT_TOKEN`): the bearer token of `/v1/export/*` (unset: those answer 404). A long random secret; requests with a wrong one count against the address's rate limit.
 
 ## Hosting
 

crates/horizon-telemetry-server/src/export.rs

@@ -0,0 +1,281 @@
+//! The protected, read-only export for the people who tune performance: the raw benchmark
+//! reports and the per-hardware totals, with no minimum number of installs (the public
+//! `/v1/aggregate` and `/v1/benchmarks/summary` hide small groups).
+//!
+//! - `GET /v1/export/benchmarks?after=<id>&limit=100`: `{"reports": [{"id", "day", "version",
+//!   "report"}], "next": <id>|null}`, oldest first. Pass `next` as `after` for the next page.
+//! - `GET /v1/export/hardware?days=30&offset=0&limit=200`: `{"machines": [...], "next_offset"}`:
+//!   one entry per distinct machine description (CPU threads and cores, RAM, GPU, drive, OS…),
+//!   with its installs, sessions, hours and frame-time numbers, busiest first.
+//!
+//! Both need `Authorization: Bearer <TELEMETRY_EXPORT_TOKEN>`, are off (404) while the server has
+//! no such token, answer gzip to a client that accepts it, and are never cached. Install IDs
+//! never leave the database: a report has no owner here, and a machine only a count of them.
+use std::collections::{HashMap, HashSet};
+use std::io::Write;
+use std::sync::Arc;
+
+use axum::extract::{Query, Request, State};
+use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use horizon_telemetry::payload::BUCKET_EDGES_MS;
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+use crate::store::{Extra, Store, unpack};
+use crate::{App, peer, status};
+
+/// Equal, in a time that depends on the token's length only, not on where they differ.
+pub(crate) fn same(sent: &[u8], token: &[u8]) -> bool {
+    let mut diff = sent.len() ^ token.len();
+    for (i, t) in token.iter().enumerate() {
+        diff |= usize::from(sent.get(i).copied().unwrap_or(0) ^ t);
+    }
+    diff == 0
+}
+
+/// The answer to send instead when the request's `Bearer` token doesn't check out: 404 while the
+/// export is off, 401 for a missing or wrong token, 429 once an address has failed too often.
+/// None: go ahead.
+fn refuse(
+    app: &App,
+    headers: &HeaderMap,
+    req_peer: Option<std::net::SocketAddr>,
+) -> Option<Response> {
+    let Some(token) = app.config.export_token.as_deref() else {
+        return Some(StatusCode::NOT_FOUND.into_response());
+    };
+    let sent = headers
+        .get(header::AUTHORIZATION)
+        .and_then(|v| v.to_str().ok())
+        .and_then(|v| v.strip_prefix("Bearer "))
+        .unwrap_or("");
+    if same(sent.as_bytes(), token.as_bytes()) {
+        return None;
+    }
+    // Only failures take from the address's bucket, so guessing is slow and using the token isn't.
+    if !app.allow(app.client(headers, req_peer)) {
+        return Some(status(StatusCode::TOO_MANY_REQUESTS, "too many requests"));
+    }
+    Some(StatusCode::UNAUTHORIZED.into_response())
+}
+
+/// `value` as JSON, gzipped when the client accepts it.
+fn answer(headers: &HeaderMap, value: Value) -> Response {
+    let json = value.to_string().into_bytes();
+    let gzip = headers
+        .get(header::ACCEPT_ENCODING)
+        .and_then(|v| v.to_str().ok())
+        .is_some_and(|v| {
+            v.split(',').any(|c| {
+                c.trim()
+                    .split(';')
+                    .next()
+                    .is_some_and(|c| c.eq_ignore_ascii_case("gzip"))
+            })
+        });
+    let mut res = if gzip {
+        let mut z = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        let packed = z.write_all(&json).and_then(|()| z.finish());
+        match packed {
+            Ok(packed) => {
+                let mut res = packed.into_response();
+                res.headers_mut()
+                    .insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
+                res
+            }
+            Err(_) => json.into_response(),
+        }
+    } else {
+        json.into_response()
+    };
+    let h = res.headers_mut();
+    h.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("application/json"),
+    );
+    h.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+    h.insert(header::VARY, HeaderValue::from_static("Accept-Encoding"));
+    res
+}
+
+#[derive(Deserialize)]
+pub struct Page {
+    after: Option<i64>,
+    limit: Option<u32>,
+}
+
+/// `GET /v1/export/benchmarks`: the raw benchmark reports, `limit` (1-500, default 100) a page.
+pub async fn benchmarks(
+    State(app): State<Arc<App>>,
+    Query(page): Query<Page>,
+    req: Request,
+) -> Response {
+    if let Some(res) = refuse(&app, req.headers(), peer(&req)) {
+        return res;
+    }
+    let limit = page.limit.unwrap_or(100).clamp(1, 500);
+    let listed = app
+        .store()
+        .export_benchmarks(page.after.unwrap_or(0), limit);
+    match listed {
+        Ok(reports) => {
+            let next = (reports.len() as u32 == limit)
+                .then(|| reports.last().map(|r| r["id"].clone()))
+                .flatten();
+            answer(req.headers(), json!({ "reports": reports, "next": next }))
+        }
+        Err(e) => {
+            eprintln!("horizon-telemetry-server: could not export the benchmarks: {e}");
+            status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
+        }
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Machines {
+    days: Option<u32>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+}
+
+/// `GET /v1/export/hardware`: play sessions of the last `days` (1-366, default 30) per machine
+/// description, `limit` (1-1000, default 200) machines a page from `offset`.
+pub async fn hardware(
+    State(app): State<Arc<App>>,
+    Query(q): Query<Machines>,
+    req: Request,
+) -> Response {
+    if let Some(res) = refuse(&app, req.headers(), peer(&req)) {
+        return res;
+    }
+    let days = q.days.unwrap_or(30).clamp(1, 366);
+    let (offset, limit) = (q.offset.unwrap_or(0), q.limit.unwrap_or(200).clamp(1, 1000));
+    let to = (app.today)();
+    let made = app.store().export_machines(to + 1 - days.min(to + 1), to);
+    match made {
+        Ok(all) => {
+            let total = all.len();
+            let machines: Vec<Value> = all.into_iter().skip(offset).take(limit).collect();
+            let next = (offset + machines.len() < total).then_some(offset + machines.len());
+            answer(
+                req.headers(),
+                json!({
+                    "from_day": to + 1 - days.min(to + 1), "to_day": to, "total_machines": total,
+                    "bucket_edges_ms": BUCKET_EDGES_MS.map(|e| (f64::from(e) * 10.0).round() / 10.0),
+                    "machines": machines, "next_offset": next,
+                }),
+            )
+        }
+        Err(e) => {
+            eprintln!("horizon-telemetry-server: could not export the machines: {e}");
+            status(StatusCode::SERVICE_UNAVAILABLE, "try again later")
+        }
+    }
+}
+
+impl Store {
+    /// Benchmark reports with an id above `after`, oldest first.
+    pub fn export_benchmarks(&self, after: i64, limit: u32) -> rusqlite::Result<Vec<Value>> {
+        let mut q = self.db.prepare(
+            "SELECT rowid, day, version, body FROM reports WHERE kind = 3 AND rowid > ?1
+             ORDER BY rowid LIMIT ?2",
+        )?;
+        q.query_map(rusqlite::params![after, limit], |r| {
+            Ok(json!({
+                "id": r.get::<_, i64>(0)?, "day": r.get::<_, u32>(1)?, "version": r.get::<_, String>(2)?,
+                "report": crate::store::inflate(&r.get::<_, Vec<u8>>(3)?),
+            }))
+        })?
+        .collect()
+    }
+
+    /// Play sessions in `from..=to` per machine description, the busiest first. A machine is its
+    /// CPU, memory, GPU, drive and system: two sessions differing only in the screen or in
+    /// battery against mains count as one. No group is left out for being small.
+    pub fn export_machines(&self, from: u32, to: u32) -> rusqlite::Result<Vec<Value>> {
+        #[derive(Default)]
+        struct Machine {
+            installs: HashSet<Vec<u8>>,
+            sessions: u64,
+            seconds: u64,
+            p50_sum: f64,
+            p99_sum: f64,
+            buckets: Vec<u64>,
+            peak_ram_mb: u32,
+            description: Value,
+        }
+        let mut machines: HashMap<String, Machine> = HashMap::new();
+        let mut q = self.db.prepare(
+            "SELECT perf.install, perf.seconds, perf.frames, perf.peak_ram_mb, systems.os, systems.os_version,
+             systems.arch, systems.cpu, systems.threads, systems.ram_gb, systems.gpu, systems.driver,
+             systems.backend, systems.extra
+             FROM perf JOIN systems ON systems.id = perf.system WHERE perf.day BETWEEN ?1 AND ?2",
+        )?;
+        let mut rows = q.query([from, to])?;
+        while let Some(r) = rows.next()? {
+            let Some(frames) = unpack(&r.get::<_, Vec<u8>>(2)?) else {
+                continue;
+            };
+            let extra: Extra = serde_json::from_str(&r.get::<_, String>(13)?).unwrap_or_default();
+            let description = json!({
+                "os": r.get::<_, String>(4)?, "os_version": r.get::<_, String>(5)?,
+                "arch": r.get::<_, String>(6)?, "cpu": r.get::<_, String>(7)?,
+                "threads": r.get::<_, u16>(8)?, "cores": extra.cores,
+                "cpu_base_mhz": extra.cpu_base_mhz, "cpu_boost_mhz": extra.cpu_boost_mhz,
+                "ram_gb": r.get::<_, u16>(9)?,
+                "gpu": r.get::<_, String>(10)?, "gpu_vendor": extra.gpu_vendor, "gpu_kind": extra.gpu_kind,
+                "vram_mb": extra.vram_mb, "driver": r.get::<_, String>(11)?,
+                "backend": r.get::<_, String>(12)?, "storage": extra.storage, "form": extra.form,
+            });
+            let m = machines.entry(description.to_string()).or_default();
+            if m.buckets.is_empty() {
+                m.buckets = vec![0; frames.buckets.len()];
+                m.description = description;
+            }
+            m.installs.insert(r.get(0)?);
+            m.sessions += 1;
+            m.seconds += r.get::<_, u64>(1)?;
+            m.peak_ram_mb = m.peak_ram_mb.max(r.get(3)?);
+            m.p50_sum += f64::from(frames.p50_ms);
+            m.p99_sum += f64::from(frames.p99_ms);
+            for (sum, n) in m.buckets.iter_mut().zip(&frames.buckets) {
+                *sum += u64::from(*n);
+            }
+        }
+        let mut list: Vec<(String, Machine)> = machines.into_iter().collect();
+        // Busiest first; the description breaks ties so pages are stable.
+        list.sort_by(|a, b| b.1.sessions.cmp(&a.1.sessions).then(a.0.cmp(&b.0)));
+        let avg = |sum: f64, n: u64| (sum / n as f64 * 10.0).round() / 10.0;
+        Ok(list
+            .into_iter()
+            .map(|(_, m)| {
+                let mut out = m.description;
+                out["installs"] = m.installs.len().into();
+                out["sessions"] = m.sessions.into();
+                out["hours"] = ((m.seconds as f64 / 360.0).round() / 10.0).into();
+                out["p50_ms"] = avg(m.p50_sum, m.sessions).into();
+                out["p99_ms"] = avg(m.p99_sum, m.sessions).into();
+                out["peak_ram_mb"] = m.peak_ram_mb.into();
+                out["frame_buckets"] = m.buckets.into();
+                out
+            })
+            .collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::same;
+
+    #[test]
+    fn tokens_are_compared_whole() {
+        assert!(same(b"abc", b"abc"));
+        assert!(!same(b"abd", b"abc"));
+        assert!(!same(b"ab", b"abc"));
+        assert!(!same(b"abcd", b"abc"));
+        assert!(!same(b"", b"abc"));
+        // A token with a trailing NUL byte isn't the token without it.
+        assert!(!same(b"abc\0", b"abc"));
+    }
+}

crates/horizon-telemetry-server/src/lib.rs

@@ -5,6 +5,8 @@
 //! - `GET /v1/aggregate?days=30`: the anonymous totals the website reads, cached.
 //! - `GET /v1/benchmarks/summary`: players' benchmark results grouped by GPU and CPU, groups of at
 //!   least 3 installs only (`summary`), cached; the website's Benchmarks page reads it.
+//! - `GET /v1/export/benchmarks` and `/v1/export/hardware`: raw benchmark reports and per-machine
+//!   totals for the perf work, behind a bearer token (`export`); off without one.
 //! - `POST /v1/live`: a game's anonymous "I'm playing" ping (`live`, in memory only).
 //! - `GET /v1/live`: how many play now, exactly, cached briefly.
 //! - `GET /v1/live/history?range=24h|7d|30d|all&series=players|official`: that count, or the
@@ -32,6 +34,7 @@ use axum::routing::{delete, get, post};
 use horizon_telemetry::batch;
 
 pub mod classes;
+pub mod export;
 pub mod github;
 pub mod history;
 pub mod hub;
@@ -81,6 +84,8 @@ pub struct Config {
     pub hub_reports_per_day: u32,
     /// Days reports are kept (opened as issues long before).
     pub hub_keep_days: u32,
+    /// The bearer token the protected export (`export`) wants; unset, it answers 404.
+    pub export_token: Option<String>,
     /// Opens the website's bug reports as issues; unset, `POST /hub/report` answers 503.
     pub github_token: Option<String>,
     /// Where those issues go, `owner/name`.
@@ -106,6 +111,7 @@ impl Default for Config {
             hub_posts_per_hour: 20.0,
             hub_reports_per_day: 30,
             hub_keep_days: 90,
+            export_token: None,
             github_token: None,
             github_repo: "horizon-rust/horizon".into(),
         }
@@ -118,7 +124,8 @@ impl Config {
     /// `…_LIVE_PER_HOUR`, `…_LIVE_WINDOW_SECONDS`, `…_LIVE_MAX_SESSIONS`, `…_LIVE_MINUTE_DAYS`,
     /// `…_LIVE_HOUR_DAYS`, `…_OFFICIAL_SERVER` (`off` for none), `…_HUB_PER_HOUR`,
     /// `…_HUB_POSTS_PER_HOUR`, `…_HUB_REPORTS_PER_DAY`, `…_HUB_KEEP_DAYS`, `…_GITHUB_TOKEN` and
-    /// `…_GITHUB_REPO`. An empty token counts as unset.
+    /// `…_GITHUB_REPO`, plus `TELEMETRY_EXPORT_TOKEN` (or `HORIZON_TELEMETRY_EXPORT_TOKEN`) for the
+    /// protected export. An empty token counts as unset.
     pub fn from_env() -> Self {
         let var = |k: &str| std::env::var(format!("HORIZON_TELEMETRY_{k}")).ok();
         let secret = |k: &str| {
@@ -174,6 +181,11 @@ impl Config {
             hub_keep_days: var("HUB_KEEP_DAYS")
                 .and_then(|v| v.parse().ok())
                 .unwrap_or(d.hub_keep_days),
+            export_token: std::env::var("TELEMETRY_EXPORT_TOKEN")
+                .ok()
+                .map(|v| v.trim().to_owned())
+                .filter(|v| !v.is_empty())
+                .or_else(|| secret("EXPORT_TOKEN")),
             github_token: secret("GITHUB_TOKEN"),
             github_repo: var("GITHUB_REPO").unwrap_or(d.github_repo),
         }
@@ -718,6 +730,8 @@ pub fn router(app: Arc<App>) -> Router {
         .route("/v1/installs/{id}", delete(forget))
         .route("/v1/aggregate", get(aggregate))
         .route("/v1/benchmarks/summary", get(benchmarks))
+        .route("/v1/export/benchmarks", get(export::benchmarks))
+        .route("/v1/export/hardware", get(export::hardware))
         .route("/hub/vote", post(hub::vote).options(hub::preflight))
         .route("/hub/votes", get(hub::votes))
         .route("/hub/report", post(intake::report).options(hub::preflight))

crates/horizon-telemetry-server/tests/export.rs

@@ -0,0 +1,313 @@
+//! The protected export through its HTTP interface: off without a token, 401 without the right
+//! one, raw benchmark reports paginated, per-machine totals with the CPU core counts, gzip.
+use std::io::Read;
+use std::sync::Arc;
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use horizon_telemetry::Item;
+use horizon_telemetry::batch::{self, Batch};
+use horizon_telemetry::payload::{Payload, Perf, Settings, Source, System, Timings};
+use horizon_telemetry_server::store::Store;
+use horizon_telemetry_server::{App, Config, router};
+use http_body_util::BodyExt;
+use serde_json::{Value, json};
+use tower::ServiceExt;
+
+const DAY: u32 = 20_741;
+const TOKEN: &str = "a-long-random-export-token";
+
+fn install(n: u32) -> String {
+    format!("{n:032x}")
+}
+
+fn app(token: Option<&str>) -> Arc<App> {
+    App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            requests_per_hour: 100_000.0,
+            export_token: token.map(str::to_owned),
+            ..Config::default()
+        },
+        Box::new(|| DAY),
+    )
+}
+
+fn perf(cpu: &str, threads: u16, cores: u16, storage: &str) -> Item {
+    let mut t = Timings::new();
+    (0..3000).for_each(|i| t.record(6.0 + (i % 50) as f32 / 10.0));
+    let mut item = Item::new(
+        Source::Game,
+        "v0.2.97",
+        Payload::Perf(Perf {
+            system: System {
+                os: "windows".into(),
+                cpu: cpu.into(),
+                threads,
+                cores,
+                ram_gb: 32,
+                gpu: "NVIDIA GeForce RTX 4070".into(),
+                gpu_vendor: "nvidia".into(),
+                gpu_kind: "discrete".into(),
+                backend: "Dx12".into(),
+                storage: storage.into(),
+                ..System::default()
+            },
+            settings: Settings {
+                render_distance: 12,
+                ..Settings::default()
+            },
+            seconds: 3600,
+            frames: t.summary(),
+            peak_ram_mb: 2500,
+            ..Perf::default()
+        }),
+    );
+    item.day = DAY;
+    item
+}
+
+fn benchmark(n: u32) -> Item {
+    let mut item = Item::new(
+        Source::Game,
+        "v0.2.97",
+        Payload::Benchmark(json!({
+            "format": "horizon-benchmark-report", "version": 1, "n": n,
+            "hardware": { "os": "windows", "cpu_threads": 32, "cpu_cores": 16, "storage": "nvme" },
+        })),
+    );
+    item.day = DAY;
+    item
+}
+
+async fn send(app: &Arc<App>, req: Request<Body>) -> (StatusCode, Vec<u8>, Option<String>) {
+    let res = router(app.clone()).oneshot(req).await.unwrap();
+    let code = res.status();
+    let encoding = res
+        .headers()
+        .get("content-encoding")
+        .map(|v| v.to_str().unwrap().to_owned());
+    (
+        code,
+        res.into_body().collect().await.unwrap().to_bytes().to_vec(),
+        encoding,
+    )
+}
+
+async fn upload(app: &Arc<App>, id: u32, items: Vec<Item>) {
+    let body = batch::encode(&Batch::new(&install(id), items));
+    let req = Request::post("/v1/batch")
+        .header("content-encoding", "zstd")
+        .body(Body::from(body))
+        .unwrap();
+    assert_eq!(send(app, req).await.0, StatusCode::NO_CONTENT);
+}
+
+async fn get(app: &Arc<App>, uri: &str, token: Option<&str>) -> (StatusCode, Value) {
+    let mut req = Request::get(uri);
+    if let Some(t) = token {
+        req = req.header("authorization", format!("Bearer {t}"));
+    }
+    let (code, body, encoding) = send(app, req.body(Body::empty()).unwrap()).await;
+    assert_eq!(encoding, None, "plain unless gzip is asked for");
+    (code, serde_json::from_slice(&body).unwrap_or(Value::Null))
+}
+
+#[tokio::test]
+async fn it_is_off_without_a_token_and_refuses_a_wrong_one() {
+    for uri in ["/v1/export/benchmarks", "/v1/export/hardware"] {
+        let off = app(None);
+        assert_eq!(get(&off, uri, None).await.0, StatusCode::NOT_FOUND);
+        assert_eq!(get(&off, uri, Some(TOKEN)).await.0, StatusCode::NOT_FOUND);
+
+        let on = app(Some(TOKEN));
+        assert_eq!(get(&on, uri, None).await.0, StatusCode::UNAUTHORIZED);
+        for wrong in [
+            "",
+            "a-long-random-export-toke",
+            "a-long-random-export-tokenX",
+            "nope",
+        ] {
+            assert_eq!(
+                get(&on, uri, Some(wrong)).await.0,
+                StatusCode::UNAUTHORIZED,
+                "{wrong:?}"
+            );
+        }
+        // The wrong scheme, even with the right secret.
+        let req = Request::get(uri)
+            .header("authorization", format!("Basic {TOKEN}"))
+            .body(Body::empty())
+            .unwrap();
+        assert_eq!(send(&on, req).await.0, StatusCode::UNAUTHORIZED);
+        assert_eq!(get(&on, uri, Some(TOKEN)).await.0, StatusCode::OK);
+    }
+}
+
+#[tokio::test]
+async fn guessing_is_rate_limited_but_using_the_token_is_not() {
+    let app = App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            requests_per_hour: 10.0,
+            export_token: Some(TOKEN.into()),
+            ..Config::default()
+        },
+        Box::new(|| DAY),
+    );
+    let mut codes = Vec::new();
+    for _ in 0..5 {
+        codes.push(get(&app, "/v1/export/hardware", Some("guess")).await.0);
+    }
+    assert_eq!(codes[0], StatusCode::UNAUTHORIZED);
+    assert_eq!(codes[4], StatusCode::TOO_MANY_REQUESTS);
+    for _ in 0..20 {
+        assert_eq!(
+            get(&app, "/v1/export/hardware", Some(TOKEN)).await.0,
+            StatusCode::OK
+        );
+    }
+}
+
+#[tokio::test]
+async fn benchmark_reports_come_out_raw_and_in_pages() {
+    let app = app(Some(TOKEN));
+    for n in 0..5 {
+        upload(&app, n + 1, vec![benchmark(n)]).await;
+    }
+    // A play session is not a benchmark.
+    upload(&app, 9, vec![perf("AMD Ryzen 9 7950X", 32, 16, "nvme")]).await;
+
+    let (code, first) = get(&app, "/v1/export/benchmarks?limit=2", Some(TOKEN)).await;
+    assert_eq!(code, StatusCode::OK);
+    let reports = first["reports"].as_array().unwrap();
+    assert_eq!(reports.len(), 2);
+    assert_eq!(reports[0]["report"]["hardware"]["cpu_threads"], 32);
+    assert_eq!(reports[0]["report"]["hardware"]["cpu_cores"], 16);
+    assert_eq!(
+        (reports[0]["day"].as_u64(), reports[0]["version"].as_str()),
+        (Some(DAY.into()), Some("v0.2.97"))
+    );
+
+    let mut seen = vec![
+        reports[0]["report"]["n"].clone(),
+        reports[1]["report"]["n"].clone(),
+    ];
+    let mut next = first["next"].clone();
+    let mut pages = 1;
+    while let Some(after) = next.as_i64() {
+        let (_, page) = get(
+            &app,
+            &format!("/v1/export/benchmarks?limit=2&after={after}"),
+            Some(TOKEN),
+        )
+        .await;
+        seen.extend(
+            page["reports"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|r| r["report"]["n"].clone()),
+        );
+        next = page["next"].clone();
+        pages += 1;
+    }
+    assert_eq!(seen, [0, 1, 2, 3, 4].map(Value::from));
+    assert_eq!(pages, 3);
+
+    // Whose report it was never leaves the database.
+    let (_, all) = get(&app, "/v1/export/benchmarks", Some(TOKEN)).await;
+    assert_eq!(all["next"], Value::Null);
+    assert!(!all.to_string().contains(&install(1)));
+    assert!(!all.to_string().contains("install"));
+}
+
+#[tokio::test]
+async fn machines_come_with_their_core_counts_and_no_minimum() {
+    let app = app(Some(TOKEN));
+    // One install on a 16-core / 32-thread machine, two on a 4-core / 8-thread one.
+    upload(&app, 1, vec![perf("AMD Ryzen 9 7950X", 32, 16, "nvme")]).await;
+    upload(&app, 2, vec![perf("Intel Core i5-8250U", 8, 4, "ssd")]).await;
+    upload(&app, 3, vec![perf("Intel Core i5-8250U", 8, 4, "ssd")]).await;
+
+    let (code, out) = get(&app, "/v1/export/hardware?days=30", Some(TOKEN)).await;
+    assert_eq!(code, StatusCode::OK);
+    assert_eq!(out["total_machines"], 2);
+    let machines = out["machines"].as_array().unwrap();
+    // Busiest first.
+    assert_eq!(machines[0]["cpu"], "Intel Core i5-8250U");
+    assert_eq!(
+        (
+            machines[0]["threads"].as_u64(),
+            machines[0]["cores"].as_u64()
+        ),
+        (Some(8), Some(4))
+    );
+    assert_eq!(
+        (
+            machines[0]["installs"].as_u64(),
+            machines[0]["sessions"].as_u64()
+        ),
+        (Some(2), Some(2))
+    );
+    assert_eq!(machines[0]["storage"], "ssd");
+    assert_eq!(machines[0]["hours"], 2.0);
+    assert!(machines[0]["p99_ms"].as_f64().unwrap() > 0.0);
+    assert_eq!(
+        machines[0]["frame_buckets"].as_array().unwrap().len(),
+        out["bucket_edges_ms"].as_array().unwrap().len() + 1
+    );
+    assert_eq!(machines[1]["cpu"], "AMD Ryzen 9 7950X");
+    assert_eq!(
+        (
+            machines[1]["threads"].as_u64(),
+            machines[1]["cores"].as_u64()
+        ),
+        (Some(32), Some(16))
+    );
+    assert_eq!(machines[1]["installs"], 1);
+    assert!(!out.to_string().contains(&install(1)));
+
+    // Pages by offset.
+    let (_, page) = get(&app, "/v1/export/hardware?limit=1", Some(TOKEN)).await;
+    assert_eq!(page["machines"].as_array().unwrap().len(), 1);
+    assert_eq!(page["next_offset"], 1);
+    let (_, page) = get(&app, "/v1/export/hardware?limit=1&offset=1", Some(TOKEN)).await;
+    assert_eq!(page["machines"][0]["cpu"], "AMD Ryzen 9 7950X");
+    assert_eq!(page["next_offset"], Value::Null);
+
+    // Outside the window: nothing.
+    let old = App::with_clock(
+        Store::open(":memory:").unwrap(),
+        Config {
+            export_token: Some(TOKEN.into()),
+            ..Config::default()
+        },
+        Box::new(|| DAY + 90),
+    );
+    let (_, none) = get(&old, "/v1/export/hardware?days=30", Some(TOKEN)).await;
+    assert_eq!(none["total_machines"], 0);
+}
+
+#[tokio::test]
+async fn gzip_when_asked() {
+    let app = app(Some(TOKEN));
+    upload(&app, 1, vec![benchmark(7)]).await;
+    let req = Request::get("/v1/export/benchmarks")
+        .header("authorization", format!("Bearer {TOKEN}"))
+        .header("accept-encoding", "br, gzip;q=0.8")
+        .body(Body::empty())
+        .unwrap();
+    let res = router(app.clone()).oneshot(req).await.unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(res.headers()["content-encoding"], "gzip");
+    assert_eq!(res.headers()["content-type"], "application/json");
+    assert_eq!(res.headers()["cache-control"], "no-store");
+    let body = res.into_body().collect().await.unwrap().to_bytes();
+    let mut json = String::new();
+    flate2::read::GzDecoder::new(&body[..])
+        .read_to_string(&mut json)
+        .unwrap();
+    let out: Value = serde_json::from_str(&json).unwrap();
+    assert_eq!(out["reports"][0]["report"]["n"], 7);
+}

docs/telemetry.md

@@ -108,6 +108,7 @@ The launcher reads the count when it opens and every minute after, and shows "N
 - `DELETE /v1/installs/<install_id>`: deletes everything sent under that ID.
 - `GET /v1/aggregate`: the anonymous totals the website's results page reads. Its `breakdowns` split the play sessions of the period by hardware class (below), each class with its installs, sessions, hours and mean p50 and p99 frame times.
 - `GET /v1/benchmarks/summary`: players' benchmark uploads grouped by GPU, CPU, render distance and game for playhorizon.dev/benchmarks. A group shows only with at least 3 different installs, and only its medians and counts: no install ID, day, OS version, driver, exact memory or screen. Uploads are never published one by one. Each group's `breakdowns` count its installs per hardware class.
+- `GET /v1/export/benchmarks` and `GET /v1/export/hardware`: for the maintainers' performance work only, behind a bearer token the public cannot use (404 while the server has none). The raw benchmark reports and the play sessions per machine description, including CPU threads and physical cores, with no minimum group size. Nothing identifying: no install ID, and only what a player already agreed to send (this page's lists).
 - `POST /v1/live` and `GET /v1/live`: the "I'm playing" ping and the live count.
 - `GET /v1/live/history?range=24h|7d|30d|all&series=players|official`: the live count, or the official server's, over time (above).
 

tests/test_startup_trace.py

@@ -0,0 +1,32 @@
+import tempfile
+import unittest
+import pathlib
+from tools import startup_trace
+
+
+class StartupTraceTests(unittest.TestCase):
+    def test_steps_and_marks_from_the_games_lines(self):
+        lines = [
+            '{"ph":"M","name":"thread_name","pid":1,"tid":1,"args":{"name":"main"}}',
+            '{"pid":1,"tid":1,"ph":"X","name":"renderer.boot","ts":20000,"dur":150000}',
+            '{"ph":"M","name":"thread_name","pid":1,"tid":2,"args":{"name":"world-loader"}}',
+            '{"pid":1,"tid":2,"ph":"X","name":"ui.load","ts":5000,"dur":400000}',
+            '{"pid":1,"tid":1,"ph":"i","s":"g","name":"title.shown","ts":450000}',
+            '{"pid":1,"tid":1,"ph":"i","s":"g","name":"title.shown","ts":900000}',
+            '{"pid":1,"tid":1,"ph":"X","name":"torn',
+        ]
+        with tempfile.TemporaryDirectory() as d:
+            path = pathlib.Path(d) / "trace.jsonl"
+            path.write_text("\n".join(lines))
+            steps, marks = startup_trace.summarize(startup_trace.read_trace(path))
+        self.assertEqual(steps, [(5.0, 405.0, "ui.load", "world-loader"),
+                                 (20.0, 170.0, "renderer.boot", "main")])
+        # The first time a mark happened.
+        self.assertEqual(marks, {"title.shown": 450.0})
+
+    def test_a_missing_trace_has_no_events(self):
+        self.assertEqual(startup_trace.read_trace("/nonexistent/trace.jsonl"), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/startup_trace.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""The game's start, step by step: launches it the way a player does (no
+control socket, so the title shows while the world tables load), with
+`HORIZON_STARTUP_TRACE` set, and prints the timeline the game wrote: every
+named step (resource tables, the renderer's device and pipelines, the world's
+open on the server thread) on the thread that ran it, and the marks (title
+drawn, first frame in a world, the view filled).
+
+    python3 tools/startup_trace.py                         # to the title screen
+    python3 tools/startup_trace.py --world T --seed 1234   # into a world, until the view is full
+    python3 tools/startup_trace.py --server 127.0.0.1:25565  # a server's join
+    python3 tools/startup_trace.py --runs 3 --warm         # 3 later starts (caches built by a first one)
+
+Every run gets fresh scratch folders (HORIZON_HOME, launcher and XDG dirs, the
+working directory where saves go); `--warm` keeps one scratch for all runs and
+starts once first, so the runs measured are later starts. `--keep FILE` writes
+the trace wrapped as a JSON array: Perfetto (ui.perfetto.dev) opens it.
+"""
+import argparse
+import json
+import pathlib
+import shutil
+import signal
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+
+try:
+    from . import lab
+except ImportError:
+    import lab
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+# What each `--until` waits for.
+UNTIL = {"title": "title.shown", "world": "world.view_full", "frame": "world.first_frame"}
+
+
+def read_trace(path):
+    """The trace's events: one JSON object per line, a torn last line skipped."""
+    events = []
+    try:
+        text = pathlib.Path(path).read_text()
+    except FileNotFoundError:
+        return events
+    for line in text.splitlines():
+        try:
+            events.append(json.loads(line))
+        except json.JSONDecodeError:
+            pass
+    return events
+
+
+def summarize(events):
+    """(steps, marks): steps as (start ms, end ms, name, thread) sorted by
+    start, marks as {name: first ms}."""
+    threads = {e["tid"]: e["args"]["name"] for e in events if e.get("ph") == "M"}
+    steps, marks = [], {}
+    for e in events:
+        if e.get("ph") == "X":
+            start = e["ts"] / 1000
+            steps.append((start, start + e["dur"] / 1000, e["name"], threads.get(e["tid"], "?")))
+        elif e.get("ph") == "i":
+            marks.setdefault(e["name"], e["ts"] / 1000)
+    steps.sort()
+    return steps, marks
+
+
+def scratch_env(home):
+    """lab.game_env with every folder in `home`, so a later start (`--warm`) finds what the first one built."""
+    for d in ("home", "launcher", "data", "config", "trash"):
+        (home / d).mkdir(parents=True, exist_ok=True)
+    return lab.game_env(
+        HORIZON_HOME=home / "home",
+        HORIZON_LAUNCHER_HOME=home / "launcher",
+        XDG_DATA_HOME=home / "data",
+        XDG_CONFIG_HOME=home / "config",
+        HORIZON_LAUNCHER_TRASH=home / "trash",
+        HORIZON_TELEMETRY_URL="off",
+    )
+
+
+def run_once(args, home, label):
+    trace = home / f"trace-{label}.jsonl"
+    env = scratch_env(home)
+    env["HORIZON_STARTUP_TRACE"] = str(trace)
+    cmd = [args.bin, "--resources", args.resources]
+    if args.world:
+        cmd += ["--world", args.world, "--seed", str(args.seed), "--creative",
+                "--render-distance", str(args.render_distance)]
+    elif args.server:
+        cmd += ["--server", args.server, "--render-distance", str(args.render_distance)]
+    cmd += args.extra
+    want = UNTIL[args.until]
+    log = open(home / f"game-{label}.log", "w")
+    proc = subprocess.Popen(cmd, cwd=home, env=env, stdout=log, stderr=subprocess.STDOUT)
+    deadline = time.monotonic() + args.timeout
+    marks = {}
+    while time.monotonic() < deadline and proc.poll() is None:
+        _, marks = summarize(read_trace(trace))
+        if want in marks:
+            time.sleep(args.linger)
+            break
+        time.sleep(0.05)
+    if proc.poll() is None:
+        proc.send_signal(signal.SIGTERM)
+        try:
+            proc.wait(15)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+    log.close()
+    steps, marks = summarize(read_trace(trace))
+    if want not in marks:
+        print(f"{label}: no {want} within {args.timeout} s (log: {home / f'game-{label}.log'})",
+              file=sys.stderr)
+    return steps, marks, trace
+
+
+def print_timeline(steps, marks, floor_ms):
+    rows = [(s, e, f"{name}  [{thread}]") for s, e, name, thread in steps if e - s >= floor_ms]
+    rows += [(t, None, f"* {name}") for name, t in marks.items()]
+    rows.sort(key=lambda r: r[0])
+    for start, end, what in rows:
+        if end is None:
+            print(f"{start:8.0f}            {what}")
+        else:
+            print(f"{start:8.0f} {end - start:8.0f} ms  {what}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--bin", default=str(ROOT / "target/release/horizon-client"))
+    ap.add_argument("--resources", default=str(ROOT / ".local/horizon-client/26.3"))
+    ap.add_argument("--world", help="open (or create) this world from the command line")
+    ap.add_argument("--seed", type=int, default=1234)
+    ap.add_argument("--server", help="join this server from the command line")
+    ap.add_argument("--render-distance", type=int, default=12)
+    ap.add_argument("--until", choices=sorted(UNTIL), help="default: world with --world/--server, else title")
+    ap.add_argument("--runs", type=int, default=1)
+    ap.add_argument("--warm", action="store_true", help="start once first in the same scratch: later starts")
+    ap.add_argument("--timeout", type=float, default=180)
+    ap.add_argument("--linger", type=float, default=0.3, help="seconds to keep running after the mark")
+    ap.add_argument("--floor", type=float, default=5, help="hide steps shorter than this (ms)")
+    ap.add_argument("--keep", help="write the last run's trace here as a JSON array (Perfetto)")
+    ap.add_argument("--json", action="store_true", help="print the marks of every run as JSON")
+    ap.add_argument("extra", nargs="*", help="more game arguments (after --)")
+    args = ap.parse_args()
+    args.until = args.until or ("world" if args.world or args.server else "title")
+    # The game runs in its scratch folder.
+    args.bin, args.resources = (str(pathlib.Path(p).resolve()) for p in (args.bin, args.resources))
+
+    shared = pathlib.Path(tempfile.mkdtemp(prefix="horizon-startup-")) if args.warm else None
+    if shared:
+        run_once(args, shared, "warmup")
+    results = []
+    for i in range(args.runs):
+        home = shared or pathlib.Path(tempfile.mkdtemp(prefix="horizon-startup-"))
+        steps, marks, trace = run_once(args, home, f"run{i + 1}")
+        results.append(marks)
+        print(f"== run {i + 1} ({'later start' if shared else 'first start'}, {home})")
+        print_timeline(steps, marks, args.floor)
+        if args.keep:
+            pathlib.Path(args.keep).write_text(json.dumps(read_trace(trace)))
+        if not shared:
+            shutil.rmtree(home, ignore_errors=True)
+    if shared:
+        shutil.rmtree(shared, ignore_errors=True)
+    names = sorted({n for m in results for n in m}, key=lambda n: statistics.median(
+        [m[n] for m in results if n in m]))
+    print("== medians (ms since start)")
+    for name in names:
+        values = [m[name] for m in results if name in m]
+        print(f"{statistics.median(values):8.0f}  {name}  (n={len(values)})")
+    if args.json:
+        print(json.dumps(results))
+
+
+if __name__ == "__main__":
+    main()