Merge remote-tracking branch 'origin/main' into perf/mesh-view-first

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/horizon-client-render/src/gfx.rs

@@ -8926,6 +8926,13 @@ pub struct Frustum {
     planes: [Vec4; 6],
 }
 impl Frustum {
+    /// A view's frustum, without bobbing (`Renderer::camera_frustum`'s with
+    /// no renderer to take the aspect from).
+    pub fn of_view(view: &View) -> Self {
+        Self {
+            planes: frustum(view.view_proj(Mat4::IDENTITY)),
+        }
+    }
     /// Whether the camera-relative box `min..max` is at least partly inside.
     pub fn intersects(&self, min: Vec3, max: Vec3) -> bool {
         !self.planes.iter().any(|p| {

crates/horizon-client-render/src/streaming.rs

@@ -244,6 +244,41 @@ pub enum Done {
     /// A proto chunk back from `Job::LightView`, with its view.
     LightView(i32, i32, Proto, Option<Chunk>),
 }
+/// What a mesh job for a section the camera doesn't see adds to its
+/// priority: such sections wait for every one it sees (their distances
+/// stay far below it).
+const OUT_OF_VIEW: i64 = 1 << 32;
+/// The camera's view frustum (camera-relative, `Renderer::camera_frustum`)
+/// and where the camera is.
+#[derive(Clone, Copy, Debug)]
+pub struct CameraView {
+    pub frustum: crate::gfx::Frustum,
+    pub eye: [f64; 3],
+}
+impl CameraView {
+    /// Whether the block box `min..max` is at least partly in view.
+    fn sees(&self, min: [i32; 3], max: [i32; 3]) -> bool {
+        let rel = |p: [i32; 3]| {
+            glam::Vec3::new(
+                (f64::from(p[0]) - self.eye[0]) as f32,
+                (f64::from(p[1]) - self.eye[1]) as f32,
+                (f64::from(p[2]) - self.eye[2]) as f32,
+            )
+        };
+        self.frustum.intersects(rel(min), rel(max))
+    }
+    /// Whether the block box `min..max` is entirely in view.
+    fn holds(&self, min: [i32; 3], max: [i32; 3]) -> bool {
+        let rel = |p: [i32; 3]| {
+            glam::Vec3::new(
+                (f64::from(p[0]) - self.eye[0]) as f32,
+                (f64::from(p[1]) - self.eye[1]) as f32,
+                (f64::from(p[2]) - self.eye[2]) as f32,
+            )
+        };
+        self.frustum.contains(rel(min), rel(max))
+    }
+}
 struct Queued {
     priority: i64,
     /// The column the job is for and what it adds to the column's distance
@@ -416,6 +451,14 @@ pub struct Streaming {
     deferred: Vec<Done>,
     sequence: u64,
     pub center: (i32, i32),
+    /// The camera's view: the sections in it mesh before the rest, as vanilla
+    /// compiles only the sections it sees (`LevelExtractor.extract` walks
+    /// `LevelRenderer.visibleSections`). None (no camera yet): nearest first.
+    pub view: Option<CameraView>,
+    /// Sections in `view` that the last `update` left ready to mesh, being
+    /// meshed, or meshed and not yet handed over: vanilla's compile queue
+    /// (`SectionRenderDispatcher.isQueueEmpty`).
+    view_pending: usize,
     /// The view distance around `center`; below 0, no area there (a dedicated
     /// server nobody is on).
     pub radius: i32,
@@ -917,6 +960,8 @@ impl Streaming {
             deferred: Vec::new(),
             sequence: 0,
             center: (0, 0),
+            view: None,
+            view_pending: 0,
             radius: 8,
             forced: HashSet::new(),
             paused: false,
@@ -1074,6 +1119,32 @@ impl Streaming {
     /// Queue a job for the column `at`, nearest first: its distance (`distance`)
     /// times four plus `bias`, which orders a column's jobs. A job without a
     /// column (a save, meshes to free) goes ahead of every other.
+    /// `submit` for several jobs, under one lock.
+    fn submit_all(&mut self, jobs: Vec<(Option<(i32, i32)>, i64, Job)>) {
+        if jobs.is_empty() {
+            return;
+        }
+        let queued: Vec<Queued> = jobs
+            .into_iter()
+            .map(|(at, bias, job)| {
+                self.order += 1;
+                Queued {
+                    priority: at.map_or(i64::MIN, |(x, z)| self.distance(x, z) * 4 + bias),
+                    at,
+                    bias,
+                    order: self.order,
+                    job,
+                }
+            })
+            .collect();
+        let many = queued.len() > 1;
+        self.shared.queue.lock().unwrap().extend(queued);
+        if many {
+            self.jobs.notify_all();
+        } else {
+            self.jobs.notify();
+        }
+    }
     fn submit(&mut self, at: Option<(i32, i32)>, bias: i64, job: Job) {
         self.order += 1;
         let priority = at.map_or(i64::MIN, |(x, z)| self.distance(x, z) * 4 + bias);
@@ -1236,6 +1307,47 @@ impl Streaming {
     fn distance(&self, x: i32, z: i32) -> i64 {
         self.ranges().priority(x, z, self.loading)
     }
+    /// `sections` of column (x, z) split into those the camera sees and the
+    /// rest (all seen without a view). The column is looked at whole first:
+    /// most are either out of view or entirely in it.
+    fn split_seen(
+        &self,
+        world: &World,
+        x: i32,
+        z: i32,
+        sections: SectionMask,
+    ) -> (SectionMask, SectionMask) {
+        let Some(view) = &self.view else {
+            return (sections, SectionMask::default());
+        };
+        let layout = world.layout();
+        let (min, max) = (
+            [x * 16, layout.min_y(), z * 16],
+            [x * 16 + 16, layout.max_y() + 1, z * 16 + 16],
+        );
+        if !view.sees(min, max) {
+            return (SectionMask::default(), sections);
+        }
+        if view.holds(min, max) {
+            return (sections, SectionMask::default());
+        }
+        let (mut seen, mut unseen) = (SectionMask::default(), SectionMask::default());
+        for sy in sections.iter() {
+            let y = layout.section_y(sy);
+            if view.sees([x * 16, y, z * 16], [x * 16 + 16, y + 16, z * 16 + 16]) {
+                seen.insert(sy);
+            } else {
+                unseen.insert(sy);
+            }
+        }
+        (seen, unseen)
+    }
+    /// Whether the camera sees any of section (x, sy, z) (true without a view).
+    fn section_seen(&self, world: &World, x: i32, sy: i32, z: i32) -> bool {
+        let y = world.layout().section_y(sy as usize);
+        self.view
+            .is_none_or(|view| view.sees([x * 16, y, z * 16], [x * 16 + 16, y + 16, z * 16 + 16]))
+    }
     /// The 3x3 a light job reads: the finished columns, and for a neighbour
     /// that will not finish (past the area), its decorated proto chunk, as
     /// vanilla lights its last FULL ring against neighbours at FEATURES.
@@ -1385,7 +1497,7 @@ impl Streaming {
     }
     /// How much work each stage holds: queued jobs, then what is in flight or
     /// waiting at each step from generation to meshing.
-    pub fn stages(&self) -> [(&'static str, usize); 7] {
+    pub fn stages(&self) -> [(&'static str, usize); 8] {
         [
             ("queued", self.pending()),
             ("generating", self.generating.len()),
@@ -1397,6 +1509,7 @@ impl Streaming {
             ("finishing", self.finishing.len()),
             ("lighting", self.lighting.len()),
             ("meshing", self.meshing.len()),
+            ("meshing_in_view", self.view_pending),
         ]
     }
     /// Advance streaming: collect finished work, then schedule more.
@@ -1805,6 +1918,7 @@ impl Streaming {
         // Column by column: the checks below hold for all of a column's
         // sections, and columns waiting for light cost one look per frame.
         let mut candidates: Vec<(i64, i32, i32, SectionMask)> = Vec::new();
+        let mut view_pending = 0;
         let columns: Vec<_> = world.dirty_sections.columns().collect();
         for ((x, z), sections) in columns {
             if !world.chunks.contains_key(&(x, z)) {
@@ -1833,9 +1947,20 @@ impl Streaming {
             if !ready {
                 continue;
             }
-            candidates.push((self.distance(x, z), x, z, sections));
+            // The sections the camera sees go first, as vanilla compiles only
+            // those; the rest of the column waits for them (dirty until then).
+            let (seen, unseen) = self.split_seen(world, x, z, sections);
+            let rank = self.distance(x, z) * 4;
+            view_pending += seen.len();
+            if !seen.is_empty() {
+                candidates.push((rank, x, z, seen));
+            }
+            if !unseen.is_empty() {
+                candidates.push((rank + OUT_OF_VIEW, x, z, unseen));
+            }
         }
-        // Nearest first, and only as many sections as the workers can chew.
+        // What the camera sees first, then nearest first, and only as many
+        // sections as the workers can chew.
         // Every ready column used to be submitted at once, so at a wide
         // render distance the queue — and `meshing`, and each job's 3x3
         // region snapshot — grew with the number of loaded columns and the
@@ -1844,18 +1969,26 @@ impl Streaming {
         // capacity frees; `requeue` then has a small queue to keep close.
         let room = (self.workers * 32).saturating_sub(self.meshing.len());
         nearest_first(&mut candidates, room);
-        for (_, x, z, sections) in candidates {
+        // Queued together, so a worker takes them in priority order rather
+        // than in the order they were made.
+        let mut jobs = Vec::new();
+        for (rank, x, z, sections) in candidates {
             let Some(region) = Self::region(world, x, z) else {
                 continue;
             };
-            world.dirty_sections.take_column(x, z);
+            world.dirty_sections.take(x, z, sections);
+            let behind = rank >= OUT_OF_VIEW;
+            if !behind {
+                // Counted above as ready; counted below as meshing.
+                view_pending -= sections.len();
+            }
             for sy in sections.iter().map(|sy| sy as i32) {
                 let first = self.meshed.insert((x, z));
                 self.sequence += 1;
                 self.meshing.insert((x, sy, z), self.sequence);
-                self.submit(
+                jobs.push((
                     Some((x, z)),
-                    if first { 2 } else { 0 },
+                    if first { 2 } else { 0 } + if behind { OUT_OF_VIEW } else { 0 },
                     Job::Mesh(
                         x,
                         sy,
@@ -1873,9 +2006,19 @@ impl Streaming {
                             leaf_culling: self.leaf_culling,
                         },
                     ),
-                );
+                ));
             }
         }
+        self.submit_all(jobs);
+        view_pending += match self.view {
+            Some(_) => self
+                .meshing
+                .keys()
+                .filter(|&&(x, sy, z)| self.section_seen(world, x, sy, z))
+                .count(),
+            None => self.meshing.len(),
+        };
+        self.view_pending = view_pending;
         (uploads, removed)
     }
     fn schedule_generation(&mut self, world: &World, workers: usize) {
@@ -2654,6 +2797,115 @@ fn same_values(
 mod tests {
     use super::*;
 
+    /// A remote client's columns all arrive at once around a camera looking
+    /// east: every section it sees meshes before any it doesn't see, and the
+    /// count of sections in view still to mesh falls to 0 with them.
+    #[test]
+    fn sections_in_the_camera_view_mesh_before_those_behind_it() {
+        let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("../../.local/horizon-client")
+            .join(horizon_core::VANILLA_VERSION);
+        if !root.join("items.json").exists() {
+            return;
+        }
+        let assets = Arc::new(crate::assets::Assets::load(&root).unwrap());
+        let render = Arc::new(assets.reload(&root).unwrap());
+        let facts = crate::blocks::BlockFacts::load(&root, assets.registry).unwrap();
+        let visibility = crate::models::load_visibility(&root, assets.registry).unwrap();
+        let biomes = Arc::new(crate::biome::Biomes::load(&root).unwrap());
+        let context = Arc::new(WorkerContext {
+            tables: Arc::new(
+                crate::mesher::MeshTables::new(&assets, &render, &facts, visibility).unwrap(),
+            ),
+            light: Arc::new(
+                crate::light::LightTables::load(&root, assets.registry, &facts).unwrap(),
+            ),
+            biomes: biomes.clone(),
+            assets: assets.clone(),
+            render,
+        });
+        let terrain = crate::testgen::TestTerrain::new(assets.registry, &biomes).unwrap();
+        let stone = assets.registry.default_state("minecraft:stone").unwrap().0;
+        let mut world = World::new(horizon_world::registry::BlockStateId(0), 0);
+        let layout = world.layout();
+        // One worker: sections mesh in the order the queue gives them.
+        let mut streaming = Streaming::new(Arc::new(terrain), context, 1);
+        streaming.remote = true;
+        // Far enough that every column is in the view distance.
+        streaming.radius = 12;
+        let view = CameraView {
+            frustum: crate::gfx::Frustum::of_view(&crate::gfx::View::screen(
+                -90.0,
+                0.0,
+                70.0,
+                16.0 / 9.0,
+                512.0,
+            )),
+            eye: [8.0, 72.0, 8.0],
+        };
+        streaming.view = Some(view);
+        for x in -7..=7 {
+            for z in -7..=7 {
+                let mut chunk = Chunk::with_layout(x, z, layout);
+                chunk.set(3, 70, 5, stone);
+                chunk.lit = true;
+                streaming.insert_remote(&mut world, chunk);
+            }
+        }
+        let column_seen = |&(x, _, z): &(i32, i32, i32)| {
+            view.sees(
+                [x * 16, layout.min_y(), z * 16],
+                [x * 16 + 16, layout.max_y() + 1, z * 16 + 16],
+            )
+        };
+        let section_seen = |&(x, sy, z): &(i32, i32, i32)| {
+            let y = layout.section_y(sy as usize);
+            view.sees([x * 16, y, z * 16], [x * 16 + 16, y + 16, z * 16 + 16])
+        };
+        // The columns with all their neighbours, 13 x 13, each section meshed once.
+        let all = 169 * layout.sections();
+        let mut order = Vec::new();
+        let mut pending_in_view = Vec::new();
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);
+        loop {
+            assert!(std::time::Instant::now() < deadline, "meshing never ended");
+            let (uploads, _) = streaming.update(&mut world);
+            let stages: FxHashMap<_, _> = streaming.stages().into_iter().collect();
+            pending_in_view.push((stages["meshing_in_view"], stages["meshing"]));
+            order.extend(uploads.iter().map(|(key, _)| *key));
+            streaming.free(uploads);
+            // The edge columns stay dirty: their neighbours never come (`ready`).
+            if order.len() >= all && !streaming.busy() {
+                break;
+            }
+            std::thread::sleep(std::time::Duration::from_millis(1));
+        }
+        assert_eq!(order.len(), all);
+        // No section the camera sees waits for one it doesn't see, in its own
+        // column (above or below the view) or another: once a worker took one
+        // out of view, the sections in view still to finish are those the
+        // pool's other workers held then (`order` is the order they finished).
+        let behind = order
+            .iter()
+            .position(|k| !section_seen(k))
+            .expect("sections behind");
+        assert!(behind > 0, "sections in view");
+        let late: Vec<_> = order[behind..].iter().filter(|k| section_seen(k)).collect();
+        assert!(
+            late.len() < horizon_threads::jobs::pool().threads(),
+            "sections in view meshed after {:?} (#{behind}), out of view: {late:?}",
+            order[behind]
+        );
+        assert!(order.iter().any(|k| !column_seen(k)), "columns behind");
+        // Once the sections in view are done the count is 0 while others still mesh.
+        assert!(
+            pending_in_view
+                .iter()
+                .any(|&(in_view, all)| in_view == 0 && all > 0)
+        );
+        assert_eq!(pending_in_view.last(), Some(&(0, 0)));
+    }
+
     #[test]
     fn a_resent_column_keeps_its_sections_but_takes_new_biomes() {
         let mut old = Chunk::empty(2, -3);

crates/horizon-client-render/src/world.rs

@@ -32,6 +32,17 @@ impl DirtySections {
     pub fn take_column(&mut self, x: i32, z: i32) -> SectionMask {
         self.columns.remove(&(x, z)).unwrap_or_default()
     }
+    /// Section indexes `sections` of column (x, z), no longer dirty.
+    pub fn take(&mut self, x: i32, z: i32, sections: SectionMask) {
+        if let Some(mask) = self.columns.get_mut(&(x, z)) {
+            for sy in sections.iter() {
+                mask.remove(sy);
+            }
+            if mask.is_empty() {
+                self.columns.remove(&(x, z));
+            }
+        }
+    }
     /// Keep the columns `keep` accepts.
     pub fn retain_columns(&mut self, mut keep: impl FnMut(i32, i32) -> bool) {
         self.columns.retain(|&(x, z), _| keep(x, z));

crates/horizon-client/src/game.rs

@@ -16351,6 +16351,14 @@ impl Game<'_> {
                 (p.position.z.floor() as i32) >> 4,
             );
         }
+        // The sections the camera saw last frame mesh first, as vanilla compiles only those.
+        self.streaming.view =
+            self.view_frustum
+                .zip(self.locator_camera)
+                .map(|(frustum, camera)| horizon_client::streaming::CameraView {
+                    frustum,
+                    eye: camera.0,
+                });
         // The loading screen's chunk map fills ring by ring, as vanilla's.
         self.streaming.loading =
             self.options.execution_role.is_client() && self.loading_reason().is_some();

tests/test_perf_bench.py

@@ -194,6 +194,22 @@ class WaitLoadedTests(unittest.TestCase):
         tail = [self.status(901, 20000 + 20 * i, stages=40 if i % 40 == 0 else 0) for i in range(400)]
         self.assertAlmostEqual(self.run_script(loading + tail), 1.0, places=6)
 
+    def test_sections_behind_the_camera_do_not_hold_the_load(self):
+        # A build that counts the sections in view is loaded once those are meshed, though the
+        # ones behind the camera mesh for 2 s more: vanilla's driver waits only for what it sees.
+        def status(chunks, in_view, meshing):
+            s = self.status(chunks, 0, stages=meshing)
+            s["streaming"].update(queued=meshing, meshing_in_view=in_view)
+            return s
+        loading = [status(900, 300, 500) for _ in range(20)]
+        behind = [status(901, 0, 400) for _ in range(40)]
+        quiet = [status(901, 0, 0) for _ in range(40)]
+        self.assertAlmostEqual(self.run_script(loading + behind + quiet), 1.0, places=6)
+        # Without the count, the same game is loaded when every section is meshed.
+        for s in loading + behind + quiet:
+            del s["streaming"]["meshing_in_view"]
+        self.assertAlmostEqual(self.run_script(loading + behind + quiet), 2.95, places=6)
+
     def test_old_builds_wait_for_meshing_to_go_quiet(self):
         busy = [self.status(901, 1000 * i) for i in range(20)]
         quiet = [self.status(901, 20000 + i) for i in range(40)]

tools/perf_bench.py

@@ -444,7 +444,10 @@ def wait_loaded(control, near, limit, min_chunks=0):
     only remesh in short bursts. Older builds fall back to meshing going quiet
     (fewer than QUIET_MESHES_PER_S section meshes in a second), which also
     waits through that tail. (pending_chunks alone misses work the threads have
-    already taken, and sections waiting for their neighbours' light.)"""
+    already taken, and sections waiting for their neighbours' light.) Builds
+    that report `meshing_in_view` are drawn once the sections the camera sees
+    are, as vanilla's driver counts its compile queue (`isQueueEmpty`), which
+    only holds sections in view (`LevelExtractor.extract`)."""
     deadline = time.time() + limit
     last, changed, previous, meshing = None, time.time(), None, time.time()
     # Right after a teleport (and at spawn) Horizon reports nothing queued and steady counts for over a second
@@ -474,7 +477,11 @@ def wait_loaded(control, near, limit, min_chunks=0):
             sample = (now, s["meshed_sections"])
             if stages is not None:
                 # Protos outside the view distance wait for neighbours that are never generated.
-                if any(v for k, v in stages.items() if k != "protos"):
+                # Builds that count the sections in the camera's view still to mesh are drawn once
+                # those are (vanilla's driver waits for its compile queue, which holds only the
+                # sections it sees); the ones behind the camera mesh after them.
+                idle = {"protos", "queued", "meshing"} if "meshing_in_view" in stages else {"protos"}
+                if any(v for k, v in stages.items() if k not in idle):
                     meshing = now
                 quiet = 0.5
             else: