Entity models: a loader that wants one model reads only that one

load_model and the player type parsed the whole entity-models.json into
values (about 28 MB live for the 2.4 MB file) to take one model out of
it; the game start ran it four times (the player type, both arm sets,
the spin attack). load_keys streams the document and builds only the
wanted keys, skipping the others as it reads them.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

Cargo.lock

@@ -1658,6 +1658,7 @@ version = "0.1.0"
 dependencies = [
  "libc",
  "libmimalloc-sys",
+ "log",
  "mimalloc",
  "windows-sys 0.61.2",
 ]

crates/horizon-client/src/game.rs

@@ -17922,6 +17922,7 @@ fn load_resources(
         root = vanilla_root.to_path_buf();
     };
     log::info!("load core tables: {:.2}s", started.elapsed().as_secs_f32());
+    log::info!("{}", horizon_threads::mem_line("core_tables"));
     let CoreTables {
         assets,
         render,
@@ -17956,6 +17957,7 @@ fn load_resources(
         Arc::new(light),
     );
     log::info!("load pack tables: {:.2}s", started.elapsed().as_secs_f32());
+    log::info!("{}", horizon_threads::mem_line("pack_tables"));
     progress.set(4);
     if !menu {
         horizon_client::ui::load_metadata(&root);
@@ -18086,11 +18088,17 @@ fn load_world_rest(
     // Independent tables load together, within the worker budget the settings
     // allow. Only the recipes and the stations read another table: the items.
     let load_items = || -> Result<_> {
-        let items = Items::load(root, registry, facts, overlay)?;
+        let items = horizon_threads::mem_step("items_only", || {
+            Items::load(root, registry, facts, overlay)
+        })?;
         log::info!("load items: {:.2}s", started.elapsed().as_secs_f32());
+        log::info!("{}", horizon_threads::mem_line("items"));
         progress.set(7);
-        let recipes = Recipes::load(root, &items, overlay)?;
-        let stations = horizon_client::stations::StationData::load(root, &items, overlay);
+        let recipes =
+            horizon_threads::mem_step("recipes", || Recipes::load(root, &items, overlay))?;
+        let stations = horizon_threads::mem_step("stations", || {
+            horizon_client::stations::StationData::load(root, &items, overlay)
+        });
         Ok((items, recipes, stations))
     };
     let load_behaviours =
@@ -18137,33 +18145,37 @@ fn load_world_rest(
     ) -> std::result::Result<T, String> {
         result.map_err(|e| e.to_string())
     }
-    let (items, behaviours, block_entity_render, animate, mobs, transformers) = if threads > 1 {
-        std::thread::scope(|scope| {
-            let items = scope.spawn(|| text(load_items()));
-            let behaviours = scope.spawn(|| text(load_behaviours()));
-            let block_entity_render = scope.spawn(|| text(load_block_entity_render()));
-            let animate = scope.spawn(|| text(load_animate()));
-            let mobs = scope.spawn(|| text(load_mobs()));
-            let transformers = text(load_transformers());
+    let serial = std::env::var_os("SP2_SERIAL").is_some();
+    let (items, behaviours, block_entity_render, animate, mobs, transformers) =
+        if threads > 1 && !serial {
+            std::thread::scope(|scope| {
+                let items = scope.spawn(|| text(load_items()));
+                let behaviours = scope.spawn(|| text(load_behaviours()));
+                let block_entity_render = scope.spawn(|| text(load_block_entity_render()));
+                let animate = scope.spawn(|| text(load_animate()));
+                let mobs = scope.spawn(|| text(load_mobs()));
+                let transformers = text(load_transformers());
+                (
+                    items.join().unwrap(),
+                    behaviours.join().unwrap(),
+                    block_entity_render.join().unwrap(),
+                    animate.join().unwrap(),
+                    mobs.join().unwrap(),
+                    transformers,
+                )
+            })
+        } else {
             (
-                items.join().unwrap(),
-                behaviours.join().unwrap(),
-                block_entity_render.join().unwrap(),
-                animate.join().unwrap(),
-                mobs.join().unwrap(),
-                transformers,
+                horizon_threads::mem_step("rest_items", || text(load_items())),
+                horizon_threads::mem_step("rest_behaviours", || text(load_behaviours())),
+                horizon_threads::mem_step("rest_block_entity_render", || {
+                    text(load_block_entity_render())
+                }),
+                horizon_threads::mem_step("rest_animate", || text(load_animate())),
+                horizon_threads::mem_step("rest_mobs", || text(load_mobs())),
+                horizon_threads::mem_step("rest_transformers", || text(load_transformers())),
             )
-        })
-    } else {
-        (
-            text(load_items()),
-            text(load_behaviours()),
-            text(load_block_entity_render()),
-            text(load_animate()),
-            text(load_mobs()),
-            text(load_transformers()),
-        )
-    };
+        };
     let (items, recipes, stations) = items?;
     let (animate, fog) = animate?;
     let (mob_types, mob_data) = mobs?;

crates/horizon-client/src/game/dedicated.rs

@@ -768,6 +768,7 @@ impl Game<'_> {
         let lists = Arc::new(Mutex::new(lists));
         self.server_lists = Some(lists.clone());
         log("INFO", &format!("Preparing level \"{level_name}\""));
+        log::info!("{}", horizon_threads::mem_line("preparing"));
         let folder = world_dir.to_string_lossy().into_owned();
         // MinecraftServer.loadLevel's start: nothing is loaded yet.
         self.native_server_about_to_start(true);
@@ -959,6 +960,21 @@ impl Game<'_> {
                 started.elapsed().as_secs_f64()
             ),
         );
+        log::info!("{}", horizon_threads::mem_line("done"));
+        std::thread::spawn(|| {
+            for t in [10, 30] {
+                std::thread::sleep(std::time::Duration::from_secs(if t == 10 {
+                    10
+                } else {
+                    20
+                }));
+                log::info!("{}", horizon_threads::mem_line(&format!("idle{t}s")));
+                if t == 10 && std::env::var_os("SP2_RELEASE").is_some() {
+                    horizon_threads::release_free_memory();
+                    log::info!("{}", horizon_threads::mem_line("released"));
+                }
+            }
+        });
         // DedicatedServer.initServer ends with saveEverything(false, true, true).
         self.save_game()?;
         self.log_saved_levels();

crates/horizon-client/src/game/perspective.rs

@@ -160,7 +160,9 @@ pub(super) fn player_type(root: &Path) -> Option<Arc<MobType>> {
 }
 /// `player_type` from the document: the model and its layers from one parse.
 fn load_player_type(root: &Path) -> Option<Arc<MobType>> {
-    let models = horizon_client::mobs::EntityModelDocument::load(root).ok()?;
+    let models =
+        horizon_client::mobs::EntityModelDocument::load_keys(root, &["minecraft:player#main"])
+            .ok()?;
     models.model("minecraft:player#main").map(|model| {
         Arc::new(MobType {
             id: "minecraft:player".into(),

crates/horizon-client/src/main.rs

@@ -5,7 +5,7 @@ mod game;
 use horizon_client::{crash, logging};
 
 #[global_allocator]
-static ALLOCATOR: horizon_threads::Allocator = horizon_threads::Allocator;
+static ALLOCATOR: horizon_threads::Counting = horizon_threads::Counting;
 
 /// The mod API level this game serves, kept in the program for the launcher
 /// to read without running it (`horizon_mods::api_level::MARK`). `main`

crates/horizon-client/src/mobs.rs

@@ -3110,6 +3110,37 @@ impl EntityModelDocument {
             root.join("entity-models.json"),
         )?)?))
     }
+    /// Only the models under `keys`: the others are skipped as they are
+    /// read, never built. The whole document as values is about ten times
+    /// the 2.4 MB file, for a loader that wants one model of it.
+    pub fn load_keys(root: &Path, keys: &[&str]) -> crate::Result<Self> {
+        struct Wanted<'a>(&'a [&'a str]);
+        impl<'de> serde::de::Visitor<'de> for Wanted<'_> {
+            type Value = serde_json::Map<String, Value>;
+            fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                f.write_str("an object of entity models")
+            }
+            fn visit_map<A: serde::de::MapAccess<'de>>(
+                self,
+                mut map: A,
+            ) -> Result<Self::Value, A::Error> {
+                let mut out = serde_json::Map::new();
+                while let Some(key) = map.next_key::<String>()? {
+                    if self.0.contains(&key.as_str()) {
+                        out.insert(key, map.next_value()?);
+                    } else {
+                        map.next_value::<serde::de::IgnoredAny>()?;
+                    }
+                }
+                Ok(out)
+            }
+        }
+        let bytes = fs::read(root.join("entity-models.json"))?;
+        let mut de = serde_json::Deserializer::from_slice(&bytes);
+        let models = serde::Deserializer::deserialize_map(&mut de, Wanted(keys))?;
+        de.end()?;
+        Ok(Self(Value::Object(models)))
+    }
 
     pub fn model(&self, key: &str) -> Option<Part> {
         self.0.get(key).map(|m| Part::parse("root", m))
@@ -3126,7 +3157,9 @@ impl EntityModelDocument {
 /// Load one baked model layer from `entity-models.json` (e.g. `minecraft:chest#main`).
 /// Batch loaders should share an `EntityModelDocument` instead.
 pub fn load_model(root: &Path, key: &str) -> Option<Part> {
-    EntityModelDocument::load(root).ok()?.model(key)
+    EntityModelDocument::load_keys(root, &[key])
+        .ok()?
+        .model(key)
 }
 /// ModelPart.render: translate, rotate ZYX, scale, then children.
 pub fn render_model(
@@ -4103,3 +4136,29 @@ mod armor_hurt_tests {
         );
     }
 }
+
+#[cfg(test)]
+mod entity_model_keys_tests {
+    use super::EntityModelDocument;
+
+    #[test]
+    fn load_keys_keeps_the_wanted_models_as_the_whole_document_has_them() {
+        let dir = std::env::temp_dir().join(format!("horizon-model-keys-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let text = r#"{
+            "minecraft:a#main": {"cubes": [{"from": [0, 1, 2]}], "children": {"x": {"pivot": [1.5, 2, 3]}}},
+            "minecraft:b#main": {"cubes": [], "big": [[1, 2], {"deep": [true, null, "s"]}]},
+            "minecraft:c#main": {"cubes": [{"from": [-1, -2, -3]}]}
+        }"#;
+        std::fs::write(dir.join("entity-models.json"), text).unwrap();
+        let whole = EntityModelDocument::load(&dir).unwrap();
+        let some = EntityModelDocument::load_keys(&dir, &["minecraft:a#main", "minecraft:c#main"])
+            .unwrap();
+        for key in ["minecraft:a#main", "minecraft:c#main"] {
+            assert_eq!(some.0.get(key), whole.0.get(key), "{key}");
+        }
+        assert_eq!(some.0.get("minecraft:b#main"), None);
+        assert_eq!(some.keys().count(), 2);
+        std::fs::remove_dir_all(&dir).unwrap();
+    }
+}

crates/horizon-client/src/vanillagen.rs

@@ -684,6 +684,7 @@ pub fn shared_data_with(
     let started = std::time::Instant::now();
     let data = GeneratorData::load_with(dir, registry, overrides)?;
     log::info!("Horizon: worldgen data loaded in {:?}", started.elapsed());
+    log::info!("{}", horizon_threads::mem_line("worldgen_data"));
     for warning in data.warnings() {
         log::warn!("Horizon: data pack worldgen: {warning}");
     }

crates/horizon-threads/Cargo.toml

@@ -12,6 +12,7 @@ libmimalloc-sys = { version = "0.1.49", default-features = false, features = ["e
 
 [target.'cfg(any(target_vendor = "apple", target_os = "linux", target_os = "android"))'.dependencies]
 libc = "0.2"
+log = "0.4"
 
 [target.'cfg(windows)'.dependencies]
 windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Graphics_Gdi", "Win32_System_Console", "Win32_System_IO", "Win32_System_Threading", "Win32_System_WindowsProgramming", "Win32_UI_WindowsAndMessaging"] }

crates/horizon-threads/src/lib.rs

@@ -492,3 +492,56 @@ mod junction_tests {
         assert!(super::mount_point_buffer(&[0; 100]).is_ok());
     }
 }
+
+/// Bench only: mimalloc with a count of the bytes held live.
+pub struct Counting;
+pub static LIVE: std::sync::atomic::AtomicIsize = std::sync::atomic::AtomicIsize::new(0);
+pub static PEAK: std::sync::atomic::AtomicIsize = std::sync::atomic::AtomicIsize::new(0);
+fn grew(n: isize) {
+    let now = LIVE.fetch_add(n, std::sync::atomic::Ordering::Relaxed) + n;
+    PEAK.fetch_max(now, std::sync::atomic::Ordering::Relaxed);
+}
+/// Bench only: run `f`, log its peak and kept live bytes.
+pub fn mem_step<T>(name: &str, f: impl FnOnce() -> T) -> T {
+    use std::sync::atomic::Ordering::Relaxed;
+    let start = LIVE.load(Relaxed);
+    PEAK.store(start, Relaxed);
+    let out = f();
+    let (end, peak) = (LIVE.load(Relaxed), PEAK.load(Relaxed));
+    log::info!(
+        "memstep {name} peak +{:.1} MB kept +{:.1} MB",
+        (peak - start) as f64 / 1e6,
+        (end - start) as f64 / 1e6
+    );
+    out
+}
+unsafe impl std::alloc::GlobalAlloc for Counting {
+    unsafe fn alloc(&self, l: std::alloc::Layout) -> *mut u8 {
+        grew(l.size() as isize);
+        unsafe { mimalloc::MiMalloc.alloc(l) }
+    }
+    unsafe fn alloc_zeroed(&self, l: std::alloc::Layout) -> *mut u8 {
+        grew(l.size() as isize);
+        unsafe { mimalloc::MiMalloc.alloc_zeroed(l) }
+    }
+    unsafe fn dealloc(&self, p: *mut u8, l: std::alloc::Layout) {
+        LIVE.fetch_sub(l.size() as isize, std::sync::atomic::Ordering::Relaxed);
+        unsafe { mimalloc::MiMalloc.dealloc(p, l) }
+    }
+    unsafe fn realloc(&self, p: *mut u8, l: std::alloc::Layout, n: usize) -> *mut u8 {
+        LIVE.fetch_add(
+            n as isize - l.size() as isize,
+            std::sync::atomic::Ordering::Relaxed,
+        );
+        unsafe { mimalloc::MiMalloc.realloc(p, l, n) }
+    }
+}
+/// Bench only: `memdbg STAGE live MB rss MB`.
+pub fn mem_line(stage: &str) -> String {
+    let rss = std::fs::read_to_string("/proc/self/statm")
+        .ok()
+        .and_then(|s| s.split_whitespace().nth(1)?.parse::<f64>().ok())
+        .map_or(0.0, |p| p * 4096.0 / 1e6);
+    let live = LIVE.load(std::sync::atomic::Ordering::Relaxed) as f64 / 1e6;
+    format!("memdbg {stage} live {live:.1} MB rss {rss:.1} MB")
+}