Merge pull request #3015 from DanielGodriNeto/perf/sprite-disk-cache

🤖 Agent: requeued by contrib-gate-28 after merging main (B9: sprite_cache.rs moved into horizon-client-render with sprites.rs); fmt, clippy -D warnings, horizon-client and horizon-client-render tests pass. Proven earlier by cg27 (sprite_cache_parity OK).

crates/horizon-client-render/src/assets.rs

@@ -196,7 +196,7 @@ impl Assets {
 impl RenderAssets {
     pub fn load(root: &Path, assets: &Assets) -> Result<Self> {
         let registry = assets.registry;
-        let mut sprites = crate::sprites::Sprites::default();
+        let mut sprites = crate::sprites::Sprites::with_disk_cache(root);
         for name in MATERIALS {
             sprites.insert(root, &format!("minecraft:block/{name}"))?;
         }
@@ -217,6 +217,7 @@ impl RenderAssets {
         for stage in 0..10 {
             sprites.insert(root, &format!("minecraft:block/destroy_stage_{stage}"))?;
         }
+        sprites.save_disk_cache();
         sprites.finish();
         // `finish` grows the buffer by doubling: keep only the bytes in use.
         sprites.pixels.shrink_to_fit();

crates/horizon-client-render/src/atlas_sources.rs

@@ -187,7 +187,14 @@ pub fn list_textures(root: &Path) -> Vec<String> {
             };
             for e in entries.flatten() {
                 let p = e.path();
-                if p.is_dir() {
+                // `file_type` comes with the listing; `Path::is_dir` is a stat per
+                // entry, kept for links (which it follows).
+                let dir = match e.file_type() {
+                    Ok(t) if t.is_symlink() => p.is_dir(),
+                    Ok(t) => t.is_dir(),
+                    Err(_) => p.is_dir(),
+                };
+                if dir {
                     stack.push(p);
                 } else if p.extension().is_some_and(|x| x == "png")
                     && let Ok(rel) = p.strip_prefix(&base)

crates/horizon-client-render/src/lib.rs

@@ -70,6 +70,7 @@ pub mod skins;
 pub mod sound_engine;
 pub mod sound_events;
 pub mod spans;
+mod sprite_cache;
 pub mod sprites;
 pub mod streaming;
 pub mod tab_list;

crates/horizon-client-render/src/sprite_cache.rs

@@ -0,0 +1,345 @@
+//! Decoded block sprites kept on disk between launches (M-11): reading and
+//! decoding ~1300 small PNGs is most of `Sprites::load`. One file per resource
+//! root, valid only while the `textures` trees still have the same paths,
+//! sizes and modification times and the format version is the same; anything
+//! else (missing, corrupt, stale, unwritable) means the normal decoding path.
+use crate::mipmap::Strategy;
+use std::{
+    collections::HashMap,
+    fs,
+    hash::{DefaultHasher, Hash, Hasher},
+    path::{Path, PathBuf},
+};
+
+const MAGIC: &[u8; 4] = b"HSPR";
+/// Bump when `Decoded` or the decoding in `Sprites::decode` changes.
+const VERSION: u32 = 1;
+const FILE: &str = "horizon-sprites.cache";
+
+/// What decoding one texture gives, before it takes a layer.
+#[derive(Clone)]
+pub(crate) struct Decoded {
+    pub side: u32,
+    pub frames: Vec<Vec<u8>>,
+    /// The `.mcmeta` `animation` value, as text.
+    pub animation: Option<String>,
+    pub strategy: Strategy,
+    pub alpha_cutoff_bias: f32,
+    pub has_transparent: bool,
+    pub opaque: bool,
+    pub translucent: bool,
+}
+
+pub(crate) struct SpriteCache {
+    path: PathBuf,
+    fingerprint: u64,
+    entries: HashMap<String, Decoded>,
+    dirty: bool,
+}
+
+/// Paths, sizes and modification times of every file under `assets/*/textures`,
+/// summed so the directory order does not matter.
+fn fingerprint(root: &Path) -> Option<u64> {
+    fn walk(dir: &Path, base: &Path, sum: &mut u64) -> Option<()> {
+        for entry in fs::read_dir(dir).ok()? {
+            let entry = entry.ok()?;
+            let meta = entry.metadata().ok()?;
+            if meta.is_dir() {
+                walk(&entry.path(), base, sum)?;
+                continue;
+            }
+            let mut hasher = DefaultHasher::new();
+            entry.path().strip_prefix(base).ok()?.hash(&mut hasher);
+            meta.len().hash(&mut hasher);
+            meta.modified().ok()?.hash(&mut hasher);
+            *sum = sum.wrapping_add(hasher.finish());
+        }
+        Some(())
+    }
+    let mut sum = 0;
+    for namespace in fs::read_dir(root.join("assets")).ok()? {
+        let textures = namespace.ok()?.path().join("textures");
+        if textures.is_dir() {
+            walk(&textures, root, &mut sum)?;
+        }
+    }
+    Some(sum)
+}
+
+impl SpriteCache {
+    /// The cache of `root`: its stored entries when still valid, else empty.
+    /// `None` when the textures cannot be fingerprinted (no caching then).
+    pub fn open(root: &Path) -> Option<Self> {
+        let fingerprint = fingerprint(root)?;
+        let path = root.join(FILE);
+        let entries = fs::read(&path)
+            .ok()
+            .and_then(|bytes| parse(&bytes, fingerprint))
+            .unwrap_or_default();
+        Some(Self {
+            path,
+            fingerprint,
+            entries,
+            dirty: false,
+        })
+    }
+    pub fn get(&self, name: &str) -> Option<&Decoded> {
+        self.entries.get(name)
+    }
+    pub fn insert(&mut self, name: &str, decoded: Decoded) {
+        self.entries.insert(name.to_owned(), decoded);
+        self.dirty = true;
+    }
+    /// Write new entries; a failure only costs the next launch the decoding.
+    pub fn save(&mut self) {
+        if !self.dirty {
+            return;
+        }
+        self.dirty = false;
+        let temporary = self
+            .path
+            .with_extension(format!("{}.tmp", std::process::id()));
+        if fs::write(&temporary, serialize(self.fingerprint, &self.entries)).is_ok()
+            && fs::rename(&temporary, &self.path).is_err()
+        {
+            let _ = fs::remove_file(&temporary);
+        }
+    }
+}
+
+fn serialize(fingerprint: u64, entries: &HashMap<String, Decoded>) -> Vec<u8> {
+    let mut out = Vec::new();
+    out.extend_from_slice(MAGIC);
+    out.extend_from_slice(&VERSION.to_le_bytes());
+    out.extend_from_slice(&fingerprint.to_le_bytes());
+    out.extend_from_slice(&(entries.len() as u32).to_le_bytes());
+    for (name, d) in entries {
+        put_bytes(&mut out, name.as_bytes());
+        out.extend_from_slice(&d.side.to_le_bytes());
+        out.extend_from_slice(&(d.frames.len() as u32).to_le_bytes());
+        for frame in &d.frames {
+            put_bytes(&mut out, frame);
+        }
+        put_bytes(&mut out, d.animation.as_deref().unwrap_or("").as_bytes());
+        out.push(u8::from(d.animation.is_some()));
+        out.push(d.strategy as u8);
+        out.extend_from_slice(&d.alpha_cutoff_bias.to_bits().to_le_bytes());
+        out.push(
+            u8::from(d.has_transparent) | u8::from(d.opaque) << 1 | u8::from(d.translucent) << 2,
+        );
+    }
+    out
+}
+fn put_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
+    out.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
+    out.extend_from_slice(bytes);
+}
+
+struct Reader<'a>(&'a [u8]);
+impl<'a> Reader<'a> {
+    fn take(&mut self, n: usize) -> Option<&'a [u8]> {
+        let (head, tail) = self.0.split_at_checked(n)?;
+        self.0 = tail;
+        Some(head)
+    }
+    fn u32(&mut self) -> Option<u32> {
+        Some(u32::from_le_bytes(self.take(4)?.try_into().ok()?))
+    }
+    fn bytes(&mut self) -> Option<&'a [u8]> {
+        let n = self.u32()? as usize;
+        self.take(n)
+    }
+}
+fn parse(bytes: &[u8], fingerprint: u64) -> Option<HashMap<String, Decoded>> {
+    let mut r = Reader(bytes);
+    if r.take(4)? != MAGIC || r.u32()? != VERSION {
+        return None;
+    }
+    if u64::from_le_bytes(r.take(8)?.try_into().ok()?) != fingerprint {
+        return None;
+    }
+    let count = r.u32()?;
+    let mut entries = HashMap::new();
+    for _ in 0..count {
+        let name = String::from_utf8(r.bytes()?.to_vec()).ok()?;
+        let side = r.u32()?;
+        let frame_count = r.u32()?;
+        let mut frames = Vec::new();
+        for _ in 0..frame_count {
+            frames.push(r.bytes()?.to_vec());
+        }
+        let animation = String::from_utf8(r.bytes()?.to_vec()).ok()?;
+        let has_animation = r.take(1)?[0] != 0;
+        let strategy = match r.take(1)?[0] {
+            0 => Strategy::Auto,
+            1 => Strategy::Mean,
+            2 => Strategy::Cutout,
+            3 => Strategy::StrictCutout,
+            4 => Strategy::DarkCutout,
+            _ => return None,
+        };
+        let alpha_cutoff_bias = f32::from_bits(r.u32()?);
+        let flags = r.take(1)?[0];
+        let frame_len = (side as usize).checked_mul(side as usize)?.checked_mul(4)?;
+        if frames.is_empty() || frames.iter().any(|f| f.len() != frame_len) {
+            return None;
+        }
+        entries.insert(
+            name,
+            Decoded {
+                side,
+                frames,
+                animation: has_animation.then_some(animation),
+                strategy,
+                alpha_cutoff_bias,
+                has_transparent: flags & 1 != 0,
+                opaque: flags & 2 != 0,
+                translucent: flags & 4 != 0,
+            },
+        );
+    }
+    r.0.is_empty().then_some(entries)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sample(blue: u8, animation: Option<&str>) -> Decoded {
+        Decoded {
+            side: 2,
+            frames: vec![vec![blue; 16], vec![blue + 1; 16]],
+            animation: animation.map(str::to_owned),
+            strategy: Strategy::Cutout,
+            alpha_cutoff_bias: 0.25,
+            has_transparent: true,
+            opaque: false,
+            translucent: true,
+        }
+    }
+    fn scratch(tag: &str) -> PathBuf {
+        let dir =
+            std::env::temp_dir().join(format!("horizon-sprite-cache-{tag}-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&dir);
+        fs::create_dir_all(dir.join("assets/minecraft/textures/block")).unwrap();
+        dir
+    }
+    #[test]
+    fn round_trips() {
+        let mut entries = HashMap::new();
+        entries.insert("a:b".to_owned(), sample(1, Some("{\"frametime\":2}")));
+        entries.insert("a:c".to_owned(), sample(9, None));
+        let back = parse(&serialize(7, &entries), 7).unwrap();
+        for (name, d) in &entries {
+            let e = &back[name];
+            assert_eq!(
+                (d.side, &d.frames, &d.animation),
+                (e.side, &e.frames, &e.animation)
+            );
+            assert_eq!(d.alpha_cutoff_bias, e.alpha_cutoff_bias);
+            assert_eq!(
+                (d.has_transparent, d.opaque, d.translucent, d.strategy as u8),
+                (e.has_transparent, e.opaque, e.translucent, e.strategy as u8)
+            );
+        }
+    }
+    #[test]
+    fn wrong_fingerprint_truncation_and_garbage_are_misses() {
+        let mut entries = HashMap::new();
+        entries.insert("a:b".to_owned(), sample(1, None));
+        let bytes = serialize(7, &entries);
+        assert!(parse(&bytes, 8).is_none());
+        for cut in 0..bytes.len() {
+            assert!(parse(&bytes[..cut], 7).is_none(), "cut {cut}");
+        }
+        assert!(parse(b"garbage", 7).is_none());
+        let mut extra = bytes.clone();
+        extra.push(0);
+        assert!(parse(&extra, 7).is_none());
+    }
+    #[test]
+    fn changed_texture_invalidates_and_corrupt_file_is_ignored() {
+        let root = scratch("invalidate");
+        let png = root.join("assets/minecraft/textures/block/x.png");
+        fs::write(&png, b"one").unwrap();
+        let mut cache = SpriteCache::open(&root).unwrap();
+        cache.insert("minecraft:block/x", sample(1, None));
+        cache.save();
+        assert!(
+            SpriteCache::open(&root)
+                .unwrap()
+                .get("minecraft:block/x")
+                .is_some()
+        );
+        // Same length, new content and time: still stale.
+        std::thread::sleep(std::time::Duration::from_millis(30));
+        fs::write(&png, b"two").unwrap();
+        assert!(
+            SpriteCache::open(&root)
+                .unwrap()
+                .get("minecraft:block/x")
+                .is_none()
+        );
+        // A new file also invalidates.
+        let mut cache = SpriteCache::open(&root).unwrap();
+        cache.insert("minecraft:block/x", sample(1, None));
+        cache.save();
+        fs::write(root.join("assets/minecraft/textures/block/y.png"), b"y").unwrap();
+        assert!(
+            SpriteCache::open(&root)
+                .unwrap()
+                .get("minecraft:block/x")
+                .is_none()
+        );
+        // Corrupt cache file: empty, no error.
+        fs::write(root.join(FILE), b"HSPR junk").unwrap();
+        assert!(
+            SpriteCache::open(&root)
+                .unwrap()
+                .get("minecraft:block/x")
+                .is_none()
+        );
+        let _ = fs::remove_dir_all(&root);
+    }
+    fn write_png(path: &Path, rgba: [u8; 4]) {
+        let mut encoder = png::Encoder::new(fs::File::create(path).unwrap(), 16, 16);
+        encoder.set_color(png::ColorType::Rgba);
+        encoder.set_depth(png::BitDepth::Eight);
+        let pixels: Vec<u8> = rgba.iter().copied().cycle().take(16 * 16 * 4).collect();
+        encoder
+            .write_header()
+            .unwrap()
+            .write_image_data(&pixels)
+            .unwrap();
+    }
+    fn sprite_pixels(root: &Path) -> Vec<u8> {
+        let mut sprites = crate::sprites::Sprites::with_disk_cache(root);
+        sprites.insert(root, "minecraft:block/x").unwrap();
+        sprites.save_disk_cache();
+        sprites.finish();
+        sprites.pixels
+    }
+    #[test]
+    fn altered_png_is_not_served_stale_and_cache_matches_decoding() {
+        let root = scratch("stale");
+        let png = root.join("assets/minecraft/textures/block/x.png");
+        write_png(&png, [255, 0, 0, 255]);
+        let decoded = sprite_pixels(&root);
+        assert!(root.join(FILE).exists());
+        assert_eq!(
+            sprite_pixels(&root),
+            decoded,
+            "cache hit differs from decoding"
+        );
+        std::thread::sleep(std::time::Duration::from_millis(30));
+        write_png(&png, [0, 255, 0, 255]);
+        let fresh = sprite_pixels(&root);
+        assert_eq!(&fresh[..4], &[0, 255, 0, 255]);
+        // Corrupt and deleted caches fall back to decoding.
+        fs::write(root.join(FILE), b"HSPR\x01\0\0\0 junk").unwrap();
+        assert_eq!(sprite_pixels(&root), fresh);
+        fs::remove_file(root.join(FILE)).unwrap();
+        assert_eq!(sprite_pixels(&root), fresh);
+        let _ = fs::remove_dir_all(&root);
+    }
+}

crates/horizon-client-render/src/sprites.rs

@@ -89,6 +89,8 @@ pub struct Sprites {
     /// textures under their own names, and one layer serves them all (the
     /// GPU's array may hold as few as 2048).
     still: std::collections::HashMap<(Vec<u8>, u32, [u32; 2], [bool; 4]), u32>,
+    /// Decoded textures kept on disk between launches ([`Sprites::with_disk_cache`]).
+    disk: Option<crate::sprite_cache::SpriteCache>,
 }
 /// Edge length of vanilla's block textures: the smallest array layer.
 pub const SPRITE_SIZE: u32 = 16;
@@ -111,6 +113,20 @@ pub(crate) fn resample(frame: &[u8], from: u32, to: u32) -> Vec<u8> {
     out
 }
 impl Sprites {
+    /// Sprites that reuse textures decoded by an earlier launch, kept in a
+    /// cache file in `root`.
+    pub fn with_disk_cache(root: &Path) -> Self {
+        Self {
+            disk: crate::sprite_cache::SpriteCache::open(root),
+            ..Self::default()
+        }
+    }
+    /// Write what this run decoded to the cache file.
+    pub fn save_disk_cache(&mut self) {
+        if let Some(disk) = &mut self.disk {
+            disk.save();
+        }
+    }
     /// How many array layers the textures take.
     pub fn count(&self) -> usize {
         self.native.len()
@@ -201,6 +217,21 @@ impl Sprites {
         }
     }
     fn load(&mut self, root: &Path, name: &str) -> Result<u32> {
+        let cached = self.disk.as_ref().and_then(|d| d.get(name)).cloned();
+        let decoded = match cached {
+            Some(decoded) => decoded,
+            None => {
+                let decoded = Self::decode(root, name)?;
+                if let Some(disk) = &mut self.disk {
+                    disk.insert(name, decoded.clone());
+                }
+                decoded
+            }
+        };
+        self.register(name, decoded)
+    }
+    /// Read and decode one texture and its `.mcmeta`.
+    fn decode(root: &Path, name: &str) -> Result<crate::sprite_cache::Decoded> {
         let (namespace, path) = name.split_once(':').ok_or("invalid sprite identifier")?;
         let path = root.join(format!("assets/{namespace}/textures/{path}.png"));
         let metadata = match fs::read_to_string(path.with_extension("png.mcmeta")) {
@@ -210,10 +241,13 @@ impl Sprites {
             Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
             Err(error) => return Err(error.into()),
         };
-        let animation = metadata
+        let animation_text = metadata
             .as_ref()
             .and_then(|v| v.get("animation"))
-            .map(|v| AnimationMetadata::from_json(v.get()))
+            .map(|v| v.get().to_owned());
+        let animation = animation_text
+            .as_deref()
+            .map(AnimationMetadata::from_json)
             .transpose()?;
         let mut decoder = png::Decoder::new(BufReader::new(File::open(path)?));
         decoder.set_transformations(png::Transformations::EXPAND | png::Transformations::STRIP_16);
@@ -247,7 +281,6 @@ impl Sprites {
             };
             pixels.extend(rgba);
         }
-        let layer = u32::try_from(self.native.len())?;
         let columns = info.width / width;
         let count = columns * (info.height / height);
         // Layers are square; a frame of another shape is nearest-resampled to
@@ -268,28 +301,56 @@ impl Sprites {
             }
             frames.push(frame);
         }
-        let settings = crate::mipmap::Settings {
-            native_size: side,
-            strategy: texture_section(metadata.as_ref())
+        let texture = texture_section(metadata.as_ref());
+        Ok(crate::sprite_cache::Decoded {
+            side,
+            frames,
+            animation: animation_text,
+            strategy: texture
                 .as_ref()
                 .and_then(|t| t["mipmap_strategy"].as_str())
                 .and_then(crate::mipmap::Strategy::parse)
                 .unwrap_or_default(),
-            alpha_cutoff_bias: texture_section(metadata.as_ref())
+            alpha_cutoff_bias: texture
                 .as_ref()
                 .and_then(|t| t["alpha_cutoff_bias"].as_f64())
                 .unwrap_or(0.0) as f32,
             has_transparent: pixels.as_chunks::<4>().0.iter().any(|p| p[3] == 0),
+            opaque: pixels.as_chunks::<4>().0.iter().all(|p| p[3] == 255),
+            translucent: pixels
+                .as_chunks::<4>()
+                .0
+                .iter()
+                .any(|p| p[3] != 0 && p[3] != 255),
+        })
+    }
+    /// Give a decoded texture its layer (or the one an identical still has).
+    fn register(&mut self, name: &str, decoded: crate::sprite_cache::Decoded) -> Result<u32> {
+        let crate::sprite_cache::Decoded {
+            side,
+            frames,
+            animation: animation_text,
+            strategy,
+            alpha_cutoff_bias,
+            has_transparent,
+            opaque,
+            translucent,
+        } = decoded;
+        let animation = animation_text
+            .as_deref()
+            .map(AnimationMetadata::from_json)
+            .transpose()?;
+        let count = frames.len() as u32;
+        let layer = u32::try_from(self.native.len())?;
+        let settings = crate::mipmap::Settings {
+            native_size: side,
+            strategy,
+            alpha_cutoff_bias,
+            has_transparent,
             item: name
                 .split_once(':')
                 .is_some_and(|(_, p)| p.starts_with("item/")),
         };
-        let opaque = pixels.as_chunks::<4>().0.iter().all(|p| p[3] == 255);
-        let translucent = pixels
-            .as_chunks::<4>()
-            .0
-            .iter()
-            .any(|p| p[3] != 0 && p[3] != 255);
         let still = (animation.is_none() && frames.len() == 1).then(|| {
             (
                 frames[0].clone(),

crates/horizon-client/examples/sprite_cache_parity.rs

@@ -0,0 +1,58 @@
+//! M-11 gate: block assets loaded through the sprite disk cache are byte-identical
+//! to the ones decoded from the PNGs. Deletes the cache file in ROOT, loads (decodes
+//! and writes it), loads again (reads it) and compares the sprite arrays.
+//! cargo run --profile fast -p horizon-client --example sprite_cache_parity -- ROOT
+use horizon_client::assets::{Assets, RenderAssets};
+use std::{
+    hash::{DefaultHasher, Hash, Hasher},
+    path::Path,
+    time::Instant,
+};
+
+fn digest(render: &RenderAssets) -> u64 {
+    let mut h = DefaultHasher::new();
+    render.pixels.hash(&mut h);
+    (
+        render.sprite_size,
+        render.sprite_minimum_size,
+        render.texture_layers,
+    )
+        .hash(&mut h);
+    render.sprite_layers.hash(&mut h);
+    render.translucent_layers.hash(&mut h);
+    format!("{:?}", render.sprite_mips).hash(&mut h);
+    for animation in &render.animations {
+        animation.layer.hash(&mut h);
+        for frame in 0..animation.frame_count() {
+            animation.pixels(frame).hash(&mut h);
+        }
+        for tick in 0..400 {
+            format!("{:?}", animation.blend(tick)).hash(&mut h);
+        }
+    }
+    h.finish()
+}
+
+fn main() -> horizon_client::Result<()> {
+    let root = std::env::args()
+        .nth(1)
+        .ok_or("usage: sprite_cache_parity ROOT")?;
+    let root = Path::new(&root);
+    let assets = Assets::load(root)?;
+    let _ = std::fs::remove_file(root.join("horizon-sprites.cache"));
+    let start = Instant::now();
+    let cold = digest(&assets.reload(root)?);
+    let cold_ms = start.elapsed().as_secs_f64() * 1000.;
+    let size = std::fs::metadata(root.join("horizon-sprites.cache"))?.len();
+    let start = Instant::now();
+    let warm = digest(&assets.reload(root)?);
+    let warm_ms = start.elapsed().as_secs_f64() * 1000.;
+    println!(
+        "cold={cold:016x} ({cold_ms:.1} ms) warm={warm:016x} ({warm_ms:.1} ms) cache_bytes={size}"
+    );
+    if cold != warm {
+        return Err("cached sprites differ from decoded sprites".into());
+    }
+    println!("PARITY OK");
+    Ok(())
+}