Loot tables from horizon-data.bin, parsed on first use (#4267)

horizon-prepare writes the prepared folder's data file with the loot
tables as their exported bytes, after parsing each one, and
--verify-data parses every table and compares it with its file. The game
maps it and parses a table when it is first read; a data pack's or
content mod's table still replaces the file's. A missing or stale file
(keyed by each source file's id, size and time) is rebuilt with the same
writer; a folder it can't write parses every table at load as before.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

Cargo.lock

@@ -1222,6 +1222,15 @@ dependencies = [
  "libm",
 ]
 
+[[package]]
+name = "horizon-data-file"
+version = "0.1.0"
+dependencies = [
+ "memmap2",
+ "serde_json",
+ "sha2",
+]
+
 [[package]]
 name = "horizon-dualsense"
 version = "0.1.0"
@@ -1301,6 +1310,7 @@ name = "horizon-prepare"
 version = "0.1.0"
 dependencies = [
  "crc32fast",
+ "horizon-data-file",
  "libc",
  "regex",
  "sha1",
@@ -1421,6 +1431,7 @@ version = "0.1.0"
 dependencies = [
  "flate2",
  "horizon-core",
+ "horizon-data-file",
  "libm",
  "lz4_flex",
  "postcard",

Cargo.toml

@@ -29,6 +29,7 @@ members = [
     "crates/horizon-shaders",
     "crates/horizon-shader-api",
     "crates/horizon-release-sign",
+    "crates/horizon-data-file",
     "xtask",
 ]
 resolver = "2"

crates/horizon-data-file/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "horizon-data-file"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+rust-version = "1.85"
+description = "horizon-data.bin: prepared vanilla tables in one file that horizon-prepare writes from the player's own copy and the game maps (#4267)"
+
+[dependencies]
+# All already in Cargo.lock: memmap2 for winit's Wayland client, the others for horizon-world.
+memmap2 = "0.9"
+serde_json = "1.0.151"
+sha2 = "0.10"
+
+[lints]
+workspace = true

crates/horizon-data-file/src/lib.rs

@@ -0,0 +1,425 @@
+//! `horizon-data.bin` (#4267): prepared vanilla tables in one file that
+//! `horizon-prepare` writes from the player's own copy of the game, next to
+//! the files it re-encodes, and that the game maps instead of parsing them
+//! into its heap. The pages are file-backed: shared between processes and
+//! read in only when touched. Nothing here is ever shipped.
+//!
+//! Layout (little-endian): a header, a section table, then the sections,
+//! each 8-aligned.
+//!
+//! ```text
+//! magic "HZDATA\0\0" | format u32 | sections u32 | key [u8; 32]
+//! sections × (name [u8; 24], offset u64, length u64)
+//! ```
+//!
+//! `key` names what the file was built from (the caller hashes its source
+//! files): a file whose key, format, magic or any bound does not match is
+//! refused, and the caller falls back to its normal loader.
+//!
+//! A [`Table`] section maps sorted keys to byte strings:
+//!
+//! ```text
+//! count u32 | 0 u32 | count × (key offset u32, key length u32,
+//!                              value offset u32, value length u32) | bytes
+//! ```
+//! with offsets into the bytes after the entries.
+use std::{
+    fs,
+    io::{self, Write},
+    path::{Path, PathBuf},
+    sync::atomic::{AtomicU64, Ordering},
+};
+
+pub mod prepared;
+
+pub const FILE_NAME: &str = "horizon-data.bin";
+const MAGIC: [u8; 8] = *b"HZDATA\0\0";
+const FORMAT: u32 = 1;
+const NAME_BYTES: usize = 24;
+const HEADER_BYTES: usize = 8 + 4 + 4 + 32;
+const SECTION_BYTES: usize = NAME_BYTES + 8 + 8;
+const ENTRY_BYTES: usize = 16;
+
+fn invalid(message: impl Into<String>) -> io::Error {
+    io::Error::new(io::ErrorKind::InvalidData, message.into())
+}
+fn u32_at(bytes: &[u8], at: usize) -> Option<u32> {
+    Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?))
+}
+fn u64_at(bytes: &[u8], at: usize) -> Option<u64> {
+    Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?))
+}
+
+/// Builds a data file in memory, then writes it in one rename.
+pub struct Writer {
+    key: [u8; 32],
+    sections: Vec<(String, Vec<u8>)>,
+}
+impl Writer {
+    pub fn new(key: [u8; 32]) -> Self {
+        Self {
+            key,
+            sections: Vec::new(),
+        }
+    }
+    /// A raw section. Names are at most 24 bytes and unique.
+    pub fn section(&mut self, name: &str, bytes: Vec<u8>) -> io::Result<()> {
+        if name.is_empty() || name.len() > NAME_BYTES {
+            return Err(invalid(format!("section name {name:?}")));
+        }
+        if self.sections.iter().any(|(n, _)| n == name) {
+            return Err(invalid(format!("section {name} twice")));
+        }
+        self.sections.push((name.to_owned(), bytes));
+        Ok(())
+    }
+    /// A [`Table`] section of `entries` (keys unique).
+    pub fn table(&mut self, name: &str, mut entries: Vec<(Vec<u8>, Vec<u8>)>) -> io::Result<()> {
+        entries.sort_by(|a, b| a.0.cmp(&b.0));
+        if entries.windows(2).any(|w| w[0].0 == w[1].0) {
+            return Err(invalid(format!("section {name}: a key twice")));
+        }
+        let count = u32::try_from(entries.len()).map_err(|_| invalid("too many entries"))?;
+        let mut head = Vec::with_capacity(8 + entries.len() * ENTRY_BYTES);
+        head.extend(count.to_le_bytes());
+        head.extend(0u32.to_le_bytes());
+        let mut blob = Vec::new();
+        for (key, value) in &entries {
+            for part in [key, value] {
+                let offset = u32::try_from(blob.len()).map_err(|_| invalid("table too big"))?;
+                let length = u32::try_from(part.len()).map_err(|_| invalid("entry too big"))?;
+                head.extend(offset.to_le_bytes());
+                head.extend(length.to_le_bytes());
+                blob.extend_from_slice(part);
+            }
+        }
+        head.extend(blob);
+        self.section(name, head)
+    }
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let align = |n: usize| n.next_multiple_of(8);
+        let mut offset = align(HEADER_BYTES + self.sections.len() * SECTION_BYTES);
+        let mut out = Vec::with_capacity(offset);
+        out.extend(MAGIC);
+        out.extend(FORMAT.to_le_bytes());
+        out.extend((self.sections.len() as u32).to_le_bytes());
+        out.extend(self.key);
+        for (name, bytes) in &self.sections {
+            let mut padded = [0u8; NAME_BYTES];
+            padded[..name.len()].copy_from_slice(name.as_bytes());
+            out.extend(padded);
+            out.extend((offset as u64).to_le_bytes());
+            out.extend((bytes.len() as u64).to_le_bytes());
+            offset = align(offset + bytes.len());
+        }
+        for (_, bytes) in &self.sections {
+            out.resize(align(out.len()), 0);
+            out.extend_from_slice(bytes);
+        }
+        out
+    }
+    /// Write to `path` through a temporary file of this call's own and a
+    /// rename. However many write at once (a window and its server thread,
+    /// two servers on one install), a reader maps the old file or a new one,
+    /// never one being written, and a game keeps the file it has mapped.
+    pub fn write(&self, path: &Path) -> io::Result<()> {
+        static NEXT: AtomicU64 = AtomicU64::new(0);
+        let mut temp = PathBuf::from(path);
+        temp.set_extension(format!(
+            "tmp{}-{}",
+            std::process::id(),
+            NEXT.fetch_add(1, Ordering::Relaxed)
+        ));
+        let mut file = fs::OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&temp)?;
+        file.write_all(&self.to_bytes())
+            .and_then(|()| file.sync_all())
+            .and_then(|()| fs::rename(&temp, path))
+            .inspect_err(|_| {
+                let _ = fs::remove_file(&temp);
+            })
+    }
+}
+
+enum Bytes {
+    Mapped(memmap2::Mmap),
+    #[cfg_attr(not(test), allow(dead_code))]
+    Owned(Vec<u8>),
+}
+impl std::ops::Deref for Bytes {
+    type Target = [u8];
+    fn deref(&self) -> &[u8] {
+        match self {
+            Self::Mapped(map) => map,
+            Self::Owned(bytes) => bytes,
+        }
+    }
+}
+
+/// An opened data file: its sections, every bound checked once.
+pub struct DataFile {
+    bytes: Bytes,
+    sections: Vec<(String, usize, usize)>,
+}
+impl DataFile {
+    /// Map `path` if it was built from `key`.
+    pub fn open(path: &Path, key: &[u8; 32]) -> io::Result<Self> {
+        let file = fs::File::open(path)?;
+        // SAFETY: the file is only ever replaced by a rename (`Writer::write`),
+        // never written in place, so the mapped bytes do not change under us.
+        #[allow(unsafe_code)]
+        let map = unsafe { memmap2::Mmap::map(&file)? };
+        Self::parse(Bytes::Mapped(map), key)
+    }
+    fn parse(bytes: Bytes, key: &[u8; 32]) -> io::Result<Self> {
+        if bytes.get(..8) != Some(&MAGIC[..]) {
+            return Err(invalid("not a Horizon data file"));
+        }
+        if u32_at(&bytes, 8) != Some(FORMAT) {
+            return Err(invalid("data file format"));
+        }
+        if bytes.get(16..48) != Some(&key[..]) {
+            return Err(invalid("data file built from other files"));
+        }
+        let count = u32_at(&bytes, 12).ok_or_else(|| invalid("truncated header"))? as usize;
+        let mut sections = Vec::with_capacity(count);
+        for i in 0..count {
+            let at = HEADER_BYTES + i * SECTION_BYTES;
+            let name = bytes
+                .get(at..at + NAME_BYTES)
+                .ok_or_else(|| invalid("truncated section table"))?;
+            let name = std::str::from_utf8(name)
+                .map_err(|_| invalid("section name"))?
+                .trim_end_matches('\0')
+                .to_owned();
+            let offset = u64_at(&bytes, at + NAME_BYTES).ok_or_else(|| invalid("section"))?;
+            let length = u64_at(&bytes, at + NAME_BYTES + 8).ok_or_else(|| invalid("section"))?;
+            let (offset, length) = (offset as usize, length as usize);
+            if offset % 8 != 0
+                || offset
+                    .checked_add(length)
+                    .is_none_or(|end| end > bytes.len())
+            {
+                return Err(invalid(format!("section {name} out of bounds")));
+            }
+            sections.push((name, offset, length));
+        }
+        Ok(Self { bytes, sections })
+    }
+    pub fn section(&self, name: &str) -> Option<&[u8]> {
+        let (_, offset, length) = self.sections.iter().find(|(n, ..)| n == name)?;
+        self.bytes.get(*offset..offset + length)
+    }
+    /// A [`Table`] section, its entries checked.
+    pub fn table(&self, name: &str) -> io::Result<Table<'_>> {
+        let bytes = self
+            .section(name)
+            .ok_or_else(|| invalid(format!("no section {name}")))?;
+        Table::new(bytes).map_err(|e| invalid(format!("section {name}: {e}")))
+    }
+    /// A [`Table`] section kept with its file, checked once here.
+    pub fn into_table(self, name: &str) -> io::Result<OwnedTable> {
+        let count = self.table(name)?.len();
+        let (_, offset, length) = self
+            .sections
+            .iter()
+            .find(|(n, ..)| n == name)
+            .cloned()
+            .ok_or_else(|| invalid(format!("no section {name}")))?;
+        Ok(OwnedTable {
+            file: self,
+            offset,
+            length,
+            count,
+        })
+    }
+}
+
+/// A checked [`Table`] section that owns its file.
+pub struct OwnedTable {
+    file: DataFile,
+    offset: usize,
+    length: usize,
+    count: usize,
+}
+impl OwnedTable {
+    pub fn table(&self) -> Table<'_> {
+        let bytes = &self.file.bytes[self.offset..self.offset + self.length];
+        let end = 8 + self.count * ENTRY_BYTES;
+        Table {
+            entries: &bytes[8..end],
+            blob: &bytes[end..],
+        }
+    }
+}
+
+/// Sorted keys to byte strings, read in place.
+#[derive(Clone, Copy)]
+pub struct Table<'a> {
+    entries: &'a [u8],
+    blob: &'a [u8],
+}
+impl<'a> Table<'a> {
+    fn new(bytes: &'a [u8]) -> io::Result<Self> {
+        let count = u32_at(bytes, 0).ok_or_else(|| invalid("truncated table"))? as usize;
+        let end = count
+            .checked_mul(ENTRY_BYTES)
+            .and_then(|n| n.checked_add(8))
+            .filter(|&end| end <= bytes.len())
+            .ok_or_else(|| invalid("truncated table"))?;
+        let table = Self {
+            entries: &bytes[8..end],
+            blob: &bytes[end..],
+        };
+        let mut previous: Option<&[u8]> = None;
+        for i in 0..count {
+            let key = table
+                .part(i, 0)
+                .ok_or_else(|| invalid("key out of bounds"))?;
+            table
+                .part(i, 8)
+                .ok_or_else(|| invalid("value out of bounds"))?;
+            if previous.is_some_and(|p| p >= key) {
+                return Err(invalid("keys not sorted"));
+            }
+            previous = Some(key);
+        }
+        Ok(table)
+    }
+    fn part(&self, i: usize, at: usize) -> Option<&'a [u8]> {
+        let offset = u32_at(self.entries, i * ENTRY_BYTES + at)? as usize;
+        let length = u32_at(self.entries, i * ENTRY_BYTES + at + 4)? as usize;
+        self.blob.get(offset..offset.checked_add(length)?)
+    }
+    pub fn len(&self) -> usize {
+        self.entries.len() / ENTRY_BYTES
+    }
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+    pub fn get(&self, key: &[u8]) -> Option<&'a [u8]> {
+        self.value(self.position(key)?)
+    }
+    /// The index of `key`'s entry, in key order.
+    pub fn position(&self, key: &[u8]) -> Option<usize> {
+        let (mut low, mut high) = (0, self.len());
+        while low < high {
+            let mid = (low + high) / 2;
+            match self.part(mid, 0)?.cmp(key) {
+                std::cmp::Ordering::Less => low = mid + 1,
+                std::cmp::Ordering::Greater => high = mid,
+                std::cmp::Ordering::Equal => return Some(mid),
+            }
+        }
+        None
+    }
+    pub fn value(&self, i: usize) -> Option<&'a [u8]> {
+        self.part(i, 8)
+    }
+    pub fn iter(&self) -> impl Iterator<Item = (&'a [u8], &'a [u8])> + '_ {
+        (0..self.len()).filter_map(|i| Some((self.part(i, 0)?, self.part(i, 8)?)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sample() -> Writer {
+        let mut writer = Writer::new([7; 32]);
+        writer.section("raw", b"abc".to_vec()).unwrap();
+        writer
+            .table(
+                "loot",
+                vec![
+                    (b"minecraft:b".to_vec(), b"{\"pools\":[]}".to_vec()),
+                    (b"minecraft:a".to_vec(), b"{}".to_vec()),
+                ],
+            )
+            .unwrap();
+        writer
+    }
+
+    #[test]
+    fn sections_and_tables_round_trip() {
+        let file = DataFile::parse(Bytes::Owned(sample().to_bytes()), &[7; 32]).unwrap();
+        assert_eq!(file.section("raw"), Some(&b"abc"[..]));
+        let table = file.table("loot").unwrap();
+        assert_eq!(table.len(), 2);
+        assert_eq!(table.get(b"minecraft:a"), Some(&b"{}"[..]));
+        assert_eq!(table.get(b"minecraft:b"), Some(&b"{\"pools\":[]}"[..]));
+        assert_eq!(table.get(b"minecraft:c"), None);
+        let keys: Vec<_> = table.iter().map(|(k, _)| k).collect();
+        assert_eq!(keys, [&b"minecraft:a"[..], &b"minecraft:b"[..]]);
+    }
+
+    #[test]
+    fn other_sources_and_damage_are_refused() {
+        let bytes = sample().to_bytes();
+        assert!(DataFile::parse(Bytes::Owned(bytes.clone()), &[8; 32]).is_err());
+        assert!(
+            DataFile::parse(Bytes::Owned(bytes[..bytes.len() - 1].to_vec()), &[7; 32]).is_err()
+        );
+        let mut wrong = bytes.clone();
+        wrong[0] = b'X';
+        assert!(DataFile::parse(Bytes::Owned(wrong), &[7; 32]).is_err());
+        // A table entry pointing past its bytes.
+        let file = DataFile::parse(Bytes::Owned(bytes), &[7; 32]).unwrap();
+        let (_, offset, _) = file
+            .sections
+            .iter()
+            .find(|(n, ..)| n == "loot")
+            .unwrap()
+            .clone();
+        let mut broken = file.bytes.to_vec();
+        broken[offset + 8 + 4..offset + 8 + 8].copy_from_slice(&u32::MAX.to_le_bytes());
+        let file = DataFile::parse(Bytes::Owned(broken), &[7; 32]).unwrap();
+        assert!(file.table("loot").is_err());
+    }
+
+    /// Writers at once never show a reader a file being written: each
+    /// writes its own temporary file and renames it whole.
+    #[test]
+    fn concurrent_writers_never_show_a_partial_file() {
+        let dir = std::env::temp_dir().join(format!("horizon-data-race-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&dir);
+        fs::create_dir_all(&dir).unwrap();
+        let path = dir.join(FILE_NAME);
+        let mut big = Writer::new([7; 32]);
+        big.section("raw", vec![5; 4 << 20]).unwrap();
+        big.write(&path).unwrap();
+        std::thread::scope(|s| {
+            for _ in 0..4 {
+                s.spawn(|| (0..8).for_each(|_| big.write(&path).unwrap()));
+            }
+            for _ in 0..4 {
+                s.spawn(|| {
+                    for _ in 0..64 {
+                        let file = DataFile::open(&path, &[7; 32]).unwrap();
+                        let raw = file.section("raw").unwrap();
+                        assert!(raw.len() == 4 << 20 && raw.iter().all(|&b| b == 5));
+                    }
+                });
+            }
+        });
+        // No temporary file left behind.
+        assert_eq!(fs::read_dir(&dir).unwrap().count(), 1);
+        let _ = fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn writes_by_rename_and_maps() {
+        let dir = std::env::temp_dir().join(format!("horizon-data-file-{}", std::process::id()));
+        fs::create_dir_all(&dir).unwrap();
+        let path = dir.join(FILE_NAME);
+        sample().write(&path).unwrap();
+        let file = DataFile::open(&path, &[7; 32]).unwrap();
+        assert_eq!(
+            file.table("loot").unwrap().get(b"minecraft:a"),
+            Some(&b"{}"[..])
+        );
+        let _ = fs::remove_dir_all(&dir);
+    }
+}

crates/horizon-data-file/src/prepared.rs

@@ -0,0 +1,200 @@
+//! The data file of a prepared resource folder, built from the files in it.
+//! `horizon-prepare` and the game (when the file is missing or stale) call the
+//! same [`write`], and the game's loaders list their source files through the
+//! same [`json_files`], so the ids are the same by construction.
+use crate::{DataFile, FILE_NAME, OwnedTable, Writer, invalid};
+use sha2::{Digest, Sha256};
+use std::{
+    collections::HashMap,
+    fs, io,
+    path::{Path, PathBuf},
+    time::{Duration, Instant, UNIX_EPOCH},
+};
+
+/// `data/<ns>/loot_table/**.json` of every namespace: the exported bytes by id.
+pub const LOOT_TABLES: &str = "loot_tables";
+
+/// Every `<data>/<ns>/<kind>/**.json` as (`ns:path`, file), namespaces in name
+/// order.
+pub fn json_files(data: &Path, kind: &str) -> Vec<(String, PathBuf)> {
+    let Ok(entries) = fs::read_dir(data) else {
+        return Vec::new();
+    };
+    let mut namespaces: Vec<String> = entries
+        .flatten()
+        .filter(|e| e.path().is_dir())
+        .filter_map(|e| e.file_name().to_str().map(str::to_owned))
+        .collect();
+    namespaces.sort();
+    let mut out = Vec::new();
+    for ns in namespaces {
+        json_dir(&data.join(&ns).join(kind), &ns, "", &mut out);
+    }
+    out
+}
+
+/// The `.json` files under `dir` as (`namespace:prefix/path`, file).
+pub fn json_dir(dir: &Path, namespace: &str, prefix: &str, out: &mut Vec<(String, PathBuf)>) {
+    let Ok(entries) = fs::read_dir(dir) else {
+        return;
+    };
+    for entry in entries.flatten() {
+        let path = entry.path();
+        let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else {
+            continue;
+        };
+        if path.is_dir() {
+            json_dir(&path, namespace, &format!("{prefix}{stem}/"), out);
+        } else if path.extension().is_some_and(|e| e == "json") {
+            out.push((format!("{namespace}:{prefix}{stem}"), path));
+        }
+    }
+}
+
+/// The loot table files of `root`, one per id (the last listed, as a map
+/// keeps it), in id order.
+fn loot_files(root: &Path) -> Vec<(String, PathBuf)> {
+    let files: HashMap<String, PathBuf> = json_files(&root.join("data"), "loot_table")
+        .into_iter()
+        .collect();
+    let mut files: Vec<_> = files.into_iter().collect();
+    files.sort();
+    files
+}
+
+/// What a data file is built from: each source file's id, size and
+/// modification time. A changed, added or removed file makes it stale.
+fn key(files: &[(String, PathBuf)]) -> io::Result<[u8; 32]> {
+    let mut hash = Sha256::new();
+    hash.update(LOOT_TABLES);
+    for (id, path) in files {
+        let meta = fs::metadata(path)?;
+        let modified = meta
+            .modified()?
+            .duration_since(UNIX_EPOCH)
+            .map_or(0, |d| d.as_nanos());
+        hash.update([0]);
+        hash.update(id);
+        hash.update([0]);
+        hash.update(meta.len().to_le_bytes());
+        hash.update(modified.to_le_bytes());
+    }
+    Ok(hash.finalize().into())
+}
+
+/// Write `root`'s data file from the files in it and return how many loot
+/// tables it holds. Every table must parse here, so a bad one still fails
+/// at load and never at its first use. A root without loot tables gets no
+/// file.
+pub fn write(root: &Path) -> io::Result<usize> {
+    let files = loot_files(root);
+    if files.is_empty() {
+        return Ok(0);
+    }
+    let key = key(&files)?;
+    let mut entries = Vec::with_capacity(files.len());
+    for (id, path) in files {
+        let bytes = fs::read(&path)?;
+        serde_json::from_slice::<serde_json::Value>(&bytes)
+            .map_err(|e| invalid(format!("{}: {e}", path.display())))?;
+        entries.push((id.into_bytes(), bytes));
+    }
+    let count = entries.len();
+    let mut writer = Writer::new(key);
+    writer.table(LOOT_TABLES, entries)?;
+    writer.write(&root.join(FILE_NAME))?;
+    Ok(count)
+}
+
+/// `root`'s loot tables from its data file, if the file is current.
+pub fn loot_tables(root: &Path) -> io::Result<OwnedTable> {
+    let key = key(&loot_files(root))?;
+    DataFile::open(&root.join(FILE_NAME), &key)?.into_table(LOOT_TABLES)
+}
+
+/// What [`verify`] found.
+pub struct Verified {
+    pub tables: usize,
+    /// The slowest single table parse and its id: what a first use costs at most.
+    pub slowest: (Duration, String),
+}
+
+/// Parse every table of `root`'s data file and compare it with its source
+/// file's parse.
+pub fn verify(root: &Path) -> io::Result<Verified> {
+    let files = loot_files(root);
+    let mapped = loot_tables(root)?;
+    let table = mapped.table();
+    if table.len() != files.len() {
+        return Err(invalid(format!(
+            "{} loot tables in {FILE_NAME}, {} files",
+            table.len(),
+            files.len()
+        )));
+    }
+    let mut slowest = (Duration::ZERO, String::new());
+    for (id, path) in &files {
+        let bytes = table
+            .get(id.as_bytes())
+            .ok_or_else(|| invalid(format!("{id} missing from {FILE_NAME}")))?;
+        let start = Instant::now();
+        let value: serde_json::Value = serde_json::from_slice(bytes)
+            .map_err(|e| invalid(format!("{id} in {FILE_NAME}: {e}")))?;
+        let took = start.elapsed();
+        let source: serde_json::Value = serde_json::from_slice(&fs::read(path)?)
+            .map_err(|e| invalid(format!("{}: {e}", path.display())))?;
+        if value != source {
+            return Err(invalid(format!("{id} differs from {}", path.display())));
+        }
+        if took > slowest.0 {
+            slowest = (took, id.clone());
+        }
+    }
+    Ok(Verified {
+        tables: files.len(),
+        slowest,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn root(name: &str) -> PathBuf {
+        let root =
+            std::env::temp_dir().join(format!("horizon-prepared-{name}-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&root);
+        let dir = root.join("data/minecraft/loot_table/blocks");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(dir.join("stone.json"), br#"{"pools":[]}"#).unwrap();
+        fs::write(root.join("data/minecraft/loot_table/empty.json"), b"{}").unwrap();
+        root
+    }
+
+    #[test]
+    fn writes_maps_verifies_and_goes_stale() {
+        let root = root("stale");
+        assert_eq!(write(&root).unwrap(), 2);
+        let mapped = loot_tables(&root).unwrap();
+        assert_eq!(
+            mapped.table().get(b"minecraft:blocks/stone"),
+            Some(&br#"{"pools":[]}"#[..])
+        );
+        assert_eq!(verify(&root).unwrap().tables, 2);
+        // A new table makes the file stale until it is written again.
+        fs::write(root.join("data/minecraft/loot_table/more.json"), b"[]").unwrap();
+        assert!(loot_tables(&root).is_err());
+        assert_eq!(write(&root).unwrap(), 3);
+        assert!(loot_tables(&root).is_ok());
+        let _ = fs::remove_dir_all(&root);
+    }
+
+    #[test]
+    fn a_bad_table_fails_the_write() {
+        let root = root("bad");
+        fs::write(root.join("data/minecraft/loot_table/bad.json"), b"{").unwrap();
+        assert!(write(&root).is_err());
+        assert!(!root.join(FILE_NAME).exists());
+        let _ = fs::remove_dir_all(&root);
+    }
+}

crates/horizon-items/src/items.rs

@@ -263,7 +263,7 @@ pub struct Items {
     /// Item tags in vanilla's order (`TagLoader`: file order, nested tags in
     /// place, first occurrence kept), the order `TagSlotDisplay` resolves.
     pub item_tag_order: HashMap<String, Vec<String>>,
-    loot: Arc<HashMap<String, Value>>,
+    loot: Arc<horizon_world::loot::LootTables>,
     /// The `jukebox_song` registry.
     pub songs: HashMap<String, Song>,
     /// The enchantment registry: loot functions enchant the items they roll

crates/horizon-prepare/Cargo.toml

@@ -6,6 +6,7 @@ license = "MIT"
 description = "Prepares Horizon's resources from the player's own Minecraft: imports and checks the pinned jars, runs the vanilla exporters, writes the resource folder"
 
 [dependencies]
+horizon-data-file = { path = "../horizon-data-file" }
 crc32fast = "1"
 regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] }
 sha1 = "0.10"

crates/horizon-prepare/src/lib.rs

@@ -504,6 +504,12 @@ fn prepare_impl(options: &Options, log: Log, compile_only: bool) -> Result<PathB
         return Ok(run.bundle()?.to_path_buf());
     }
     steps(&run, &lock, &oracle, options.minecraft_dir.as_deref())?;
+    let tables = horizon_data_file::prepared::write(&resources)
+        .map_err(|e| format!("{}: {e}", horizon_data_file::FILE_NAME))?;
+    log(&format!(
+        "Wrote {} ({tables} loot tables)",
+        horizon_data_file::FILE_NAME
+    ));
     log(&format!(
         "Prepared Horizon native-client resources: {}",
         resources.display()
@@ -511,6 +517,26 @@ fn prepare_impl(options: &Options, log: Log, compile_only: bool) -> Result<PathB
     Ok(resources)
 }
 
+/// `--verify-data`: decode every table of the prepared folder's data file and
+/// compare it with the file it was built from.
+pub fn verify_data(root: &Path, log: Log) -> Result<PathBuf, String> {
+    let lock = artifacts::Lock::read(root)?;
+    let resources = root
+        .join(".local")
+        .join("horizon-client")
+        .join(&lock.version);
+    let verified = horizon_data_file::prepared::verify(&resources)
+        .map_err(|e| format!("{}: {e}", horizon_data_file::FILE_NAME))?;
+    let (took, id) = &verified.slowest;
+    log(&format!(
+        "{}: {} loot tables match their files; slowest parse {id} ({} µs)",
+        horizon_data_file::FILE_NAME,
+        verified.tables,
+        took.as_micros()
+    ));
+    Ok(resources)
+}
+
 fn arg(path: &Path) -> String {
     path.display().to_string()
 }

crates/horizon-prepare/src/main.rs

@@ -16,6 +16,7 @@ const HELP: &str = "\
 horizon-prepare: prepare Horizon's resources from your own Minecraft Java Edition
 
 Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [--root PATH]
+       horizon-prepare --verify-data [--root PATH]
        horizon-prepare compile-exporters [--root PATH]
        horizon-prepare sounds [--minecraft-dir PATH] [--root PATH]
        horizon-prepare panorama [--minecraft-dir PATH] [--root PATH]
@@ -28,6 +29,7 @@ Usage: horizon-prepare [--prepare] [--server-jar PATH] [--minecraft-dir PATH] [-
                         (default: HORIZON_SERVER_JAR, or the copy imported before)
   --minecraft-dir PATH  The official launcher's game directory with your installed copy
                         (default: HORIZON_MINECRAFT_DIR, or the launcher's own)
+  --verify-data         Check that horizon-data.bin matches the prepared files it was built from
   --root PATH           The Horizon folder to prepare (default: the folder this program is
                         in, else the nearest folder above the current one with research-lock.json)
 
@@ -71,6 +73,7 @@ fn main() -> ExitCode {
         java_home: None,
     };
     let mut root = None;
+    let mut verify_data = false;
     let mut args = std::env::args_os().skip(1).peekable();
     let assets = args.next_if(|a| {
         matches!(
@@ -103,6 +106,10 @@ fn main() -> ExitCode {
                 options.import = true;
                 Ok(())
             }
+            "--verify-data" if assets.is_none() && inline.is_none() => {
+                verify_data = true;
+                Ok(())
+            }
             "--server-jar" => value("--server-jar").map(|v| options.server_jar = Some(v)),
             "--minecraft-dir" => value("--minecraft-dir").map(|v| options.minecraft_dir = Some(v)),
             "--root" => value("--root").map(|v| root = Some(v)),
@@ -151,6 +158,7 @@ fn main() -> ExitCode {
         Some("languages") => {
             horizon_prepare::languages(&options.root, options.minecraft_dir.as_deref(), &log)
         }
+        _ if verify_data => horizon_prepare::verify_data(&options.root, &log),
         _ => horizon_prepare::prepare(&options, &log),
     };
     match result {

crates/horizon-world/Cargo.toml

@@ -13,6 +13,7 @@ serde = { version = "1", features = ["derive"] }
 sha2 = "0.10"
 postcard = { version = "=1.1.3", default-features = false, features = ["alloc"] }
 horizon-core = { path = "../horizon-core" }
+horizon-data-file = { path = "../horizon-data-file" }
 flate2 = { version = "1.1.10", default-features = false, features = ["zlib-rs", "runtime_detection"] }
 lz4_flex = { version = "0.14", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] }
 xxhash-rust = { version = "0.8.19", features = ["xxh32"] }

crates/horizon-world/src/loot.rs

@@ -14,7 +14,7 @@ use std::{
     collections::{BTreeMap, HashMap},
     fs, io,
     path::Path,
-    sync::{Arc, Mutex, Weak},
+    sync::{Arc, Mutex, OnceLock, Weak},
 };
 
 /// Raw files a world's data packs add over the game's own data, by
@@ -158,7 +158,7 @@ impl Enchantment {
 
 /// Loot tables, items, enchantments and the tags they reference.
 pub struct LootData {
-    tables: Arc<HashMap<String, Value>>,
+    tables: Arc<LootTables>,
     modifiers: HashMap<String, Value>,
     predicates: HashMap<String, Value>,
     items: HashMap<String, ItemFacts>,
@@ -319,14 +319,62 @@ fn overlay_map(overlay: &PackOverlay, kind: &str, out: &mut HashMap<String, Valu
     }
 }
 
+/// Loot tables by id: the packs' tables, over the prepared data file's
+/// (#4267), each parsed on its first use. A root whose data file can't be
+/// mapped or written parses every table at load instead.
+#[derive(Default)]
+pub struct LootTables {
+    mapped: Option<(horizon_data_file::OwnedTable, Vec<OnceLock<Option<Value>>>)>,
+    parsed: HashMap<String, Value>,
+}
+impl LootTables {
+    fn load(root: &Path, overlay: &PackOverlay) -> io::Result<Self> {
+        use horizon_data_file::prepared;
+        // Missing or stale: rebuild it with horizon-prepare's writer, which
+        // parses every table, so a bad one still fails here.
+        let mapped = prepared::loot_tables(root).or_else(|_| {
+            prepared::write(root)?;
+            prepared::loot_tables(root)
+        });
+        let mut parsed = HashMap::new();
+        let mapped = match mapped {
+            Ok(file) => {
+                let values = (0..file.table().len()).map(|_| OnceLock::new()).collect();
+                Some((file, values))
+            }
+            Err(_) => {
+                load_kind(&root.join("data"), "loot_table", &mut parsed)?;
+                None
+            }
+        };
+        overlay_map(overlay, "loot_table", &mut parsed);
+        Ok(Self { mapped, parsed })
+    }
+    pub fn get(&self, id: &str) -> Option<&Value> {
+        if let Some(table) = self.parsed.get(id) {
+            return Some(table);
+        }
+        let (file, values) = self.mapped.as_ref()?;
+        let table = file.table();
+        let i = table.position(id.as_bytes())?;
+        values[i]
+            .get_or_init(|| serde_json::from_slice(table.value(i)?).ok())
+            .as_ref()
+    }
+    pub fn contains_key(&self, id: &str) -> bool {
+        self.parsed.contains_key(id)
+            || self
+                .mapped
+                .as_ref()
+                .is_some_and(|(file, _)| file.table().position(id.as_bytes()).is_some())
+    }
+}
+
 /// The loot tables of every namespace in `root`'s `data/`, with `overlay`'s
 /// over them. Shared while one is alive: `Items` and [`LootData`] of the
-/// same root and packs hold one map, not a copy each (7 MB).
-pub fn shared_tables(
-    root: &Path,
-    overlay: &PackOverlay,
-) -> io::Result<Arc<HashMap<String, Value>>> {
-    static LOADED: Mutex<Vec<(u64, Weak<HashMap<String, Value>>)>> = Mutex::new(Vec::new());
+/// same root and packs hold one, not a copy each.
+pub fn shared_tables(root: &Path, overlay: &PackOverlay) -> io::Result<Arc<LootTables>> {
+    static LOADED: Mutex<Vec<(u64, Weak<LootTables>)>> = Mutex::new(Vec::new());
     let key = {
         use std::hash::{DefaultHasher, Hash, Hasher};
         let mut hasher = DefaultHasher::new();
@@ -345,28 +393,13 @@ pub fn shared_tables(
     {
         return Ok(tables);
     }
-    let mut tables = HashMap::new();
-    load_kind(&root.join("data"), "loot_table", &mut tables)?;
-    overlay_map(overlay, "loot_table", &mut tables);
-    let tables = Arc::new(tables);
+    let tables = Arc::new(LootTables::load(root, overlay)?);
     loaded.push((key, Arc::downgrade(&tables)));
     Ok(tables)
 }
 
 fn load_kind(data: &Path, kind: &str, out: &mut HashMap<String, Value>) -> io::Result<()> {
-    let Ok(entries) = fs::read_dir(data) else {
-        return Ok(());
-    };
-    let mut namespaces: Vec<String> = entries
-        .flatten()
-        .filter(|e| e.path().is_dir())
-        .filter_map(|e| e.file_name().to_str().map(str::to_owned))
-        .collect();
-    namespaces.sort();
-    for ns in namespaces {
-        load_dir(&data.join(&ns).join(kind), &ns, "", out)?;
-    }
-    Ok(())
+    parse_files(horizon_data_file::prepared::json_files(data, kind), out)
 }
 
 fn load_dir(
@@ -375,20 +408,17 @@ fn load_dir(
     prefix: &str,
     out: &mut HashMap<String, Value>,
 ) -> io::Result<()> {
-    let Ok(entries) = fs::read_dir(dir) else {
-        return Ok(());
-    };
-    for entry in entries.flatten() {
-        let path = entry.path();
-        let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else {
-            continue;
-        };
-        if path.is_dir() {
-            load_dir(&path, namespace, &format!("{prefix}{stem}/"), out)?;
-        } else if path.extension().is_some_and(|e| e == "json") {
-            let value = serde_json::from_slice(&fs::read(&path)?)?;
-            out.insert(format!("{namespace}:{prefix}{stem}"), value);
-        }
+    let mut files = Vec::new();
+    horizon_data_file::prepared::json_dir(dir, namespace, prefix, &mut files);
+    parse_files(files, out)
+}
+
+fn parse_files(
+    files: Vec<(String, std::path::PathBuf)>,
+    out: &mut HashMap<String, Value>,
+) -> io::Result<()> {
+    for (id, path) in files {
+        out.insert(id, serde_json::from_slice(&fs::read(&path)?)?);
     }
     Ok(())
 }
@@ -1646,6 +1676,95 @@ mod namespace_tests {
         let _ = fs::remove_dir_all(&root);
     }
 
+    fn loot_root(name: &str) -> std::path::PathBuf {
+        let root = std::env::temp_dir().join(format!("horizon-loot-{name}-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&root);
+        fs::create_dir_all(root.join("data/minecraft/loot_table/chests")).unwrap();
+        for (name, text) in [
+            ("a", r#"{"pools": [{"rolls": 1.0, "entries": []}]}"#),
+            ("b", r#"{"random_sequence": "minecraft:chests/b"}"#),
+        ] {
+            fs::write(
+                root.join(format!("data/minecraft/loot_table/chests/{name}.json")),
+                text,
+            )
+            .unwrap();
+        }
+        root
+    }
+
+    /// Every table read through the data file equals today's parse of its
+    /// file (`horizon-prepare --verify-data` checks the same on a real root).
+    fn assert_mapped_matches(root: &Path) {
+        let tables = LootTables::load(root, &PackOverlay::NONE).unwrap();
+        assert!(
+            tables.mapped.is_some(),
+            "no data file in {}",
+            root.display()
+        );
+        assert!(tables.parsed.is_empty());
+        let mut parsed = HashMap::new();
+        load_kind(&root.join("data"), "loot_table", &mut parsed).unwrap();
+        assert_eq!(
+            tables.mapped.as_ref().unwrap().0.table().len(),
+            parsed.len()
+        );
+        for (id, value) in &parsed {
+            assert!(tables.contains_key(id));
+            assert_eq!(tables.get(id), Some(value), "{id}");
+        }
+        assert!(!tables.contains_key("minecraft:chests/none"));
+    }
+
+    #[test]
+    fn mapped_tables_match_the_parsed_ones() {
+        let root = loot_root("mapped");
+        assert_mapped_matches(&root);
+        // A damaged file is rebuilt rather than read.
+        fs::write(root.join(horizon_data_file::FILE_NAME), b"HZDATA").unwrap();
+        assert_mapped_matches(&root);
+        let _ = fs::remove_dir_all(&root);
+        let prepared = std::env::var_os("HORIZON_RESOURCES")
+            .map(std::path::PathBuf::from)
+            .unwrap_or_else(|| {
+                Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.local/horizon-client/26.3")
+            });
+        if prepared.join("data/minecraft/loot_table").is_dir() {
+            assert_mapped_matches(&prepared);
+        } else {
+            eprintln!("skipping the real tables: none at {}", prepared.display());
+        }
+    }
+
+    /// A data pack's or content mod's table replaces the data file's, and
+    /// adds new ones.
+    #[test]
+    fn pack_tables_win_over_the_data_file() {
+        let root = loot_root("packs");
+        let files = [
+            (
+                "loot_table/minecraft:chests/a".to_owned(),
+                vec![br#"{"pools": []}"#.to_vec()],
+            ),
+            (
+                "loot_table/minecraft:chests/c".to_owned(),
+                vec![br#"{"type": "minecraft:chest"}"#.to_vec()],
+            ),
+        ];
+        let tables = LootTables::load(&root, &PackOverlay { files: &files }).unwrap();
+        assert!(tables.mapped.is_some());
+        assert_eq!(
+            tables.get("minecraft:chests/a"),
+            Some(&serde_json::json!({"pools": []}))
+        );
+        assert_eq!(
+            tables.get("minecraft:chests/b").unwrap()["random_sequence"],
+            "minecraft:chests/b"
+        );
+        assert!(tables.contains_key("minecraft:chests/c"));
+        let _ = fs::remove_dir_all(&root);
+    }
+
     /// A content mods' root: the mods' loot tables and tags load beside
     /// vanilla's, under their own namespaces.
     #[test]

tests/test_lab.py

@@ -170,7 +170,8 @@ class ArtifactTests(unittest.TestCase):
 
 class PublicationTests(unittest.TestCase):
     def test_private_paths_rejected(self):
-        for path in [".local/minecraft/source.java", "decompiled/Foo.txt", "game.JAR", "capture.jfr", "source/Foo.java"]:
+        for path in [".local/minecraft/source.java", "decompiled/Foo.txt", "game.JAR", "capture.jfr", "source/Foo.java",
+                     "horizon-data.bin", "crates/horizon-data-file/tests/horizon-data.bin"]:
             self.assertTrue(check_public_tree.prohibited(path), path)
         for path in ["tools/lab.py", "crates/horizon-core/src/lib.rs", "Cargo.lock", "docs/analysis.md",
                      "mods/horizon/src/main/java/dev/horizonlab/HorizonLab.java",

tools/check_public_tree.py

@@ -9,6 +9,8 @@ import sys
 PRIVATE_ROOTS = {".local", "decompiled", "minecraft", "assets", "run", "logs"}
 PRIVATE_EXTENSIONS = {".jar", ".class", ".jfr", ".hprof", ".java"}
 ORIGINAL_JDK_TOOLS = {"tools/JfrSummary.java", "tests/java/JfrFixture.java"}
+# Files prepared from the player's own game copy, wherever they are (#4267).
+PRIVATE_NAMES = {"horizon-data.bin"}
 
 
 def prohibited(path):
@@ -19,6 +21,8 @@ def prohibited(path):
     # Original Horizon mod sources may also be tracked.
     if parsed.suffix == ".java" and path.startswith(("mods/horizon/src/", "mods/horizon-lab/src/")):
         return not ("/dev/horizonlab/" in path)
+    if parsed.name in PRIVATE_NAMES:
+        return True
     return bool(parsed.parts and parsed.parts[0] in PRIVATE_ROOTS) or parsed.suffix.lower() in PRIVATE_EXTENSIONS