diff --git a/crates/horizon-prepare/src/java.rs b/crates/horizon-prepare/src/java.rs index 92660bc90c..5ca590a754 100644 --- a/crates/horizon-prepare/src/java.rs +++ b/crates/horizon-prepare/src/java.rs @@ -15,7 +15,7 @@ pub struct Jdk { const TIMEOUT: Duration = Duration::from_secs(3600); /// Options for every JVM preparing starts (the exporters, the data generator, javac with `-J`). -/// They are short batch jobs that run three at a time: the C2 compiler's background work costs +/// They are short batch jobs that run side by side: the C2 compiler's background work costs /// more CPU than it wins back, and the serial collector keeps their heaps small. Measured on the /// pinned jars, a fresh preparation took 105 s with the defaults and 63-66 s with these, its /// JVMs together at most 1.25 GB (2 GB with the parallel collector, as fast), same output. diff --git a/crates/horizon-prepare/src/lib.rs b/crates/horizon-prepare/src/lib.rs index 6d73a937d4..a0ac47106f 100644 --- a/crates/horizon-prepare/src/lib.rs +++ b/crates/horizon-prepare/src/lib.rs @@ -754,7 +754,8 @@ type Entries = Vec<(&'static str, Value)>; /// The exporters, the data generator and the asset copy, on four lanes that run at once: /// the server exporter JVM's steps in a fixed order (they share its state), the client /// exporter JVM's then the command registries' JVM, the data generator's then the creative -/// inventory's, and the copying, which needs no Java. At most three JVMs run at a time. +/// inventory's, and the copying, which needs no Java. At most three JVMs run at a time, four +/// with the split below. /// From this many cores (`pool::jobs`) the generator's block behaviour gets its own server JVM. /// It pays a second Bootstrap and ~400 MB, which only pays off when the other lanes leave cores /// idle: a 6-core laptop finished its other JVMs 20 s before the server lane. diff --git a/docs/launcher-preparation.md b/docs/launcher-preparation.md index 732fc71212..62090e2957 100644 --- a/docs/launcher-preparation.md +++ b/docs/launcher-preparation.md @@ -41,13 +41,13 @@ Fresh release preparation therefore starts five exporter/data-generator JVMs, do (six with 6 cores or more, below), and invokes no compiler. `HORIZON_PREPARE_ISOLATED=1` retains separate exporter JVMs for output comparisons and diagnosis. -Those JVMs run on four lanes at once, so at most three run at a time: the server worker's -steps in their fixed order; the client worker's, then the command registries'; the data -generator, then the creative tabs'; and the Rust-only copying (client assets and data, the -world generator's bundle), spread over several threads. The world generator's tables, which -need the block facts and the reports, and the manifest come after all four. A worker stops -when its lane is done. The two sides compile at once too. Imports and checks hash the pinned -files on several threads; every file is still checked in full. +Those JVMs run on four lanes at once, so at most three run at a time (four with the split +below): the server worker's steps in their fixed order; the client worker's, then the command +registries'; the data generator, then the creative tabs'; and the Rust-only copying (client +assets and data, the world generator's bundle), spread over several threads. The world +generator's tables, which need the block facts and the reports, and the manifest come after +all four. A worker stops when its lane is done. The two sides compile at once too. Imports and +checks hash the pinned files on several threads; every file is still checked in full. `HORIZON_PREPARE_JOBS` is how many things run at once: the processors by default, at most 8. `1` runs it all one step after another (for a machine short on memory). From 6, the world diff --git a/tools/check_prepare_exporters.py b/tools/check_prepare_exporters.py index 1ac1920aef..839c8620ea 100644 --- a/tools/check_prepare_exporters.py +++ b/tools/check_prepare_exporters.py @@ -146,8 +146,9 @@ def main(): times.append(round(time.monotonic() - started, 2)) print(f'{name}: {times[-1]}s, prepared', flush=True) verify(results) - # Nothing changed since the last run: it starts no JVM and rewrites nothing. - before = hashes(resources) + # Nothing changed since the last run: it starts no JVM and leaves every file as it was. + manifest = resources / 'horizon-resources.json' + before = hashes(resources), manifest.read_bytes() trace.write_text('') with (work / 'repeat.log').open('w') as log: subprocess.run([str(binary), '--root', str(folder)], cwd=folder, env=env, @@ -155,7 +156,7 @@ def main(): launches = [line for line in trace.read_text().splitlines() if line != '-version'] if launches: raise SystemExit(f'A repeat preparation started {len(launches)} JVMs') - if hashes(resources) != before: + if (hashes(resources), manifest.read_bytes()) != before: raise SystemExit('A repeat preparation changed the resources') print('Repeat preparation: no JVM, same files') print(f'Isolated {times[:2]}s, batched {times[2:]}s')