mirror of
https://github.com/horizon-rust/horizon.git
synced 2026-11-07 11:06:11 -10:00
Ludovic MarieandClaude Opus 5.5
0b581f9ffe
horizon-client, horizon-world and horizon-protocol each had an example named `probe`. Cargo copies every crate's examples into one <profile>/examples/ directory, so parallel parity harnesses sharing a target dir could run another crate's probe (CI parity run 40055656303: command_parse ran the horizon-world dispatcher). The examples are now client_probe, world_probe and protocol_probe ([[example]] in each Cargo.toml; the files stay where they are), and every harness builds and runs its own crate's name. tools/parity.py recognises the new names when it counts the probes a harness ran, and renames the old `probe` in recorded artifacts to the crate's target (from the record's -p, else its probe sources), so the prebuild still builds them instead of dropping them as unknown. Co-Authored-By: Claude Opus 5.5 <[email protected]>
132 lines
6.7 KiB
Python
132 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare Horizon's command parser with vanilla's real Brigadier command tree.
|
|
|
|
Java (VanillaCommandParseOracle) builds vanilla's `Commands` dispatcher with
|
|
the real registries and data-pack loaded tags, exports the whole tree, and
|
|
parses a corpus of command strings. The corpus is generated by walking that
|
|
tree (every command and subcommand with valid and invalid sample values per
|
|
argument type) plus hand-written edge cases for selectors, NBT paths,
|
|
coordinates, ranges, resource locations and block/item predicates. For every
|
|
input the parse success, the exact error text and cursor, the parsed node
|
|
names and the completion suggestions at the end of the input must match.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
|
|
try:
|
|
from . import lab, test_rust_parity as base
|
|
from . import command_parse_corpus as corpus_module
|
|
except ImportError:
|
|
import lab
|
|
import test_rust_parity as base
|
|
import command_parse_corpus as corpus_module
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--prepare', action='store_true')
|
|
parser.add_argument('--permission', type=int, nargs='*', default=[4, 2, 0])
|
|
parser.add_argument('--limit', type=int, default=0, help='print at most this many mismatches per class')
|
|
parser.add_argument('--allow-gaps', action='store_true', help='report mismatches without failing')
|
|
args = parser.parse_args()
|
|
build = lab.LOCAL / 'command-parse-parity'
|
|
build.mkdir(parents=True, exist_ok=True)
|
|
report_path = build / 'report.json'
|
|
report_path.unlink(missing_ok=True)
|
|
if args.prepare:
|
|
lab.prepare()
|
|
lock, jars, libraries = base.verify_artifacts()
|
|
oracle = lab.ROOT / 'mods/horizon/src/test/java/dev/horizonlab/parity/VanillaCommandParseOracle.java'
|
|
classes = build / 'classes'
|
|
classes.mkdir(exist_ok=True)
|
|
dependency_path = os.pathsep.join(map(str, libraries))
|
|
|
|
def run(command, name):
|
|
with (build / (name + '.log')).open('w', encoding='utf-8') as log:
|
|
result = subprocess.run(list(map(str, command)), cwd=lab.ROOT, stdout=log, stderr=subprocess.STDOUT)
|
|
if result.returncode:
|
|
tail = (build / (name + '.log')).read_text(errors='replace')[-2500:]
|
|
sys.exit(f'{name} failed ({result.returncode}):\n{tail}')
|
|
|
|
run(['javac', '--release', lock['minecraft']['java_major'], '-proc:none', '-cp', str(jars['server']) + os.pathsep + dependency_path, '-d', classes, oracle], 'compile-java')
|
|
|
|
def java(*arguments):
|
|
run(['java', '-Xmx1G', '-cp', os.pathsep.join([str(classes), str(jars['server']), dependency_path]), 'dev.horizonlab.parity.VanillaCommandParseOracle', *arguments], 'java-' + arguments[0])
|
|
|
|
java('registries', str(build))
|
|
java('tree', str(build))
|
|
tree = json.loads((build / 'tree.json').read_text())
|
|
registries = json.loads((build / 'registries.json').read_text())
|
|
corpus = corpus_module.corpus(tree, registries)
|
|
(build / 'corpus.txt').write_text(''.join(line + '\n' for line in corpus), encoding='utf-8')
|
|
run(['cargo', 'build', '--release', '-p', 'horizon-client', '--example', 'client_probe'], 'compile-rust')
|
|
metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--no-deps', '--format-version', '1'], cwd=lab.ROOT, text=True))
|
|
binary = Path(metadata['target_directory']) / 'release/examples' / ('client_probe.exe' if os.name=='nt' else 'client_probe')
|
|
resources = lab.RESOURCES
|
|
total_failures = 0
|
|
summary = []
|
|
for permission in args.permission:
|
|
level_dir = build / f'perm{permission}'
|
|
level_dir.mkdir(exist_ok=True)
|
|
(level_dir / 'registries.json').write_bytes((build / 'registries.json').read_bytes())
|
|
stamp = level_dir / 'vanilla.stamp'
|
|
digest = lab.digest(build / 'corpus.txt', 'sha256') + lab.digest(oracle, 'sha256')
|
|
if not (stamp.exists() and stamp.read_text() == digest and (level_dir / 'vanilla.jsonl').exists()):
|
|
java('run', str(level_dir), str(build / 'corpus.txt'), str(permission))
|
|
stamp.write_text(digest)
|
|
run([binary, 'command_parse_fixture',resources, build / 'registries.json', build / 'corpus.txt', str(permission), level_dir / 'rust.jsonl'], f'rust{permission}')
|
|
failures = compare(level_dir / 'vanilla.jsonl', level_dir / 'rust.jsonl', args.limit)
|
|
count = sum(1 for _ in (level_dir / 'vanilla.jsonl').open())
|
|
print(f'permission {permission}: {count - failures}/{count} commands match', flush=True)
|
|
summary.append({'permission': permission, 'commands': count, 'mismatches': failures})
|
|
total_failures += failures
|
|
report = {'schema': 1, 'minecraft': lock['minecraft']['version'], 'platform': platform.platform(), 'results': summary,
|
|
'scope': 'Parse success, error text and cursor, parsed node names and end-of-input suggestions for a tree-walked corpus.'}
|
|
lab.write_json(report_path, report)
|
|
if total_failures and not args.allow_gaps:
|
|
sys.exit(f'{total_failures} command parse mismatches; see {build}')
|
|
print(f'Command parse parity report: {report_path}')
|
|
|
|
|
|
def compare(vanilla_path, rust_path, limit):
|
|
vanilla = [json.loads(line) for line in vanilla_path.open(encoding='utf-8')]
|
|
rust = [json.loads(line) for line in rust_path.open(encoding='utf-8')]
|
|
if len(vanilla) != len(rust):
|
|
raise SystemExit(f'line count differs: vanilla {len(vanilla)} rust {len(rust)}')
|
|
shown = {}
|
|
failures = 0
|
|
mismatches = []
|
|
for v, r in zip(vanilla, rust):
|
|
assert v['in'] == r['in'], (v['in'], r['in'])
|
|
kind = None
|
|
if 'crash' in v:
|
|
continue # oracle could not evaluate (missing server state)
|
|
# Node names are internal to each implementation; only success matters.
|
|
vp = {k: val for k, val in v['parse'].items() if k != 'nodes'}
|
|
rp = {k: val for k, val in r['parse'].items() if k != 'nodes'}
|
|
if vp != rp:
|
|
kind = 'parse'
|
|
elif v['sugg'] != r['sugg']:
|
|
kind = 'suggestions'
|
|
if kind:
|
|
failures += 1
|
|
mismatches.append({'kind': kind, 'in': v['in'], 'vanilla': v['parse'] if kind == 'parse' else v['sugg'], 'rust': r['parse'] if kind == 'parse' else r['sugg']})
|
|
out = vanilla_path.with_name('mismatches.jsonl')
|
|
out.write_text(''.join(json.dumps(m, ensure_ascii=False) + '\n' for m in mismatches), encoding='utf-8')
|
|
for m in mismatches:
|
|
n = shown.get(m['kind'], 0)
|
|
if limit and n >= limit:
|
|
continue
|
|
shown[m['kind']] = n + 1
|
|
print(m['kind'], repr(m['in']), '\n vanilla', json.dumps(m['vanilla'], ensure_ascii=False)[:300], '\n rust ', json.dumps(m['rust'], ensure_ascii=False)[:300])
|
|
return failures
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|