Test(Launcher): generate the runner parity fixtures

This commit is contained in:
devLucasEmiliano committed 2026-10-22 18:46:41 -03:00
1 parent 5d030c3269
commit 76be6e24c3
60 files changed
+1079

No files matched your search

+14
View File
@@ -5,3 +5,17 @@ launcher-rust/fixtures/** text eol=lf
launcher-rust/assets/i18n.json text eol=lf
# Snapshot references are images compared pixel by pixel: never touched as text.
launcher-rust/snapshots/** binary
# The runner fixtures: text outputs are LF, every input and output that is bytes is never touched as text.
launcher-rust/fixtures/runner/**/*.json text eol=lf
launcher-rust/fixtures/runner/**/*.xml text eol=lf
launcher-rust/fixtures/runner/**/*.ini text eol=lf
launcher-rust/fixtures/runner/**/*.bin binary
launcher-rust/fixtures/runner/**/*.dds binary
launcher-rust/fixtures/runner/**/*.dxt1 binary
launcher-rust/fixtures/runner/**/*.dxt5 binary
launcher-rust/fixtures/runner/**/*.png binary
launcher-rust/fixtures/runner/**/*.bmp binary
launcher-rust/fixtures/runner/**/*.WAD binary
launcher-rust/fixtures/runner/**/*.prx binary
launcher-rust/fixtures/runner/**/*.sfo binary
launcher-rust/fixtures/runner/**/*.elf binary
+1
View File
@@ -481,6 +481,7 @@ cargo test
The page snapshots under `launcher-rust/snapshots/` are compared pixel by pixel on Windows only.
`python -m unittest discover -s tests -p "test_launcher_parity_doc.py"` checks that every Python
launcher test has its row in `docs/specs/2026-10-21-rust-launcher-parity.md`.
`tools/runner_fixtures.py` generates `launcher-rust/fixtures/runner/`, the inputs and the files the Python runner's modules write over them, for the Rust runner's parity tests.
### Windows C++ tests (MSYS2 CLANG64)
@@ -0,0 +1,13 @@
{
"title_id": "CUSA01623",
"sku": "full",
"sku_source": "explicit probe setting",
"user_params": [
13,
0,
7,
4294967295
],
"mounted_addons": [],
"boot_attr": 0
}
@@ -0,0 +1,13 @@
{
"title_id": "CUSA01623",
"sku": "trial",
"sku_source": "explicit probe setting",
"user_params": [
13,
0,
7,
4294967295
],
"mounted_addons": [],
"boot_attr": 0
}
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,5 @@
{
"SPARTAN_DEFERRED_READBACK": "1",
"SPARTAN_STALE_READBACK": "1",
"SPARTAN_ASYNC_SHADERS": "1"
}
@@ -0,0 +1,10 @@
{
"last_good": {
"render_resolution": "1440p",
"engine_fps": "120",
"deferred_readback": "1",
"stale_readback": "1",
"async_shaders": "1"
},
"attempt": false
}
@@ -0,0 +1,2 @@
render_resolution=1440p
engine_fps=120
@@ -0,0 +1,5 @@
{
"SPARTAN_DEFERRED_READBACK": "1",
"SPARTAN_STALE_READBACK": "1",
"SPARTAN_ASYNC_SHADERS": "1"
}
@@ -0,0 +1,10 @@
{
"last_good": {
"render_resolution": "1440p",
"engine_fps": "120",
"deferred_readback": "1",
"stale_readback": "1",
"async_shaders": "1"
},
"attempt": false
}
@@ -0,0 +1,6 @@
render_resolution=1440p
engine_fps=120
restart_unconfirmed=0
deferred_readback=1
stale_readback=1
async_shaders=1
@@ -0,0 +1,5 @@
{
"SPARTAN_DEFERRED_READBACK": "1",
"SPARTAN_STALE_READBACK": "1",
"SPARTAN_ASYNC_SHADERS": "1"
}
@@ -0,0 +1,10 @@
{
"last_good": {
"render_resolution": "1440p",
"engine_fps": "120",
"deferred_readback": "1",
"stale_readback": "1",
"async_shaders": "1"
},
"attempt": true
}
@@ -0,0 +1,3 @@
render_resolution=4K
engine_fps=240
restart_unconfirmed=1
@@ -0,0 +1,5 @@
{
"render_resolution": "4K",
"engine_fps": "240",
"restart_unconfirmed": "1"
}
@@ -0,0 +1 @@
{}
@@ -0,0 +1,2 @@
render_resolution=1440p
engine_fps=120
Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

@@ -0,0 +1,122 @@
[
{
"offset": 0,
"length": 5,
"kind": "BLSR",
"width": 32,
"mode": "JMP",
"registers": {
"dst": 1,
"src1": 1,
"src2": 255,
"base": 255,
"index": 255
},
"mem_slot": 0,
"scale": 0,
"mem_flags": 0,
"disp": 0,
"original": "c4e270f3c9",
"relocated": ""
},
{
"offset": 5,
"length": 4,
"kind": "MOVBE_STORE",
"width": 32,
"mode": "JMP_RELOCATE",
"registers": {
"dst": 255,
"src1": 1,
"src2": 255,
"base": 0,
"index": 255
},
"mem_slot": 1,
"scale": 1,
"mem_flags": 0,
"disp": 0,
"original": "0f38f108",
"relocated": "90"
},
{
"offset": 10,
"length": 9,
"kind": "BEXTR",
"width": 32,
"mode": "JMP",
"registers": {
"dst": 0,
"src1": 255,
"src2": 1,
"base": 255,
"index": 255
},
"mem_slot": 2,
"scale": 1,
"mem_flags": 1,
"disp": 31,
"original": "c4e270f7050c000000",
"relocated": ""
},
{
"offset": 19,
"length": 5,
"kind": "LZCNT",
"width": 16,
"mode": "JMP",
"registers": {
"dst": 0,
"src1": 1,
"src2": 255,
"base": 255,
"index": 255
},
"mem_slot": 0,
"scale": 0,
"mem_flags": 0,
"disp": 0,
"original": "66f30fbdc1",
"relocated": ""
},
{
"offset": 24,
"length": 5,
"kind": "TZCNT",
"width": 64,
"mode": "JMP",
"registers": {
"dst": 0,
"src1": 1,
"src2": 255,
"base": 255,
"index": 255
},
"mem_slot": 0,
"scale": 0,
"mem_flags": 0,
"disp": 0,
"original": "f3480fbcc1",
"relocated": ""
},
{
"offset": 29,
"length": 4,
"kind": "MOVBE_STORE",
"width": 32,
"mode": "INT3",
"registers": {
"dst": 255,
"src1": 1,
"src2": 255,
"base": 0,
"index": 255
},
"mem_slot": 1,
"scale": 1,
"mem_flags": 0,
"disp": 0,
"original": "0f38f108",
"relocated": ""
}
]
@@ -0,0 +1,37 @@
{
"base": "0x10000",
"size": 4096,
"sha256": "16bcdb5c0a28f581da21dc76c9e73ed1e8ec35398b85be38352691f4ff4cba33",
"init": "0x10010",
"bindings": 1,
"tls_module_id": 2,
"fs_loads_patched": 2,
"main_tls": {
"vaddr": 3072,
"filesz": 16,
"memsz": 48,
"align": 16
},
"tls_relocations": 1,
"tls_template_bytes": 16,
"tls_memory_bytes": 48,
"imports": [
"bzQExy189ZI#q#q",
"AAAAAAAAAAA#q#q",
"DDDDDDDDDDD#q#q"
],
"relocation_counts": {
"8": 1,
"16": 1,
"1": 1,
"6": 1,
"7": 1
},
"symbol_bindings": [
{
"import_name": "bzQExy189ZI#q#q",
"address": "0x10010",
"kind": 1
}
]
}
@@ -0,0 +1,43 @@
{
"modules": [
{
"file": "libc.prx",
"base": "0x10000",
"size": 4096,
"init": "0x10010",
"tls_address": "0x10500",
"tls_memsz": 48,
"tls_filesz": 16,
"tls_module": 2,
"tls_relocations": 1,
"exports": 3,
"sha256": "16bcdb5c0a28f581da21dc76c9e73ed1e8ec35398b85be38352691f4ff4cba33"
},
{
"file": "libSceFios2.prx",
"base": "0x20000",
"size": 2048,
"init": "0x20000",
"tls_address": "0x0",
"tls_memsz": 0,
"tls_filesz": 0,
"tls_module": 0,
"tls_relocations": 0,
"exports": 1,
"sha256": "a7ae78118c4fcb8dd27129c2b8059fbcf2e0d685b4ad8b40038fa43f4d93e4f2"
}
],
"bindings": 2,
"imports": 4,
"fs_loads_patched": 4,
"main_tls": {
"vaddr": 3072,
"filesz": 16,
"memsz": 48,
"align": 16
},
"unresolved_imports": [
"AAAAAAAAAAA#libc",
"DDDDDDDDDDD#libc"
]
}
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
{"sha256": "ffed585c6a2b513546eba7f283c45ad27382a8e31cf54fe03b4e1c0ca63195d5", "applied": ["Found", "Call", "Data"], "skipped": {"Missing": "signature not found"}}
@@ -0,0 +1,44 @@
<?xml version='1.0' encoding='utf-8'?>
<Patch>
<TitleID>
<ID>CUSA01623</ID>
</TitleID>
<Metadata Name="Found" AppVer="01.02">
<PatchList>
<Line Type="bytes" Address="0x401004" Value="9090" Pattern="11 22 ?? 44 55 66" At="1" />
</PatchList>
</Metadata>
<Metadata Name="Call" AppVer="01.02">
<PatchList>
<Line Type="bytes" Address="0x401010" Value="e8{rel:there:0}" Pattern="aa bb cc dd ee" At="0" />
</PatchList>
<Target Name="there" Address="0x401100" Pattern="e9 ?? ?? ?? ?? 77 88" At="1" Ref="1" />
</Metadata>
<Metadata Name="Missing" AppVer="01.02">
<PatchList>
<Line Type="bytes" Address="0x401004" Value="9090" Pattern="11 22 ?? 44 55 66" At="1" />
<Line Type="bytes" Address="0x401020" Value="90" Pattern="de ad be ef" At="0" />
</PatchList>
</Metadata>
<Metadata Name="Data" AppVer="01.02">
<PatchList>
<Line Type="bytes32" Address="0x402000" Target="variable" Value="0x2" />
</PatchList>
<Target Name="variable" Address="0x402000" Pattern="44 89 35 ?? ?? ?? ?? c6" At="3" Ref="1" />
</Metadata>
<Metadata Name="Resolution Patch - 3440x1440" Note="Requires setting DMEM to 4608 MB or more." AppVer="01.02" AppElf="eboot.bin" isEnabled="false">
<PatchList>
<Line Type="bytes" Address="0x00401000" Value="9090" />
</PatchList>
</Metadata>
<Metadata Name="Resolution Patch - 2560x1080" AppVer="01.02" AppElf="eboot.bin" isEnabled="false">
<PatchList>
<Line Type="bytes" Address="0x00401000" Value="9090" />
</PatchList>
</Metadata>
<Metadata Name="Skip Any Video With X Button" Note="replaced by the mod" AppVer="01.02" AppElf="eboot.bin" isEnabled="true">
<PatchList>
<Line Type="bytes" Address="0x00401002" Value="90" />
</PatchList>
</Metadata>
</Patch>
@@ -0,0 +1 @@
synthetic eboot.bin of the patches case
@@ -0,0 +1,13 @@
<?xml version="1.0"?>
<Patch>
<TitleID><ID>CUSA01623</ID></TitleID>
<Metadata Name="Resolution Patch - 3440x1440" Note="Requires setting DMEM to 4608 MB or more." AppVer="01.02" AppElf="eboot.bin" isEnabled="false">
<PatchList><Line Type="bytes" Address="0x00401000" Value="9090"/></PatchList>
</Metadata>
<Metadata Name="Resolution Patch - 2560x1080" AppVer="01.02" AppElf="eboot.bin" isEnabled="false">
<PatchList><Line Type="bytes" Address="0x00401000" Value="9090"/></PatchList>
</Metadata>
<Metadata Name="Skip Any Video With X Button" Note="replaced by the mod" AppVer="01.02" AppElf="eboot.bin" isEnabled="true">
<PatchList><Line Type="bytes" Address="0x00401002" Value="90"/></PatchList>
</Metadata>
</Patch>
@@ -0,0 +1,21 @@
<?xml version="1.0"?>
<Patch>
<TitleID><ID>CUSA01623</ID></TitleID>
<Metadata Name="Found" AppVer="01.02">
<PatchList><Line Type="bytes" Address="0x401004" Value="9090" Pattern="11 22 ?? 44 55 66" At="1"/></PatchList>
</Metadata>
<Metadata Name="Call" AppVer="01.02">
<PatchList><Line Type="bytes" Address="0x401010" Value="e8{rel:there:0}" Pattern="aa bb cc dd ee" At="0"/></PatchList>
<Target Name="there" Address="0x401100" Pattern="e9 ?? ?? ?? ?? 77 88" At="1" Ref="1"/>
</Metadata>
<Metadata Name="Missing" AppVer="01.02">
<PatchList>
<Line Type="bytes" Address="0x401004" Value="9090" Pattern="11 22 ?? 44 55 66" At="1"/>
<Line Type="bytes" Address="0x401020" Value="90" Pattern="de ad be ef" At="0"/>
</PatchList>
</Metadata>
<Metadata Name="Data" AppVer="01.02">
<PatchList><Line Type="bytes32" Address="0x402000" Target="variable" Value="0x2"/></PatchList>
<Target Name="variable" Address="0x402000" Pattern="44 89 35 ?? ?? ?? ?? c6" At="3" Ref="1"/>
</Metadata>
</Patch>
@@ -0,0 +1,142 @@
{
"source_sha256": "138b11e351ce338943e47f76805cfba5380996355ea098c97be4fd586002d96c",
"source_bytes": 5400,
"sfo": {
"APP_VER": "01.02",
"CONTENT_ID": "EP9000-CUSA01623_00-GOW3REMASTERED00",
"TITLE": "ゴッド・オブ・ウォーIII リマスタード",
"TITLE_ID": "CUSA01623",
"USER_DEFINED_PARAM_1": 13,
"USER_DEFINED_PARAM_2": 0,
"USER_DEFINED_PARAM_3": 7,
"USER_DEFINED_PARAM_4": 4294967295
},
"entry": "0x10",
"image_bytes": 4096,
"program_headers": [
{
"type": 1,
"flags": 5,
"offset": 4096,
"vaddr": 0,
"paddr": 0,
"filesz": 2048,
"memsz": 2048,
"align": 16384
},
{
"type": 1,
"flags": 6,
"offset": 6144,
"vaddr": 2048,
"paddr": 2048,
"filesz": 2048,
"memsz": 2048,
"align": 16384
},
{
"type": 2,
"flags": 4,
"offset": 8192,
"vaddr": 0,
"paddr": 0,
"filesz": 192,
"memsz": 0,
"align": 8
},
{
"type": 1627389952,
"flags": 4,
"offset": 8448,
"vaddr": 0,
"paddr": 0,
"filesz": 344,
"memsz": 0,
"align": 8
},
{
"type": 1627389953,
"flags": 4,
"offset": 8960,
"vaddr": 128,
"paddr": 128,
"filesz": 0,
"memsz": 64,
"align": 8
},
{
"type": 7,
"flags": 4,
"offset": 7168,
"vaddr": 3072,
"paddr": 3072,
"filesz": 16,
"memsz": 48,
"align": 16
}
],
"self_segments": [
{
"flags": "0x800",
"offset": 768,
"size": 2048
},
{
"flags": "0x100800",
"offset": 2816,
"size": 2048
},
{
"flags": "0x200800",
"offset": 4864,
"size": 192
},
{
"flags": "0x300800",
"offset": 5056,
"size": 344
}
],
"unavailable_metadata_headers": [],
"needed": [
"libc.prx"
],
"relocation_counts": {
"8": 1,
"6": 2,
"1": 1,
"7": 2
},
"import_count": 2,
"imports": [
"bzQExy189ZI#q#q",
"AAAAAAAAAAA#q#q"
],
"import_name_hints": {
"bzQExy189ZI#q#q": "_init_env"
},
"libc_evidence": {
"sha256": "16bcdb5c0a28f581da21dc76c9e73ed1e8ec35398b85be38352691f4ff4cba33",
"init_env_is_ret": true,
"symbol": "bzQExy189ZI#C#A",
"address": "0x10",
"size": 1,
"bytes": "c3",
"ios_base_init": {
"symbol": "sqWytnhYdEg#C#A",
"address": "0x20",
"size": 3,
"code_sha256": "251447ee91a9067dcd6ab96703133f617565974cd6c4819021760c4688c91abf",
"name": "_ZNSt8ios_base4InitC1Ev"
}
},
"bundled_modules": [
"libc.prx"
],
"resources": {
"PART1.WAD": 1,
"sound": 1
},
"resource_bytes": 132,
"status": "Prepared only; execution and Vulkan are tested separately."
}
+58
View File
@@ -0,0 +1,58 @@
"""The Rust runner's parity fixtures: what tools/runner_fixtures.py wrote from the Python modules."""
import unittest
from paths import ROOT
FIXTURES = ROOT / 'launcher-rust/fixtures/runner'
DXT_NAMES = ('gradient', 'edges', 'alpha', 'noise', 'flat')
# case -> (input files, expected files), relative to the case folder's input/ and expected/.
CASES = {
'prepare': (['eboot.bin', 'sce_module/libc.prx', 'sce_sys/param.sfo', 'dvdroot_ps4/PART1.WAD'],
['boot.bin', 'eboot.elf', 'entry.bin', 'analysis.json']),
'link': (['boot.bin', 'eboot.bin', 'sce_module/libc.prx', 'sce_module/libSceFios2.prx'],
['boot-libc.bin', 'libc-link.json', 'boot-linked.bin', 'link.json']),
'isa': (['boot-linked.bin'], ['isa-rewrite.bin', 'sites.json']),
'content': (['param.sfo'], ['full/content.bin', 'full/content-profile.json',
'trial/content.bin', 'trial/content-profile.json']),
'patches': (['eboot.elf', 'eboot.bin', 'param.sfo', 'patches.xml', 'mod.xml'],
['patches-merged.xml', 'patches.bin', 'patch_status.json']),
'graphics': (['spartanport.ini', 'graphics_state.json', 'edit.json'],
[f'{state}/{name}' for state in ('good', 'tried', 'reverted')
for name in ('spartanport.ini', 'graphics_state.json', 'environment.json')]),
'dxt': ([f'{name}.png' for name in DXT_NAMES],
[f'{name}.{fmt}' for name in DXT_NAMES for fmt in ('dxt1', 'dxt5')]),
'icon': ([], ['window_icon.bmp']),
}
class RunnerFixtures(unittest.TestCase):
def test_every_case_has_its_files(self):
for case, (inputs, expected) in CASES.items():
for folder, names in (('input', inputs), ('expected', expected)):
for name in names:
path = FIXTURES / case / folder / name
self.assertTrue(path.is_file(), f'{case}/{folder}/{name}')
def test_dxt_strips_are_whole_blocks(self):
for name in DXT_NAMES:
# 64 x 64 pixels are 16 x 16 blocks of 8 (DXT1) and 16 (DXT5) bytes.
dxt1 = (FIXTURES / 'dxt/expected' / f'{name}.dxt1').read_bytes()
dxt5 = (FIXTURES / 'dxt/expected' / f'{name}.dxt5').read_bytes()
self.assertEqual((len(dxt1) % 8, len(dxt1)), (0, 16 * 16 * 8), name)
self.assertEqual((len(dxt5) % 16, len(dxt5)), (0, 16 * 16 * 16), name)
def test_content_profile_is_twenty_eight_bytes(self):
for sku in ('full', 'trial'):
data = (FIXTURES / 'content/expected' / sku / 'content.bin').read_bytes()
self.assertEqual(len(data), 28, sku)
self.assertEqual(data[:8], b'G3CONT01', sku)
def test_isa_table_of_the_c_loader_fixture_is_reproduced(self):
# The isa input is a G3PROBE5 image holding isa_fixture.bin's code: its table is that file.
table = (FIXTURES / 'isa/expected/isa-rewrite.bin').read_bytes()
self.assertEqual(table, (ROOT / 'tests/data/isa_fixture.bin').read_bytes())
if __name__ == '__main__':
unittest.main()
+492
View File
@@ -0,0 +1,492 @@
"""One-off: runs the Python runner's own modules (scripts/*.py) over synthetic inputs and stores what they
write in launcher-rust/fixtures/runner/<case>/{input,expected}/, for the Rust runner's byte-for-byte parity tests.
Run from anywhere: python tools/runner_fixtures.py (needs Pillow and iced_x86)
Every case folder holds input/ (what the module reads) and expected/ (what it wrote). A game folder keeps its
own layout inside input/ (eboot.bin, sce_module/*.prx, sce_sys/param.sfo, dvdroot_ps4/...), so a test can
copy input/ as it is. Text outputs are written with LF; no expected file holds the temporary folder (the
generator stops when one does).
prepare input: a synthetic eboot.bin SELF (two PT_LOAD, dynamic, library blob with imports and the four
relocation kinds, TLS) with sce_module/libc.prx (its _init_env is RET), sce_sys/param.sfo and a
dvdroot_ps4 folder. expected: boot.bin, eboot.elf, entry.bin, analysis.json (scripts/prepare.py).
link input: the same eboot.bin, libc.prx and libSceFios2.prx, and boot.bin from tests/test_probe.package
(a freshly prepared G3PROBE2 image with the eboot's imports). expected: boot-libc.bin and
libc-link.json (link_libc.py), boot-linked.bin and link.json (link_modules.py).
isa input: boot-linked.bin, a G3PROBE5 image (tests/test_isa_rewrite.boot_linked) whose one executable
segment is isa_rewrite.fixture_code(). expected: isa-rewrite.bin (equal to tests/data/isa_fixture.bin)
and sites.json, one entry per site: offset, length, kind and mode names, width, registers
(dst, src1, src2, base, index; 255 is none), mem_slot, scale, mem_flags, disp, original and
relocated bytes as hex.
content input: param.sfo with the four user-defined parameters. expected: full/ and trial/ with
content.bin and content-profile.json (content_profile.py).
patches input: eboot.elf holding the signatures of tests/test_patches.SignatureTests in its one PT_LOAD,
eboot.bin and param.sfo (CUSA01623, 01.02: the game folder patches.py looks the profile up in),
patches.xml (SignatureTests.XML with a TitleID so the mod file merges in) and mod.xml
(ModPatchTests.MOD). patches.py main() runs with --mod-patches mod.xml --extra "Found;Call;Missing;Data"
and SPARTAN_PATCHES_ONLY=1. expected: patches-merged.xml, patches.bin, patch_status.json (Missing is
skipped: its second signature is absent).
graphics input: spartanport.ini, graphics_state.json ({}) and edit.json, the keys written between the
launches. graphics_options.prepare runs three times, as in
test_unconfirmed_restart_is_tried_once_then_reverted; expected/<good|tried|reverted>/ holds the ini,
the state file and environment.json after each launch.
dxt input: gradient, edges, alpha, noise and flat, 64 x 64 RGBA PNGs (noise from a fixed seed).
expected: <name>.dxt1 and <name>.dxt5, the 4-row strips of glyphs.encode_region concatenated with
each DDS header (128 bytes) stripped (RGB for DXT1, RGBA for DXT5).
icon expected: window_icon.bmp, launcher/spartanport.png as run.py makes it (RGBA, 64 x 64 LANCZOS, BMP)."""
import contextlib
import io
import json
import os
import random
import shutil
import struct
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
FIXTURES = ROOT / 'launcher-rust' / 'fixtures' / 'runner'
WORK = Path(tempfile.mkdtemp(prefix='runner_fixtures_'))
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
import content_profile # noqa: E402
import glyphs # noqa: E402
import graphics_options # noqa: E402
import isa_rewrite # noqa: E402
import link_libc # noqa: E402
import link_modules # noqa: E402
import patches # noqa: E402
import prepare as prepare_module # noqa: E402
from PIL import Image # noqa: E402
from test_isa_rewrite import boot_linked # noqa: E402
from test_patches import ModPatchTests, SignatureTests # noqa: E402
from test_probe import package # noqa: E402
FS_LOAD = bytes.fromhex('64488b042500000000') # mov rax, fs:[0]: what the linkers rewrite to GS
PARAMS = {'USER_DEFINED_PARAM_1': 13, 'USER_DEFINED_PARAM_2': 0, 'USER_DEFINED_PARAM_3': 7,
'USER_DEFINED_PARAM_4': 0xffffffff}
EDIT = {'render_resolution': '4K', 'engine_fps': '240', 'restart_unconfirmed': '1'}
# -- Synthetic files ----------------------------------------------------------------------------------------
def sfo_bytes(values):
"""A param.sfo (PSF 1.1): strings (0x204) and unsigned integers (0x404), keys sorted."""
keys, data, entries = b'', b'', []
for key, value in sorted(values.items()):
if isinstance(value, int):
fmt, raw, capacity = 0x404, struct.pack('<I', value), 4
else:
raw = value.encode('utf-8') + b'\0'
fmt, capacity = 0x204, (len(raw) + 3) // 4 * 4
entries.append((len(keys), fmt, len(raw), capacity, len(data)))
keys += key.encode() + b'\0'
data += raw.ljust(capacity, b'\0')
key_table = 20 + 16 * len(entries)
data_table = (key_table + len(keys) + 3) // 4 * 4
head = struct.pack('<4sIIII', b'\0PSF', 0x101, key_table, data_table, len(entries))
return (head + b''.join(struct.pack('<HHIII', *e) for e in entries)
+ keys.ljust(data_table - key_table, b'\0') + data)
class Strings:
"""A string table: an empty string at offset 0, then each text once."""
def __init__(self):
self.data, self.at = bytearray(b'\0'), {}
def add(self, text):
if text not in self.at:
self.at[text] = len(self.data)
self.data += text.encode() + b'\0'
return self.at[text]
def symbol(strings, name, info, section=0, value=0, size=0):
return struct.pack('<IBBHQQ', strings.add(name) if name else 0, info, 0, section, value, size)
def relocation(target, kind, sym=0, addend=0):
return struct.pack('<QQq', target, (sym << 32) | kind, addend)
def identifier(ident, version, name_offset):
"""A library or module dynamic tag value: id in the top 16 bits, version, name offset."""
return (ident << 48) | (version << 32) | name_offset
def library_blob(strings, symbols, rela, plt):
"""The 0x61000000 segment (strings, symbols, relocation tables) and the dynamic tags that locate them."""
tags, blob = {}, b''
for names, content in ((('str', 0x61000035, 0x61000037), bytes(strings.data)),
(('sym', 0x61000039, 0x6100003f), b''.join(symbols)),
(('rela', 0x61000029, 0x6100002d), b''.join(rela)),
(('plt', 0x6100002f, 0x61000031), b''.join(plt))):
tags[names[1]], tags[names[2]] = len(blob), len(content)
blob += content.ljust((len(content) + 7) // 8 * 8, b'\0')
return blob, [(tag, value) for tag, value in tags.items()]
def dynamic_bytes(tags):
return b''.join(struct.pack('<QQ', *t) for t in tags + [(0, 0)])
def make_self(segments, e_type, entry=0):
"""A SELF around an ELF with these program headers. A segment is a dict: type, flags, vaddr, memsz, align,
data (its file bytes), covered (stored in the SELF; the others are metadata the unpacker reports as
unavailable) and, for a segment inside another one's bytes, inside (its index) and at (offset in it)."""
cursor = 0x1000
for s in segments:
if 'inside' in s:
s['offset'] = segments[s['inside']]['offset'] + s['at']
else:
s['offset'] = cursor
cursor = (cursor + len(s['data']) + 0xff) // 0x100 * 0x100
phdrs = b''.join(struct.pack('<IIQQQQQQ', s['type'], s['flags'], s['offset'], s['vaddr'], s['vaddr'],
len(s['data']), s['memsz'], s['align']) for s in segments)
elf_head = struct.pack('<16sHHIQQQIHHHHHH', b'\x7fELF\x02\x01\x01\x09', e_type, 62, 1, entry, 64, 0, 0,
64, 56, len(segments), 0, 0, 0) + phdrs
stored = [(i, s) for i, s in enumerate(segments) if s.get('covered') and s['data']]
base = 32 + 32 * len(stored)
position = (base + len(elf_head) + 0xff) // 0x100 * 0x100
table, payload = b'', b''
for i, s in stored:
table += struct.pack('<QQQQ', 0x800 | (i << 20), position + len(payload), len(s['data']), len(s['data']))
payload += s['data']
head = (b'O\x15=\x1d' + bytes(20) + struct.pack('<H', len(stored))).ljust(32, b'\0')
return (head + table + elf_head).ljust(position, b'\0') + payload
def load(vaddr, flags, data, memsz, **extra):
return dict(type=1, flags=flags, vaddr=vaddr, memsz=memsz, align=0x4000, data=data, covered=True, **extra)
def code_page(size, pieces):
page = bytearray(size)
for at, code in pieces:
page[at:at + len(code)] = code
return bytes(page)
def eboot_parts():
"""(SELF bytes, import names, code): the synthetic eboot of the prepare and link cases. It imports _init_env
(a function the libc exports), an unknown function and data object, and defines one symbol of its own."""
nid = prepare_module.nid
strings = Strings()
needed, libc = strings.add('libc.prx'), strings.add('libc')
names = [nid('_init_env') + '#q#q', 'AAAAAAAAAAA#q#q']
symbols = [symbol(strings, '', 0),
symbol(strings, names[0], 0x12), # import, function
symbol(strings, names[1], 0x11), # import, object
symbol(strings, 'BBBBBBBBBBB#q#q', 0x12, 1, 0x40, 8), # defined here
symbol(strings, 'CCCCCCCCCCC#q#q', 0x11, 0xfff1, 0x1234)] # absolute
rela = [relocation(0x800, 8, 0, 0x40), relocation(0x808, 6, 3, 0x10), relocation(0x810, 1, 4, 4)]
plt = [relocation(0x818, 7, 1), relocation(0x820, 6, 2), relocation(0x828, 7, 1)]
blob, tags = library_blob(strings, symbols, rela, plt)
tags = [(1, needed), (0x61000013, identifier(42, 1, libc)), (0x6100000d, identifier(42, 1, libc))] + tags
code = code_page(0x800, [(0x10, bytes.fromhex('31c0c3')), (0x20, FS_LOAD + b'\x90'), (0x40, FS_LOAD + b'\xc3')])
data = bytes(i & 255 for i in range(0x800))
segments = [load(0, 5, code, 0x800), load(0x800, 6, data, 0x800),
dict(type=2, flags=4, vaddr=0, memsz=0, align=8, data=dynamic_bytes(tags), covered=True),
dict(type=0x61000000, flags=4, vaddr=0, memsz=0, align=8, data=blob, covered=True),
dict(type=0x61000001, flags=4, vaddr=0x80, memsz=0x40, align=8, data=b''),
dict(type=7, flags=4, vaddr=0xc00, memsz=0x30, align=16, data=data[0x400:0x410],
inside=1, at=0x400)]
return make_self(segments, 0xfe10, 0x10), names, code
def libc_self():
"""A libc.prx: _init_env is RET, one more function, one object, a TLS template, base-relative, TLS, import
and defined-symbol relocations and an init function (dynamic tag 12)."""
nid = prepare_module.nid
strings = Strings()
name = strings.add('libc')
symbols = [symbol(strings, '', 3),
symbol(strings, nid('_init_env') + '#C#A', 0x12, 1, 0x10, 1),
symbol(strings, 'DDDDDDDDDDD#C#A', 0x12),
symbol(strings, nid('_ZNSt8ios_base4InitC1Ev') + '#C#A', 0x12, 1, 0x20, 3),
symbol(strings, 'EEEEEEEEEEE#C#A', 0x11, 1, 0x600, 8)]
rela = [relocation(0x400, 8, 0, 0x10), relocation(0x408, 16), relocation(0x410, 1, 1), relocation(0x418, 6, 2)]
plt = [relocation(0x420, 7, 2)]
blob, tags = library_blob(strings, symbols, rela, plt)
tags = [(12, 0x10), (0x61000013, identifier(2, 1, name)), (0x6100000d, identifier(0, 1, name))] + tags
code = code_page(0x800, [(0x10, b'\xc3'), (0x20, bytes.fromhex('31c0c3')), (0x30, FS_LOAD + b'\xc3')])
segments = [load(0, 5, code, 0x1000),
dict(type=2, flags=4, vaddr=0, memsz=0, align=8, data=dynamic_bytes(tags), covered=True),
dict(type=0x61000000, flags=4, vaddr=0, memsz=0, align=8, data=blob, covered=True),
dict(type=7, flags=4, vaddr=0x500, memsz=0x30, align=16, data=bytes(range(16)),
inside=0, at=0x500)]
return make_self(segments, 0xfe18)
def fios_self():
"""A libSceFios2.prx: one export, a weak undefined module_start, an import served by libc's same-NID export
(library libSceLibcInternal), a defined-symbol and a base-relative relocation; no TLS."""
strings = Strings()
fios, internal = strings.add('libSceFios2'), strings.add('libSceLibcInternal')
symbols = [symbol(strings, '', 0),
symbol(strings, 'Ov1r5aKFXlE#D#B', 0x12, 1, 0x10, 1),
symbol(strings, 'module_start', 0x20),
symbol(strings, prepare_module.nid('_init_env') + '#E#F', 0x12)]
rela = [relocation(0x200, 8, 0, 0x20), relocation(0x208, 1, 2), relocation(0x210, 7, 3), relocation(0x218, 6, 1)]
blob, tags = library_blob(strings, symbols, rela, [])
tags = [(0x61000013, identifier(3, 1, fios)), (0x61000013, identifier(4, 1, internal)),
(0x6100000d, identifier(1, 1, fios)), (0x6100000d, identifier(5, 1, internal))] + tags
code = code_page(0x400, [(0x10, b'\xc3'), (0x30, FS_LOAD + b'\xc3')])
segments = [load(0, 5, code, 0x800),
dict(type=2, flags=4, vaddr=0, memsz=0, align=8, data=dynamic_bytes(tags), covered=True),
dict(type=0x61000000, flags=4, vaddr=0, memsz=0, align=8, data=blob, covered=True)]
return make_self(segments, 0xfe18)
# -- Helpers ------------------------------------------------------------------------------------------------
def put(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
return path
def text(value):
return json.dumps(value, indent=2, ensure_ascii=False) + '\n'
def lf(path):
"""A text file as the module wrote it, with LF: Windows text-mode writes give CRLF, the Rust runner writes \\n."""
return path.read_bytes().replace(b'\r\n', b'\n')
def quiet(function, *args):
"""function(*args) without its progress lines on the console."""
with contextlib.redirect_stdout(io.StringIO()):
return function(*args)
def case_folder(name):
folder = FIXTURES / name
shutil.rmtree(folder, ignore_errors=True)
return folder / 'input', folder / 'expected'
def stage(game, files):
"""Writes {relative path: bytes} under game."""
for relative, data in files.items():
put(game / relative, data)
return game
# -- Cases --------------------------------------------------------------------------------------------------
EBOOT, IMPORT_NAMES, EBOOT_CODE = eboot_parts()
LIBC, FIOS = libc_self(), fios_self()
PARAM_SFO = sfo_bytes({'TITLE_ID': 'CUSA01623', 'TITLE': 'ゴッド・オブ・ウォーIII リマスタード', 'APP_VER': '01.02',
'CONTENT_ID': 'EP9000-CUSA01623_00-GOW3REMASTERED00', **PARAMS})
GAME_FILES = {'eboot.bin': EBOOT, 'sce_module/libc.prx': LIBC, 'sce_sys/param.sfo': PARAM_SFO,
'dvdroot_ps4/PART1.WAD': b'WAD3' + bytes(range(28)), 'dvdroot_ps4/sound/music.bin': bytes(100)}
def case_prepare():
inputs, expected = case_folder('prepare')
stage(inputs, GAME_FILES)
out = WORK / 'prepare-out'
quiet(prepare_module.prepare, inputs, out)
for name in ('boot.bin', 'eboot.elf', 'entry.bin'):
put(expected / name, (out / name).read_bytes())
put(expected / 'analysis.json', lf(out / 'analysis.json'))
return out
def case_link(prepared):
inputs, expected = case_folder('link')
stage(inputs, {'eboot.bin': EBOOT, 'sce_module/libc.prx': LIBC, 'sce_module/libSceFios2.prx': FIOS})
relocs = [(0x800, 0, 0x40, 0), (0x818, 1, 0, 0), (0x820, 2, 1, 0)]
boot = package(EBOOT_CODE, relocs, IMPORT_NAMES, capabilities=1)
put(inputs / 'boot.bin', boot)
for linker, outputs in ((link_libc, ('boot-libc.bin', 'libc-link.json')),
(link_modules, ('boot-linked.bin', 'link.json'))):
out = WORK / f'{linker.__name__}-out'
put(out / 'boot.bin', boot)
quiet(linker.link, inputs, out)
for name in outputs:
put(expected / name, lf(out / name) if name.endswith('.json') else (out / name).read_bytes())
def site_entry(site):
kinds = {value: name[5:] for name, value in vars(isa_rewrite).items() if name.startswith('KIND_')}
modes = {value: name[5:] for name, value in vars(isa_rewrite).items() if name.startswith('MODE_')}
return dict(offset=site.offset, length=site.length, kind=kinds[site.kind], width=site.width,
mode=modes[site.mode],
registers=dict(dst=site.dst, src1=site.src1, src2=site.src2, base=site.base, index=site.index),
mem_slot=site.mem_slot, scale=site.scale, mem_flags=site.mem_flags, disp=site.disp,
original=site.original.hex(), relocated=site.relocated.hex())
def case_isa():
inputs, expected = case_folder('isa')
code = isa_rewrite.fixture_code()
boot = put(inputs / 'boot-linked.bin', boot_linked([(code, len(code), 5)]))
out = WORK / 'isa-out'
out.mkdir()
if quiet(isa_rewrite.main, [str(boot), '--out', str(out)]) != 0:
raise SystemExit('isa: the rewrite table could not be made')
table = (out / 'isa-rewrite.bin').read_bytes()
if table != (ROOT / 'tests/data/isa_fixture.bin').read_bytes():
raise SystemExit('isa: the table differs from tests/data/isa_fixture.bin')
put(expected / 'isa-rewrite.bin', table)
put(expected / 'sites.json', text([site_entry(s) for s in isa_rewrite.scan_segment(code, 0)]).encode())
def case_content():
inputs, expected = case_folder('content')
sfo = put(inputs / 'param.sfo', sfo_bytes({'TITLE_ID': 'CUSA01623', 'APP_VER': '01.02', **PARAMS}))
game = stage(WORK / 'content-game', {'sce_sys/param.sfo': sfo.read_bytes()})
for sku in ('full', 'trial'):
out = WORK / f'content-{sku}'
quiet(content_profile.prepare, game, out, sku)
put(expected / sku / 'content.bin', (out / 'content.bin').read_bytes())
put(expected / sku / 'content-profile.json', lf(out / 'content-profile.json'))
def case_patches():
inputs, expected = case_folder('patches')
tests = SignatureTests('test_sites_found_anywhere_and_rel32_recomputed')
tests.setUp() # the image with the signatures planted; its temporary folder goes at once
image = tests.image
tests.doCleanups()
# One PT_LOAD at file offset 0x100, virtual address 0, as patches.eboot_segments/eboot_image read it.
head = struct.pack('<16sHHIQQQIHHHHHH', b'\x7fELF\x02\x01\x01\x09', 2, 62, 1, 0, 64, 0, 0, 64, 56, 1, 0, 0, 0)
head += struct.pack('<IIQQQQQQ', 1, 5, 0x100, 0, 0, len(image), len(image), 0x1000)
put(inputs / 'eboot.elf', head.ljust(0x100, b'\0') + image)
put(inputs / 'eboot.bin', b'synthetic eboot.bin of the patches case\n')
put(inputs / 'param.sfo', sfo_bytes({'TITLE_ID': 'CUSA01623', 'APP_VER': '01.02', 'TITLE': 'God of War III'}))
xml = SignatureTests.XML.replace('<Patch>', '<Patch>\n <TitleID><ID>CUSA01623</ID></TitleID>', 1)
put(inputs / 'patches.xml', xml.encode())
put(inputs / 'mod.xml', ModPatchTests.MOD.encode())
game = stage(WORK / 'patches-game', {'eboot.bin': (inputs / 'eboot.bin').read_bytes(),
'sce_sys/param.sfo': (inputs / 'param.sfo').read_bytes()})
out = WORK / 'patches-out'
put(out / 'eboot.elf', (inputs / 'eboot.elf').read_bytes())
saved = patches.GAMES, sys.argv, os.environ.get('SPARTAN_PATCHES_ONLY')
patches.GAMES = [({'CUSA01623'}, WORK / 'patches.xml', '01.02', 'God of War III')]
put(WORK / 'patches.xml', xml.encode())
os.environ['SPARTAN_PATCHES_ONLY'] = '1'
sys.argv = ['patches.py', '--out', str(out), '--game-dir', str(game), '--mod-patches', str(inputs / 'mod.xml'),
'--extra', 'Found;Call;Missing;Data']
try:
quiet(patches.main)
finally:
patches.GAMES, sys.argv = saved[0], saved[1]
if saved[2] is None:
del os.environ['SPARTAN_PATCHES_ONLY']
else:
os.environ['SPARTAN_PATCHES_ONLY'] = saved[2]
put(expected / 'patches-merged.xml', lf(out / 'patches-merged.xml'))
put(expected / 'patches.bin', (out / 'patches.bin').read_bytes())
put(expected / 'patch_status.json', lf(out / 'patch_status.json'))
def case_graphics():
inputs, expected = case_folder('graphics')
put(inputs / 'spartanport.ini', b'render_resolution=1440p\nengine_fps=120\n')
put(inputs / 'graphics_state.json', b'{}\n')
put(inputs / 'edit.json', text(EDIT).encode())
work = WORK / 'graphics'
work.mkdir()
ini, state = work / 'spartanport.ini', work / 'graphics_state.json'
shutil.copyfile(inputs / 'spartanport.ini', ini)
shutil.copyfile(inputs / 'graphics_state.json', state)
for step in ('good', 'tried', 'reverted'):
if step == 'tried':
graphics_options.write_keys(ini, EDIT) # the in-game menu asking for a restart
options, _ = graphics_options.prepare(ini, state)
put(expected / step / 'spartanport.ini', lf(ini))
put(expected / step / 'graphics_state.json', lf(state))
put(expected / step / 'environment.json', text(graphics_options.environment(options)).encode())
def test_images():
"""The five 64 x 64 RGBA test images."""
size = 64
noise = random.Random(20260101)
drawings = {
'gradient': lambda x, y: (x * 4, y * 4, (x + y) * 2, 255),
'edges': lambda x, y: ((255, 255, 255, 255) if ((x + 1) // 5 + (y + 3) // 7 + (x == y) * 2) % 2 == 0
else (20, 30, 40, 255)),
'alpha': lambda x, y: (230, 90 + y * 2, 40, x * 4),
'noise': lambda x, y: tuple(noise.randrange(256) for _ in range(4)),
'flat': lambda x, y: (200, 100, 50, 255),
}
images = {}
for name, draw in drawings.items():
image = Image.new('RGBA', (size, size))
image.putdata([draw(x, y) for y in range(size) for x in range(size)])
images[name] = image
return images
def strips(image, fourcc):
"""glyphs.encode_region's 4-row strips of the whole picture, DDS headers stripped, concatenated."""
image = image.convert('RGBA' if fourcc == 'DXT5' else 'RGB')
width, height = image.size
encoded = b''
for y in range(0, height, 4):
strip = io.BytesIO()
image.crop((0, y, width, min(y + 4, height))).save(strip, 'DDS', pixel_format=fourcc)
encoded += strip.getvalue()[128:]
return encoded
def encode_region_blocks(image, fourcc):
"""The same bytes through glyphs.encode_region itself, over a texture holding just the top level."""
block = 16 if fourcc == 'DXT5' else 8
width, height = image.size
data = bytearray(128 + (width // 4) * (height // 4) * block)
struct.pack_into('<II', data, 12, height, width)
glyphs.encode_region(data, 0, len(data), image, fourcc, (0, 0, width, height))
return bytes(data[128:])
def case_dxt():
inputs, expected = case_folder('dxt')
for name, image in test_images().items():
inputs.mkdir(parents=True, exist_ok=True)
image.save(inputs / f'{name}.png')
for fourcc in ('DXT1', 'DXT5'):
blocks = strips(image, fourcc)
if blocks != encode_region_blocks(image, fourcc):
raise SystemExit(f'dxt: {name} {fourcc} differs from glyphs.encode_region')
put(expected / f'{name}.{fourcc.lower()}', blocks)
def case_icon():
_, expected = case_folder('icon')
expected.mkdir(parents=True)
with Image.open(ROOT / 'launcher' / 'spartanport.png') as icon: # run.py: the window icon
icon.convert('RGBA').resize((64, 64), Image.LANCZOS).save(expected / 'window_icon.bmp')
def check_no_temp_folder():
for path in FIXTURES.rglob('*'):
if path.is_file():
data = path.read_bytes()
for spelling in {str(WORK), str(WORK).replace('\\', '/'), WORK.name}:
if spelling.encode() in data:
raise SystemExit(f'{path.relative_to(ROOT)} holds the temporary folder {spelling}')
if __name__ == '__main__':
try:
FIXTURES.mkdir(parents=True, exist_ok=True)
prepared = case_prepare()
for case in (lambda: case_link(prepared), case_isa, case_content, case_patches, case_graphics, case_dxt,
case_icon):
case()
check_no_temp_folder()
for folder in sorted(p for p in FIXTURES.iterdir() if p.is_dir()):
print('wrote', folder.relative_to(ROOT))
finally:
shutil.rmtree(WORK, ignore_errors=True)