mirror of
https://github.com/Xeeynamo/sotn-decomp.git
synced 2026-11-07 09:16:07 -10:00
Jonathan Hohle
e6cc85dbff
This change adds `WEAPON1` to the `us` build by refactoring the overlay header and `g_HandId`. Where possible, objects are reused between both weapon overlay bundles. Based on the PSP weapon layout it appeared that the header data and `g_HandId` for each weapon were in separate objects from the weapon code and data. Separating them allows the link order to be rearranged between platforms without any code changes or macros. This also eliminates the need to special case function interfaces that differed between weapons. The header only needs to resolve the symbol at link time. A single `w_init` object can be used for all weapons, regardless of hand. `left_hand` and `right_hand` objects can be linked to `w0_*` and `w1_*` weapons, respectively. In the majority of cases, a specific weapon object just needs to be linked to the corresponding "hand" object to match for both `WEAPON0` and `WEAPON1`. This involves a lot of file edits, but the changes are pretty simple: * remove the header `Weapon` struct from each weapon source (defined in `weapon_private.h`) and move into `w_init.c` * remove the `g_HandId` global from each weapon source and move into `left_hand.c` and `right_hand.c` * configure segments for `w_init` and hand and update the build system how to handle multiple data segments. Because `w_init` is resolved at link time, all weapon API functions are no longer `static`. These functions are no longer compiled with the header, so no compile time switches are needed to synchronize the header struct interfaces with the actual function interfaces. Not all `WEAPON1` weapons are matched, primarily because of missing data. This requires either symbol resolution or data imports. This can be done incrementally. Weapons that haven't been matched are using the `asm` segment type out of convenience. The build system has bespoke rules for weapons and this avoided the complexity of generating C stubs for now. There are some weapons that cannot share the same weapon object. For example `w0_051` and `w1_051` differ by one byte in the data segment. These will need to be split into separate objects in a future change. It's not clear to me that the `progress.py` changes will work correctly. Submitting as draft for now for discussion.
499 lines
19 KiB
Python
499 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import mapfile_parser
|
|
import mapfile_parser.utils
|
|
import mapfile_parser.frontends.upload_frogress
|
|
import os
|
|
import requests
|
|
import subprocess
|
|
import sys
|
|
import yaml
|
|
from pathlib import Path
|
|
from mapfile_parser import MapFile
|
|
from mapfile_parser import ProgressStats
|
|
|
|
slug = "sotn"
|
|
|
|
parser = argparse.ArgumentParser(description="Report decompilation progress")
|
|
parser.add_argument("--version", required=False, type=str, help="Game version")
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
dest="dryrun",
|
|
default=False,
|
|
required=False,
|
|
action="store_true",
|
|
help="Print the request instead of posting it to the server",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.version == None:
|
|
args.version = os.getenv("VERSION")
|
|
if args.version == None:
|
|
args.version = "us"
|
|
|
|
|
|
def printerr(msg: str):
|
|
print(msg, file=sys.stderr)
|
|
|
|
|
|
def exiterr(msg: str):
|
|
printerr(msg)
|
|
exit(-1)
|
|
|
|
|
|
def get_git_commit_message() -> str:
|
|
return (
|
|
subprocess.check_output(["git", "show", "-s", "--format=%s"])
|
|
.decode("utf-8")
|
|
.rstrip()
|
|
)
|
|
|
|
|
|
class DecompProgressStats:
|
|
name: str
|
|
exists: bool
|
|
data_imported: int
|
|
data_total: int
|
|
code_matching: int
|
|
code_total: int
|
|
functions_matching: int
|
|
functions_total: int
|
|
code_matching_prev: int
|
|
functions_prev: int
|
|
data_prev: int
|
|
|
|
def get_nonmatchings_path(self, asm_path: str, splat_options: dict) -> Path:
|
|
"""
|
|
Returns one of the following valid paths:
|
|
`asm/us/main/nonmatchings`
|
|
`asm/us/st/wrp/nonmatchings`
|
|
`asm/pspeu/dra_psp/psp/dra_psp`
|
|
`asm/pspeu/st/wrp_psp/psp/wrp_psp`
|
|
"""
|
|
nonmatchings = (
|
|
f"{asm_path}/{splat_options.get("nonmatchings_path", "nonmatchings")}"
|
|
)
|
|
nonmatchings_psp = f"{nonmatchings}/{os.path.basename(asm_path)}"
|
|
if os.path.exists(nonmatchings_psp) and os.path.exists(f"{asm_path}/matchings"):
|
|
return Path(nonmatchings_psp)
|
|
|
|
return Path(nonmatchings)
|
|
|
|
def __init__(self, module_name: str, path: str, src_dir: str = None):
|
|
self.name = module_name
|
|
self.data_imported = 0
|
|
self.data_total = 0
|
|
self.code_matching = 0
|
|
self.code_total = 0
|
|
self.functions_matching = 0
|
|
self.functions_total = 0
|
|
|
|
if src_dir == None:
|
|
src_dir = path
|
|
|
|
splat_config_path = f"config/splat.{args.version}.{module_name}.yaml"
|
|
map_path = None
|
|
if os.path.exists(splat_config_path):
|
|
with open(
|
|
f"config/splat.{args.version}.{module_name}.yaml"
|
|
) as config_yaml_ref:
|
|
config = yaml.safe_load(config_yaml_ref)
|
|
splat_options = config["options"]
|
|
|
|
map_path = Path(f"{splat_options["build_path"]}/{module_name}.map")
|
|
else:
|
|
printerr(f"Config yaml not found {splat_config_path}")
|
|
self.exists = False
|
|
return
|
|
|
|
# first check for path override
|
|
if map_path is None or not os.path.exists(map_path):
|
|
map_path = Path(f"build/{args.version}/{path}.map")
|
|
|
|
# then fall back to path check (really only for weapons)
|
|
if not os.path.exists(map_path):
|
|
printerr(f"file '{map_path}' not found")
|
|
self.exists = False
|
|
return
|
|
self.exists = True
|
|
|
|
map_file = mapfile_parser.MapFile()
|
|
map_file.readMapFile(map_path)
|
|
|
|
asm_path = Path(splat_options["asm_path"])
|
|
nonmatchings = self.get_nonmatchings_path(asm_path, splat_options)
|
|
depth = 4 + src_dir.count("/")
|
|
self.calculate_progress(
|
|
map_file.filterBySectionType(".text"), asm_path, nonmatchings, depth
|
|
)
|
|
self.add_segment_progress(map_file.filterBySectionType(".data"))
|
|
self.add_segment_progress(map_file.filterBySectionType(".rodata"))
|
|
self.add_segment_progress(map_file.filterBySectionType(".bss"))
|
|
|
|
def add_segment_progress(self, map_file: MapFile):
|
|
for file in [file for segment in map_file for file in segment]:
|
|
if "dra_data" in str(
|
|
file.filepath
|
|
): # exclude the VAB chunk at the end of DRA.BIN
|
|
continue
|
|
self.data_total += file.size
|
|
if "src/" in str(file.filepath) or "assets/" in str(file.filepath):
|
|
self.data_imported += file.size
|
|
else:
|
|
continue
|
|
return
|
|
|
|
# modified version of mapfile_parser.MapFile.getProgress
|
|
def calculate_progress(
|
|
self, map_file: MapFile, asmPath: Path, nonmatchings: Path, pathIndex: int
|
|
):
|
|
totalStats = ProgressStats()
|
|
progressPerFolder: dict[str, ProgressStats] = dict()
|
|
for file in [file for segment in map_file for file in segment]:
|
|
if len(file) == 0:
|
|
continue
|
|
|
|
folder = file.filepath.parts[pathIndex]
|
|
if folder not in progressPerFolder:
|
|
progressPerFolder[folder] = ProgressStats()
|
|
|
|
originalFilePath = Path(*file.filepath.parts[pathIndex:])
|
|
|
|
extensionlessFilePath = originalFilePath
|
|
while extensionlessFilePath.suffix:
|
|
extensionlessFilePath = extensionlessFilePath.with_suffix("")
|
|
|
|
if asmPath != "":
|
|
fullAsmFile = asmPath / extensionlessFilePath.with_suffix(".s")
|
|
wholeFileIsUndecomped = fullAsmFile.exists()
|
|
else: # nonmatchings path does not exist, the overlay is 100% decompiled
|
|
wholeFileIsUndecomped = False
|
|
|
|
for func in file:
|
|
if func.name.endswith(".NON_MATCHING"):
|
|
continue
|
|
|
|
self.functions_total += 1
|
|
|
|
funcNonMatching = f"{func.name}.NON_MATCHING"
|
|
|
|
funcSize = 0
|
|
if func.size is not None:
|
|
funcSize = func.size
|
|
|
|
funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s"
|
|
|
|
if wholeFileIsUndecomped:
|
|
totalStats.undecompedSize += funcSize
|
|
progressPerFolder[folder].undecompedSize += funcSize
|
|
elif map_file.findSymbolByName(funcNonMatching) is not None:
|
|
totalStats.undecompedSize += funcSize
|
|
progressPerFolder[folder].undecompedSize += funcSize
|
|
elif funcAsmPath.exists():
|
|
totalStats.undecompedSize += funcSize
|
|
progressPerFolder[folder].undecompedSize += funcSize
|
|
else:
|
|
self.functions_matching += 1
|
|
totalStats.decompedSize += funcSize
|
|
progressPerFolder[folder].decompedSize += funcSize
|
|
|
|
self.code_matching = totalStats.decompedSize
|
|
self.code_total = totalStats.decompedSize + totalStats.undecompedSize
|
|
|
|
|
|
class DecompProgressWeaponStats:
|
|
name: str
|
|
exists: bool
|
|
code_matching: int
|
|
code_total: int
|
|
functions_matching: int
|
|
functions_total: int
|
|
data_imported: int
|
|
data_total: int
|
|
code_matching_prev: int
|
|
functions_prev: int
|
|
|
|
def __init__(self, hand_id: int):
|
|
self.name = f"weapon{hand_id}"
|
|
self.exists = True
|
|
self.code_matching = 0
|
|
self.code_total = 0
|
|
self.functions_matching = 0
|
|
self.functions_total = 0
|
|
self.data_imported = 0
|
|
self.data_total = 0
|
|
for i in range(0, 59):
|
|
if hand_id == 1:
|
|
name = self.name
|
|
else:
|
|
name = "weapon"
|
|
stats = DecompProgressStats(name, f"weapon/w{hand_id}_{i:03d}", "weapon")
|
|
if stats.exists:
|
|
self.code_matching += stats.code_matching
|
|
self.code_total += stats.code_total
|
|
self.functions_matching += stats.functions_matching
|
|
self.functions_total += stats.functions_total
|
|
self.data_imported += stats.data_imported
|
|
self.data_total += stats.data_total
|
|
|
|
|
|
def remove_not_existing_overlays(progresses):
|
|
new_progresses = dict[str, DecompProgressStats]()
|
|
for key in progresses:
|
|
value = progresses[key]
|
|
if value.exists == True:
|
|
new_progresses[key] = value
|
|
return new_progresses
|
|
|
|
|
|
def get_progress(module_name: str, path: str) -> DecompProgressStats:
|
|
return DecompProgressStats(module_name, path)
|
|
|
|
|
|
def hydrate_previous_metrics(progresses: dict[str, DecompProgressStats], version: str):
|
|
def fetch_metrics(category, callback):
|
|
api_base_url = os.getenv("FROGRESS_API_BASE_URL")
|
|
if api_base_url:
|
|
r = requests.get(f"{api_base_url}/data/{slug}/{version}/{category}")
|
|
else:
|
|
print("Warning: api_base_url not set")
|
|
if api_base_url is None or r.status_code == 404:
|
|
for ovl in progress:
|
|
callback(ovl, 0)
|
|
return
|
|
r.raise_for_status()
|
|
res = r.json()
|
|
if (
|
|
res == None
|
|
or res[slug] == None
|
|
or res[slug][version] == None
|
|
or res[slug][version][category] == None
|
|
):
|
|
return progress
|
|
if len(res[slug][version][category]) > 0:
|
|
last_measures = res[slug][version][category][0]["measures"]
|
|
else:
|
|
last_measures = {}
|
|
|
|
for ovl in progress:
|
|
if ovl in last_measures:
|
|
last_measure = last_measures[ovl]
|
|
if last_measure != None:
|
|
callback(ovl, last_measure)
|
|
else:
|
|
callback(ovl, 0)
|
|
|
|
def set_code_prev(ovl_name, value):
|
|
progresses[ovl_name].code_matching_prev = value
|
|
|
|
def set_func_prev(ovl_name, value):
|
|
progresses[ovl_name].functions_prev = value
|
|
|
|
def set_data_prev(ovl_name, value):
|
|
progresses[ovl_name].data_prev = value
|
|
|
|
progress = remove_not_existing_overlays(progresses)
|
|
fetch_metrics("code", set_code_prev)
|
|
fetch_metrics("functions", set_func_prev)
|
|
fetch_metrics("data", set_data_prev)
|
|
|
|
|
|
def get_progress_entry(progresses: dict[str, DecompProgressStats]):
|
|
def as_code(progresses: dict[str, DecompProgressStats]):
|
|
obj = {}
|
|
for key in progresses:
|
|
overlay_progress = progresses[key]
|
|
obj[overlay_progress.name] = overlay_progress.code_matching
|
|
obj[f"{overlay_progress.name}/total"] = overlay_progress.code_total
|
|
return obj
|
|
|
|
def as_functions(progresses: DecompProgressStats):
|
|
obj = {}
|
|
for key in progresses:
|
|
overlay_progress = progresses[key]
|
|
obj[overlay_progress.name] = overlay_progress.functions_matching
|
|
obj[f"{overlay_progress.name}/total"] = overlay_progress.functions_total
|
|
return obj
|
|
|
|
def as_data(progresses: DecompProgressStats):
|
|
obj = {}
|
|
for key in progresses:
|
|
overlay_progress = progresses[key]
|
|
obj[overlay_progress.name] = overlay_progress.data_imported
|
|
obj[f"{overlay_progress.name}/total"] = overlay_progress.data_total
|
|
return obj
|
|
|
|
return {
|
|
"timestamp": mapfile_parser.utils.getGitCommitTimestamp(),
|
|
"git_hash": mapfile_parser.utils.getGitCommitHash(),
|
|
"categories": {
|
|
"code": as_code(progresses),
|
|
"functions": as_functions(progresses),
|
|
"data": as_data(progresses),
|
|
},
|
|
}
|
|
|
|
|
|
def report_stdout(entry):
|
|
print(entry)
|
|
|
|
|
|
def report_human_readable_dryrun(progresses: dict[str, DecompProgressStats]):
|
|
for overlay in progresses:
|
|
stat = progresses[overlay]
|
|
if (
|
|
stat.code_matching != stat.code_matching_prev
|
|
or stat.data_imported != stat.data_prev
|
|
):
|
|
coverage = stat.code_matching / stat.code_total
|
|
coverage_diff = (
|
|
stat.code_matching - stat.code_matching_prev
|
|
) / stat.code_total
|
|
funcs = stat.functions_matching / stat.functions_total
|
|
funcs_diff = (
|
|
stat.functions_matching - stat.functions_prev
|
|
) / stat.functions_total
|
|
data = stat.data_imported / stat.data_total
|
|
data_diff = (stat.data_imported - stat.data_prev) / stat.data_total
|
|
report = f"{overlay.upper()} ({args.version}): "
|
|
if stat.code_matching != stat.code_matching_prev:
|
|
report += f"coverage {coverage*100:.2f}% ({coverage_diff*100:+.2f}%) "
|
|
report += f"funcs {funcs*100:.2f}% ({funcs_diff*100:+.2f}%) "
|
|
if stat.data_imported != stat.data_prev:
|
|
report += f"data {data*100:.2f}% ({data_diff*100:+.2f}%) "
|
|
print(report)
|
|
else:
|
|
print(f"{overlay.upper()} no new progress")
|
|
|
|
|
|
def report_frogress(entry, version):
|
|
api_base_url = os.getenv("FROGRESS_API_BASE_URL")
|
|
url = f"{api_base_url}/data/{slug}/{version}/"
|
|
requests.post(
|
|
url, json={"api_key": os.getenv("FROGRESS_API_SECRET"), "entries": [entry]}
|
|
).raise_for_status()
|
|
|
|
|
|
def report_discord(progresses: dict[str, DecompProgressStats]):
|
|
report = ""
|
|
for overlay in progresses:
|
|
stat = progresses[overlay]
|
|
if (
|
|
stat.code_matching != stat.code_matching_prev
|
|
or stat.data_imported != stat.data_prev
|
|
):
|
|
coverage = stat.code_matching / stat.code_total
|
|
coverage_diff = coverage - (stat.code_matching_prev / stat.code_total)
|
|
funcs = stat.functions_matching / stat.functions_total
|
|
funcs_diff = funcs - (stat.functions_prev / stat.functions_total)
|
|
data = stat.data_imported / stat.data_total
|
|
data_diff = (stat.data_imported - stat.data_prev) / stat.data_total
|
|
if coverage == 1.0 and coverage_diff < 0 and data == 1.0 and data_diff < 0:
|
|
continue # ignore flukes of negative reports with 100% progress on refactor commits
|
|
|
|
report += f"**{overlay.upper()} ({args.version})**:"
|
|
if stat.code_matching != stat.code_matching_prev:
|
|
report += f"coverage {coverage*100:.2f}% ({coverage_diff*100:+.2f}%) "
|
|
report += f"funcs {funcs*100:.2f}% ({funcs_diff*100:+.2f}%) "
|
|
if stat.data_imported != stat.data_prev:
|
|
report += f"data {data*100:.2f}% ({data_diff*100:+.2f}%) "
|
|
report += "\n"
|
|
if len(report) == 0:
|
|
# nothing to report, do not send any message to Discord
|
|
return
|
|
|
|
url = os.getenv("DISCORD_PROGRESS_WEBHOOK")
|
|
data = {
|
|
"username": "Progress",
|
|
"embeds": [
|
|
{
|
|
"title": get_git_commit_message(),
|
|
"description": report,
|
|
}
|
|
],
|
|
}
|
|
requests.post(url, json=data).raise_for_status()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
progress = dict[str, DecompProgressStats]()
|
|
progress["main"] = DecompProgressStats("main", "main")
|
|
progress["dra"] = DecompProgressStats("dra", "dra")
|
|
progress["weapon"] = DecompProgressWeaponStats(0)
|
|
progress["weapon1"] = DecompProgressWeaponStats(1)
|
|
progress["ric"] = DecompProgressStats("ric", "ric")
|
|
progress["maria"] = DecompProgressStats("maria", "maria")
|
|
progress["stare"] = DecompProgressStats("stare", "st/are")
|
|
progress["stcat"] = DecompProgressStats("stcat", "st/cat")
|
|
progress["stcen"] = DecompProgressStats("stcen", "st/cen")
|
|
progress["stchi"] = DecompProgressStats("stchi", "st/chi")
|
|
progress["stdai"] = DecompProgressStats("stdai", "st/dai")
|
|
progress["stdre"] = DecompProgressStats("stdre", "st/dre")
|
|
progress["stlib"] = DecompProgressStats("stlib", "st/lib")
|
|
progress["stmad"] = DecompProgressStats("stmad", "st/mad")
|
|
progress["stno0"] = DecompProgressStats("stno0", "st/no0")
|
|
progress["stno1"] = DecompProgressStats("stno1", "st/no1")
|
|
progress["stno2"] = DecompProgressStats("stno2", "st/no2")
|
|
progress["stno3"] = DecompProgressStats("stno3", "st/no3")
|
|
progress["stno4"] = DecompProgressStats("stno4", "st/no4")
|
|
progress["stnp3"] = DecompProgressStats("stnp3", "st/np3")
|
|
progress["stnz0"] = DecompProgressStats("stnz0", "st/nz0")
|
|
progress["stnz1"] = DecompProgressStats("stnz1", "st/nz1")
|
|
progress["stsel"] = DecompProgressStats("stsel", "st/sel")
|
|
progress["stst0"] = DecompProgressStats("stst0", "st/st0")
|
|
progress["sttop"] = DecompProgressStats("sttop", "st/top")
|
|
progress["stwrp"] = DecompProgressStats("stwrp", "st/wrp")
|
|
progress["strare"] = DecompProgressStats("strare", "st/rare")
|
|
progress["strcen"] = DecompProgressStats("strcen", "st/rcen")
|
|
progress["strcat"] = DecompProgressStats("strcat", "st/rcat")
|
|
progress["strchi"] = DecompProgressStats("strchi", "st/rchi")
|
|
progress["strdai"] = DecompProgressStats("strdai", "st/rdai")
|
|
progress["strlib"] = DecompProgressStats("strlib", "st/rlib")
|
|
progress["strno0"] = DecompProgressStats("strno0", "st/rno0")
|
|
progress["strno1"] = DecompProgressStats("strno1", "st/rno1")
|
|
progress["strno2"] = DecompProgressStats("strno2", "st/rno2")
|
|
progress["strno3"] = DecompProgressStats("strno3", "st/rno3")
|
|
progress["strno4"] = DecompProgressStats("strno4", "st/rno4")
|
|
progress["strnz0"] = DecompProgressStats("strnz0", "st/rnz0")
|
|
progress["strnz1"] = DecompProgressStats("strnz1", "st/rnz1")
|
|
progress["strtop"] = DecompProgressStats("strtop", "st/rtop")
|
|
progress["strwrp"] = DecompProgressStats("strwrp", "st/rwrp")
|
|
progress["bomar"] = DecompProgressStats("bomar", "boss/mar")
|
|
progress["bobo0"] = DecompProgressStats("bobo0", "boss/bo0")
|
|
progress["bobo1"] = DecompProgressStats("bobo1", "boss/bo1")
|
|
progress["bobo2"] = DecompProgressStats("bobo2", "boss/bo2")
|
|
progress["bobo3"] = DecompProgressStats("bobo3", "boss/bo3")
|
|
progress["bobo4"] = DecompProgressStats("bobo4", "boss/bo4")
|
|
progress["bobo5"] = DecompProgressStats("bobo5", "boss/bo5")
|
|
progress["bobo6"] = DecompProgressStats("bobo6", "boss/bo6")
|
|
progress["bobo7"] = DecompProgressStats("bobo7", "boss/bo7")
|
|
progress["borbo0"] = DecompProgressStats("borbo0", "boss/rbo0")
|
|
progress["borbo1"] = DecompProgressStats("borbo1", "boss/rbo1")
|
|
progress["borbo2"] = DecompProgressStats("borbo2", "boss/rbo2")
|
|
progress["borbo3"] = DecompProgressStats("borbo3", "boss/rbo3")
|
|
progress["borbo4"] = DecompProgressStats("borbo4", "boss/rbo4")
|
|
progress["borbo5"] = DecompProgressStats("borbo5", "boss/rbo5")
|
|
progress["borbo6"] = DecompProgressStats("borbo6", "boss/rbo6")
|
|
progress["borbo7"] = DecompProgressStats("borbo7", "boss/rbo7")
|
|
progress["borbo8"] = DecompProgressStats("borbo8", "boss/rbo8")
|
|
progress["tt_000"] = DecompProgressStats("tt_000", "servant/tt_000")
|
|
progress["tt_001"] = DecompProgressStats("tt_001", "servant/tt_001")
|
|
progress["tt_002"] = DecompProgressStats("tt_002", "servant/tt_002")
|
|
progress["tt_003"] = DecompProgressStats("tt_003", "servant/tt_003")
|
|
progress["tt_004"] = DecompProgressStats("tt_004", "servant/tt_004")
|
|
progress["tt_005"] = DecompProgressStats("tt_005", "servant/tt_005")
|
|
progress["tt_006"] = DecompProgressStats("tt_006", "servant/tt_006")
|
|
progress["fname"] = DecompProgressStats("fname", "servant/fname")
|
|
progress["sd"] = DecompProgressStats("sd", "sd")
|
|
progress["sttek"] = DecompProgressStats("sttek", "st/tek")
|
|
|
|
hydrate_previous_metrics(progress, args.version)
|
|
progress = remove_not_existing_overlays(progress)
|
|
|
|
entry = get_progress_entry(progress)
|
|
if args.dryrun == False:
|
|
report_discord(progress)
|
|
report_frogress(entry, args.version)
|
|
else:
|
|
report_stdout(entry)
|
|
report_human_readable_dryrun(progress)
|