Game-Decompilation/ss
1
0
Fork 0
mirror of https://github.com/zeldaret/ss.git synced 2024-11-26 23:10:26 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Update from dtk-template

Browse Source
This commit is contained in:
robojumper 2024-11-24 13:18:14 +01:00
parent f42f9a0e23
commit ab290a2788
4 changed files with 146 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
configure.py
View File

@ -142,9 +142,9 @@ if not config.non_matching:
# Tool versions
config.binutils_tag = "2.42-1"
config.compilers_tag = "20240706"
config.dtk_tag = "v1.1.2"
config.objdiff_tag = "v2.3.2"
config.sjiswrap_tag = "v1.1.1"
config.dtk_tag = "v1.3.0"
config.objdiff_tag = "v2.4.0"
config.sjiswrap_tag = "v1.2.0"
config.wibo_tag = "0.6.11"
# Project
@ -172,6 +172,10 @@ if args.map:
# Use for any additional files that should cause a re-configure when modified
config.reconfig_deps = []
# Optional numeric ID for decomp.me preset
# Can be overridden in libraries or objects
config.scratch_preset_id = None
# Base flags, common to most GC/Wii games.
# Generally leave untouched, with overrides added below.
cflags_base = [
@ -301,6 +305,12 @@ Equivalent = (
config.non_matching
) # Object should be linked when configured with --non-matching
# Object is only matching for specific versions
def MatchingFor(*versions):
return config.version in versions
config.warn_missing_config = False
config.warn_missing_source = False
config.libs = [

70
tools/decompctx.py
View File

@ -18,84 +18,63 @@ from typing import List
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
src_dir = os.path.join(root_dir, "src")
include_dirs = [
os.path.join(root_dir, "include"),
# Add additional include directories here
os.path.join(root_dir, "src"),
os.path.join(root_dir, "src", "PowerPC_EABI_Support", "MetroTRK"),
os.path.join(
root_dir, "src", "PowerPC_EABI_Support", "MSL", "MSL_C", "MSL_Common", "Include"
),
os.path.join(
root_dir,
"src",
"PowerPC_EABI_Support",
"MSL",
"MSL_C",
"MSL_Common_Embedded",
"Math",
"Include",
),
os.path.join(root_dir, "src", "PowerPC_EABI_Support", "MSL", "MSL_C", "PPC_EABI"),
os.path.join(
root_dir,
"src",
"PowerPC_EABI_Support",
"MSL",
"MSL_C++",
"MSL_Common",
"Include",
),
os.path.join(root_dir, "src", "PowerPC_EABI_Support", "Runtime", "Inc"),
]
include_dirs: List[str] = [] # Set with -I flag
include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]')
guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
once_pattern = re.compile(r"^#\s*pragma\s+once$")
defines = set()
deps = []
def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
def import_h_file(in_file: str, r_path: str) -> str:
rel_path = os.path.join(root_dir, r_path, in_file)
if os.path.exists(rel_path):
return import_c_file(rel_path, deps)
return import_c_file(rel_path)
for include_dir in include_dirs:
inc_path = os.path.join(include_dir, in_file)
if os.path.exists(inc_path):
return import_c_file(inc_path, deps)
return import_c_file(inc_path)
else:
print("Failed to locate", in_file)
return ""
def import_c_file(in_file: str, deps: List[str]) -> str:
def import_c_file(in_file: str) -> str:
in_file = os.path.relpath(in_file, root_dir)
deps.append(in_file)
out_text = ""
try:
with open(in_file, encoding="utf-8") as file:
out_text += process_file(in_file, list(file), deps)
out_text += process_file(in_file, list(file))
except Exception:
with open(in_file) as file:
out_text += process_file(in_file, list(file), deps)
out_text += process_file(in_file, list(file))
return out_text
def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
def process_file(in_file: str, lines: List[str]) -> str:
out_text = ""
for idx, line in enumerate(lines):
guard_match = guard_pattern.match(line.strip())
if idx == 0:
guard_match = guard_pattern.match(line.strip())
if guard_match:
if guard_match[1] in defines:
break
defines.add(guard_match[1])
else:
once_match = once_pattern.match(line.strip())
if once_match:
if in_file in defines:
break
defines.add(in_file)
print("Processing file", in_file)
include_match = include_pattern.match(line.strip())
if include_match and not include_match[1].endswith(".s"):
out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
out_text += import_h_file(include_match[1], os.path.dirname(in_file))
out_text += f'/* end "{include_match[1]}" */\n'
else:
out_text += line
@ -126,10 +105,19 @@ def main():
"--depfile",
help="""Dependency file""",
)
parser.add_argument(
"-I",
"--include",
help="""Include directory""",
action="append",
)
args = parser.parse_args()
deps = []
output = import_c_file(args.c_file, deps)
if args.include is None:
exit("No include directories specified")
global include_dirs
include_dirs = args.include
output = import_c_file(args.c_file)
with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
f.write(output)

23
tools/ninja_syntax.py
View File

@ -24,17 +24,10 @@ import textwrap
import os
from io import StringIO
from pathlib import Path
from typing import Dict, List, Match, Optional, Tuple, Union
from typing import Dict, Iterable, List, Match, Optional, Tuple, Union
NinjaPath = Union[str, Path]
NinjaPaths = Union[
List[str],
List[Path],
List[NinjaPath],
List[Optional[str]],
List[Optional[Path]],
List[Optional[NinjaPath]],
]
NinjaPaths = Iterable[Optional[NinjaPath]]
NinjaPathOrPaths = Union[NinjaPath, NinjaPaths]
@ -118,8 +111,8 @@ class Writer(object):
pool: Optional[str] = None,
dyndep: Optional[NinjaPath] = None,
) -> List[str]:
outputs = serialize_paths(outputs)
out_outputs = [escape_path(x) for x in outputs]
str_outputs = serialize_paths(outputs)
out_outputs = [escape_path(x) for x in str_outputs]
all_inputs = [escape_path(x) for x in serialize_paths(inputs)]
if implicit:
@ -154,7 +147,7 @@ class Writer(object):
for key, val in iterator:
self.variable(key, val, indent=1)
return outputs
return str_outputs
def include(self, path: str) -> None:
self._line("include %s" % path)
@ -225,9 +218,11 @@ def serialize_path(input: Optional[NinjaPath]) -> str:
def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]:
if isinstance(input, list):
if isinstance(input, str) or isinstance(input, Path):
return [serialize_path(input)] if input else []
elif input is not None:
return [serialize_path(path) for path in input if path]
return [serialize_path(input)] if input else []
return []
def escape(string: str) -> str:

141
tools/project.py
View File

@ -41,12 +41,14 @@ class Object:
"asflags": None,
"asm_dir": None,
"cflags": None,
"extra_asflags": None,
"extra_cflags": None,
"extra_asflags": [],
"extra_cflags": [],
"extra_clang_flags": [],
"host": None,
"lib": None,
"mw_version": None,
"progress_category": None,
"scratch_preset_id": None,
"shift_jis": None,
"source": name,
"src_dir": None,
@ -78,6 +80,7 @@ class Object:
set_default("asm_dir", config.asm_dir)
set_default("host", False)
set_default("mw_version", config.linker_version)
set_default("scratch_preset_id", config.scratch_preset_id)
set_default("shift_jis", config.shift_jis)
set_default("src_dir", config.src_dir)
@ -172,9 +175,13 @@ class ProjectConfig:
self.generate_compile_commands: bool = (
True # Generate compile_commands.json for clangd
)
self.extra_clang_flags: List[str] = [] # Extra flags for clangd
self.scratch_preset_id: Optional[int] = (
None # Default decomp.me preset ID for scratches
)
# Progress output, progress.json and report.json config
self.progress = True # Enable progress output
self.progress = True # Enable report.json generation and CLI progress output
self.progress_all: bool = True # Include combined "all" category
self.progress_modules: bool = True # Include combined "modules" category
self.progress_each_module: bool = (
@ -375,7 +382,7 @@ def generate_build_ninja(
decompctx = config.tools_dir / "decompctx.py"
n.rule(
name="decompctx",
command=f"$python {decompctx} $in -o $out -d $out.d",
command=f"$python {decompctx} $in -o $out -d $out.d $includes",
description="CTX $in",
depfile="$out.d",
deps="gcc",
@ -631,7 +638,7 @@ def generate_build_ninja(
)
n.newline()
def write_custom_step(step: str) -> List[str | Path]:
def write_custom_step(step: str, prev_step: Optional[str] = None) -> None:
implicit: List[str | Path] = []
if config.custom_build_steps and step in config.custom_build_steps:
n.comment(f"Custom build steps ({step})")
@ -655,7 +662,12 @@ def generate_build_ninja(
dyndep=custom_step.get("dyndep", None),
)
n.newline()
return implicit
n.build(
outputs=step,
rule="phony",
inputs=implicit,
order_only=prev_step,
)
n.comment("Host build")
n.variable("host_cflags", "-I include -Wno-trigraphs")
@ -676,7 +688,7 @@ def generate_build_ninja(
n.newline()
# Add all build steps needed before we compile (e.g. processing assets)
precompile_implicit = write_custom_step("pre-compile")
write_custom_step("pre-compile")
###
# Source files
@ -724,13 +736,12 @@ def generate_build_ninja(
rule="link",
inputs=self.inputs,
implicit=[
*precompile_implicit,
self.ldscript,
*mwld_implicit,
*postcompile_implicit,
],
implicit_outputs=elf_map,
variables={"ldflags": elf_ldflags},
order_only="post-compile",
)
else:
preplf_path = build_path / self.name / f"{self.name}.preplf"
@ -757,6 +768,7 @@ def generate_build_ninja(
implicit=mwld_implicit,
implicit_outputs=preplf_map,
variables={"ldflags": preplf_ldflags},
order_only="post-compile",
)
n.build(
outputs=plf_path,
@ -765,6 +777,7 @@ def generate_build_ninja(
implicit=[self.ldscript, preplf_path, *mwld_implicit],
implicit_outputs=plf_map,
variables={"ldflags": plf_ldflags},
order_only="post-compile",
)
n.newline()
@ -787,24 +800,19 @@ def generate_build_ninja(
# Add appropriate language flag if it doesn't exist already
# Added directly to the source so it flows to other generation tasks
if not any(flag.startswith("-lang") for flag in cflags) and (
extra_cflags is None
or not any(flag.startswith("-lang") for flag in extra_cflags)
if not any(flag.startswith("-lang") for flag in cflags) and not any(
flag.startswith("-lang") for flag in extra_cflags
):
# Ensure extra_cflags is a unique instance,
# and insert into there to avoid modifying shared sets of flags
if extra_cflags is None:
extra_cflags = []
extra_cflags = obj.options["extra_cflags"] = list(extra_cflags)
if file_is_cpp(src_path):
extra_cflags.insert(0, "-lang=c++")
else:
extra_cflags.insert(0, "-lang=c")
cflags_str = make_flags_str(cflags)
if extra_cflags is not None:
extra_cflags_str = make_flags_str(extra_cflags)
cflags_str += " " + extra_cflags_str
all_cflags = cflags + extra_cflags
cflags_str = make_flags_str(all_cflags)
used_compiler_versions.add(obj.options["mw_version"])
# Add MWCC build rule
@ -823,15 +831,26 @@ def generate_build_ninja(
implicit=(
mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit
),
order_only="pre-compile",
)
# Add ctx build rule
if obj.ctx_path is not None:
include_dirs = []
for flag in all_cflags:
if (
flag.startswith("-i ")
or flag.startswith("-I ")
or flag.startswith("-I+")
):
include_dirs.append(flag[3:])
includes = " ".join([f"-I {d}" for d in include_dirs])
n.build(
outputs=obj.ctx_path,
rule="decompctx",
inputs=src_path,
implicit=decompctx,
variables={"includes": includes},
)
# Add host build rule
@ -844,6 +863,7 @@ def generate_build_ninja(
"basedir": os.path.dirname(obj.host_obj_path),
"basefile": obj.host_obj_path.with_suffix(""),
},
order_only="pre-compile",
)
if obj.options["add_to_all"]:
host_source_inputs.append(obj.host_obj_path)
@ -860,7 +880,7 @@ def generate_build_ninja(
if obj.options["asflags"] is None:
sys.exit("ProjectConfig.asflags missing")
asflags_str = make_flags_str(obj.options["asflags"])
if obj.options["extra_asflags"] is not None:
if len(obj.options["extra_asflags"]) > 0:
extra_asflags_str = make_flags_str(obj.options["extra_asflags"])
asflags_str += " " + extra_asflags_str
@ -878,6 +898,7 @@ def generate_build_ninja(
inputs=src_path,
variables={"asflags": asflags_str},
implicit=gnu_as_implicit,
order_only="pre-compile",
)
n.newline()
@ -967,7 +988,7 @@ def generate_build_ninja(
sys.exit(f"Linker {mw_path} does not exist")
# Add all build steps needed before we link and after compiling objects
postcompile_implicit = write_custom_step("post-compile")
write_custom_step("post-compile", "pre-compile")
###
# Link
@ -978,7 +999,7 @@ def generate_build_ninja(
n.newline()
# Add all build steps needed after linking and before GC/Wii native format generation
postlink_implicit = write_custom_step("post-link")
write_custom_step("post-link", "post-compile")
###
# Generate DOL
@ -987,7 +1008,8 @@ def generate_build_ninja(
outputs=link_steps[0].output(),
rule="elf2dol",
inputs=link_steps[0].partial_output(),
implicit=[*postlink_implicit, dtk],
implicit=dtk,
order_only="post-link",
)
###
@ -1049,11 +1071,12 @@ def generate_build_ninja(
"rspfile": config.out_path() / f"rel{idx}.rsp",
"names": rel_names_arg,
},
order_only="post-link",
)
n.newline()
# Add all build steps needed post-build (re-building archives and such)
postbuild_implicit = write_custom_step("post-build")
write_custom_step("post-build", "post-link")
###
# Helper rule for building all source files
@ -1092,7 +1115,8 @@ def generate_build_ninja(
outputs=ok_path,
rule="check",
inputs=config.check_sha_path,
implicit=[dtk, *link_outputs, *postbuild_implicit],
implicit=[dtk, *link_outputs],
order_only="post-build",
)
n.newline()
@ -1114,6 +1138,7 @@ def generate_build_ninja(
python_lib,
report_path,
],
order_only="post-build",
)
###
@ -1125,11 +1150,11 @@ def generate_build_ninja(
command=f"{objdiff} report generate -o $out",
description="REPORT",
)
report_implicit: List[str | Path] = [objdiff, "all_source"]
n.build(
outputs=report_path,
rule="report",
implicit=report_implicit,
implicit=[objdiff, "all_source"],
order_only="post-build",
)
###
@ -1343,9 +1368,21 @@ def generate_objdiff_config(
unit_config["base_path"] = obj.src_obj_path
unit_config["metadata"]["source_path"] = obj.src_path
cflags = obj.options["cflags"]
# Filter out include directories
def keep_flag(flag):
return (
not flag.startswith("-i ")
and not flag.startswith("-i-")
and not flag.startswith("-I ")
and not flag.startswith("-I+")
and not flag.startswith("-I-")
)
all_cflags = list(
filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"])
)
reverse_fn_order = False
for flag in cflags:
for flag in all_cflags:
if not flag.startswith("-inline "):
continue
for value in flag.split(" ")[1].split(","):
@ -1354,24 +1391,16 @@ def generate_objdiff_config(
elif value == "nodeferred":
reverse_fn_order = False
# Filter out include directories
def keep_flag(flag):
return not flag.startswith("-i ") and not flag.startswith("-I ")
cflags = list(filter(keep_flag, cflags))
compiler_version = COMPILER_MAP.get(obj.options["mw_version"])
if compiler_version is None:
print(f"Missing scratch compiler mapping for {obj.options['mw_version']}")
else:
cflags_str = make_flags_str(cflags)
if obj.options["extra_cflags"] is not None:
extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
cflags_str += " " + extra_cflags_str
cflags_str = make_flags_str(all_cflags)
unit_config["scratch"] = {
"platform": "gc_wii",
"compiler": compiler_version,
"c_flags": cflags_str,
"preset_id": obj.options["scratch_preset_id"],
}
if src_exists:
unit_config["scratch"].update(
@ -1387,7 +1416,7 @@ def generate_objdiff_config(
progress_categories.append(category_opt)
unit_config["metadata"].update(
{
"complete": obj.completed,
"complete": obj.completed if src_exists else None,
"reverse_fn_order": reverse_fn_order,
"progress_categories": progress_categories,
}
@ -1468,7 +1497,10 @@ def generate_compile_commands(
"-I-",
"-i-",
}
CFLAG_IGNORE_PREFIX: Tuple[str, ...] = tuple()
CFLAG_IGNORE_PREFIX: Tuple[str, ...] = (
# Recursive includes are not supported by modern compilers
"-ir ",
)
# Flags to replace
CFLAG_REPLACE: Dict[str, str] = {}
@ -1505,12 +1537,28 @@ def generate_compile_commands(
(
"-lang",
{
"c": ("--language=c", "--std=c89"),
"c": ("--language=c", "--std=c99"),
"c99": ("--language=c", "--std=c99"),
"c++": ("--language=c++", "--std=c++98"),
"cplus": ("--language=c++", "--std=c++98"),
},
),
# Enum size
(
"-enum",
{
"min": ("-fshort-enums",),
"int": ("-fno-short-enums",),
},
),
# Common BSS
(
"-common",
{
"off": ("-fno-common",),
"on": ("-fcommon",),
},
),
)
# Flags to pass through
@ -1601,8 +1649,9 @@ def generate_compile_commands(
continue
append_cflags(obj.options["cflags"])
if isinstance(obj.options["extra_cflags"], list):
append_cflags(obj.options["extra_cflags"])
append_cflags(obj.options["extra_cflags"])
cflags.extend(config.extra_clang_flags)
cflags.extend(obj.options["extra_clang_flags"])
unit_config = {
"directory": Path.cwd(),
@ -1661,7 +1710,7 @@ def calculate_progress(config: ProjectConfig) -> None:
data[key] = int(value)
convert_numbers(report_data["measures"])
for category in report_data["categories"]:
for category in report_data.get("categories", []):
convert_numbers(category["measures"])
# Output to GitHub Actions job summary, if available
@ -1703,7 +1752,7 @@ def calculate_progress(config: ProjectConfig) -> None:
)
print_category("All", report_data["measures"])
for category in report_data["categories"]:
for category in report_data.get("categories", []):
if config.print_progress_categories is True or (
isinstance(config.print_progress_categories, list)
and category["id"] in config.print_progress_categories
@ -1761,7 +1810,7 @@ def calculate_progress(config: ProjectConfig) -> None:
else:
# Support for old behavior where "dol" was the main category
add_category("dol", report_data["measures"])
for category in report_data["categories"]:
for category in report_data.get("categories", []):
add_category(category["id"], category["measures"])
with open(out_path / "progress.json", "w", encoding="utf-8") as w: