From d433cda9e2d70df71c8bff8275edc80ffd3f97b6 Mon Sep 17 00:00:00 2001 From: elijah-thomas774 Date: Mon, 10 Jun 2024 00:09:01 -0400 Subject: [PATCH] sync with dtk-template --- configure.py | 1 + tools/decompctx.py | 107 ++++-- tools/download_tool.py | 43 ++- tools/ninja_syntax.py | 229 +++++++----- tools/project.py | 791 ++++++++++++++++++++++++++--------------- tools/transform_dep.py | 6 +- 6 files changed, 752 insertions(+), 425 deletions(-) diff --git a/configure.py b/configure.py index 6a6a47e5..3b36a517 100644 --- a/configure.py +++ b/configure.py @@ -119,6 +119,7 @@ if not is_windows(): config.wrapper = args.wrapper # Tool versions +config.binutils_tag = "2.42-1" config.compilers_tag = "20231018" config.dtk_tag = "v0.9.0" config.sjiswrap_tag = "v1.1.1" diff --git a/tools/decompctx.py b/tools/decompctx.py index c42b1828..e86d5ef3 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -13,60 +13,74 @@ import argparse import os import re +from typing import List script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dir = os.path.join(root_dir, "include") +include_dirs = [ + os.path.join(root_dir, "include"), + # Add additional include directories here +] include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r'^#ifndef\s+(.*)$') +guard_pattern = re.compile(r"^#ifndef\s+(.*)$") defines = set() -def import_h_file(in_file: str, r_path: str) -> str: - rel_path = os.path.join(root_dir, r_path, in_file) - inc_path = os.path.join(include_dir, in_file) - if os.path.exists(rel_path): - return import_c_file(rel_path) - elif os.path.exists(inc_path): - return import_c_file(inc_path) - else: - print("Failed to locate", in_file) - exit(1) -def import_c_file(in_file) -> str: +def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: + rel_path = os.path.join(root_dir, r_path, in_file) + if os.path.exists(rel_path): + return import_c_file(rel_path, deps) + for include_dir in include_dirs: + inc_path = os.path.join(include_dir, in_file) + if os.path.exists(inc_path): + return import_c_file(inc_path, deps) + else: + print("Failed to locate", in_file) + return "" + + +def import_c_file(in_file: str, deps: List[str]) -> str: in_file = os.path.relpath(in_file, root_dir) - out_text = '' + deps.append(in_file) + out_text = "" try: - with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file)) + with open(in_file, encoding="utf-8") as file: + out_text += process_file(in_file, list(file), deps) except Exception: - with open(in_file) as file: - out_text += process_file(in_file, list(file)) + with open(in_file) as file: + out_text += process_file(in_file, list(file), deps) return out_text -def process_file(in_file: str, lines) -> str: - out_text = '' + +def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: + out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) - if idx == 0: - if guard_match: - if guard_match[1] in defines: - break - defines.add(guard_match[1]) - print("Processing file", in_file) - include_match = include_pattern.match(line.strip()) - if include_match and not include_match[1].endswith(".s"): - out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n" - out_text += import_h_file(include_match[1], os.path.dirname(in_file)) - out_text += f"/* end \"{include_match[1]}\" */\n" - else: - out_text += line + guard_match = guard_pattern.match(line.strip()) + if idx == 0: + if guard_match: + if guard_match[1] in defines: + break + defines.add(guard_match[1]) + print("Processing file", in_file) + include_match = include_pattern.match(line.strip()) + if include_match and not include_match[1].endswith(".s"): + out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' + out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += f'/* end "{include_match[1]}" */\n' + else: + out_text += line return out_text + +def sanitize_path(path: str) -> str: + return path.replace("\\", "/").replace(" ", "\\ ") + + def main(): parser = argparse.ArgumentParser( description="""Create a context file which can be used for decomp.me""" @@ -75,13 +89,32 @@ def main(): "c_file", help="""File from which to create context""", ) + parser.add_argument( + "-o", + "--output", + help="""Output file""", + default="ctx.c", + ) + parser.add_argument( + "-d", + "--depfile", + help="""Dependency file""", + ) args = parser.parse_args() - output = import_c_file(args.c_file) + deps = [] + output = import_c_file(args.c_file, deps) - with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f: + with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) + if args.depfile: + with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f: + f.write(sanitize_path(args.output) + ":") + for dep in deps: + path = sanitize_path(dep) + f.write(f" \\\n\t{path}") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/download_tool.py b/tools/download_tool.py index fef42d6e..7b386a4b 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -18,11 +18,29 @@ import shutil import stat import urllib.request import zipfile - +from typing import Callable, Dict from pathlib import Path -def dtk_url(tag): +def binutils_url(tag): + uname = platform.uname() + system = uname.system.lower() + arch = uname.machine.lower() + if system == "darwin": + system = "macos" + arch = "universal" + elif arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/gc-wii-binutils" + return f"{repo}/releases/download/{tag}/{system}-{arch}.zip" + + +def compilers_url(tag: str) -> str: + return f"https://files.decomp.dev/compilers_{tag}.zip" + + +def dtk_url(tag: str) -> str: uname = platform.uname() suffix = "" system = uname.system.lower() @@ -38,29 +56,26 @@ def dtk_url(tag): return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" -def sjiswrap_url(tag): +def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" -def wibo_url(tag): +def wibo_url(tag: str) -> str: repo = "https://github.com/decompals/wibo" return f"{repo}/releases/download/{tag}/wibo" -def compilers_url(tag): - return f"https://files.decomp.dev/compilers_{tag}.zip" - - -TOOLS = { +TOOLS: Dict[str, Callable[[str], str]] = { + "binutils": binutils_url, + "compilers": compilers_url, "dtk": dtk_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, - "compilers": compilers_url, } -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("tool", help="Tool name") parser.add_argument("output", type=Path, help="output file path") @@ -77,7 +92,11 @@ def main(): data = io.BytesIO(response.read()) with zipfile.ZipFile(data) as f: f.extractall(output) - output.touch(mode=0o755) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime else: with open(output, "wb") as f: shutil.copyfileobj(response, f) diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index e73b21a4..7306ee1d 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -1,5 +1,3 @@ -#!/usr/bin/python - # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,82 +21,129 @@ use Python. import re import textwrap +import os +from io import StringIO +from pathlib import Path +from typing import Dict, List, Match, Optional, Tuple, Union + +NinjaPath = Union[str, Path] +NinjaPaths = Union[ + List[str], + List[Path], + List[NinjaPath], + List[Optional[str]], + List[Optional[Path]], + List[Optional[NinjaPath]], +] +NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] + + +def escape_path(word: str) -> str: + return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:") -def escape_path(word): - return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') class Writer(object): - def __init__(self, output, width=78): + def __init__(self, output: StringIO, width: int = 78) -> None: self.output = output self.width = width - def newline(self): - self.output.write('\n') + def newline(self) -> None: + self.output.write("\n") - def comment(self, text): - for line in textwrap.wrap(text, self.width - 2, break_long_words=False, - break_on_hyphens=False): - self.output.write('# ' + line + '\n') + def comment(self, text: str) -> None: + for line in textwrap.wrap( + text, self.width - 2, break_long_words=False, break_on_hyphens=False + ): + self.output.write("# " + line + "\n") - def variable(self, key, value, indent=0): - if value is None: - return - if isinstance(value, list): - value = ' '.join(filter(None, value)) # Filter out empty strings. - self._line('%s = %s' % (key, value), indent) + def variable( + self, + key: str, + value: Optional[NinjaPathOrPaths], + indent: int = 0, + ) -> None: + value = " ".join(serialize_paths(value)) + self._line("%s = %s" % (key, value), indent) - def pool(self, name, depth): - self._line('pool %s' % name) - self.variable('depth', depth, indent=1) + def pool(self, name: str, depth: int) -> None: + self._line("pool %s" % name) + self.variable("depth", str(depth), indent=1) - def rule(self, name, command, description=None, depfile=None, - generator=False, pool=None, restat=False, rspfile=None, - rspfile_content=None, deps=None): - self._line('rule %s' % name) - self.variable('command', command, indent=1) + def rule( + self, + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[NinjaPath] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[NinjaPath] = None, + rspfile_content: Optional[NinjaPath] = None, + deps: Optional[NinjaPathOrPaths] = None, + ) -> None: + self._line("rule %s" % name) + self.variable("command", command, indent=1) if description: - self.variable('description', description, indent=1) + self.variable("description", description, indent=1) if depfile: - self.variable('depfile', depfile, indent=1) + self.variable("depfile", depfile, indent=1) if generator: - self.variable('generator', '1', indent=1) + self.variable("generator", "1", indent=1) if pool: - self.variable('pool', pool, indent=1) + self.variable("pool", pool, indent=1) if restat: - self.variable('restat', '1', indent=1) + self.variable("restat", "1", indent=1) if rspfile: - self.variable('rspfile', rspfile, indent=1) + self.variable("rspfile", rspfile, indent=1) if rspfile_content: - self.variable('rspfile_content', rspfile_content, indent=1) + self.variable("rspfile_content", rspfile_content, indent=1) if deps: - self.variable('deps', deps, indent=1) + self.variable("deps", deps, indent=1) - def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, - variables=None, implicit_outputs=None, pool=None, dyndep=None): - outputs = as_list(outputs) + def build( + self, + outputs: NinjaPathOrPaths, + rule: str, + inputs: Optional[NinjaPathOrPaths] = None, + implicit: Optional[NinjaPathOrPaths] = None, + order_only: Optional[NinjaPathOrPaths] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[NinjaPathOrPaths]]], + Dict[str, Optional[NinjaPathOrPaths]], + ] + ] = None, + implicit_outputs: Optional[NinjaPathOrPaths] = None, + pool: Optional[str] = None, + dyndep: Optional[NinjaPath] = None, + ) -> List[str]: + outputs = serialize_paths(outputs) out_outputs = [escape_path(x) for x in outputs] - all_inputs = [escape_path(x) for x in as_list(inputs)] + all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: - implicit = [escape_path(x) for x in as_list(implicit)] - all_inputs.append('|') - all_inputs.extend(implicit) + implicit = [escape_path(x) for x in serialize_paths(implicit)] + all_inputs.append("|") + all_inputs.extend(map(str, implicit)) if order_only: - order_only = [escape_path(x) for x in as_list(order_only)] - all_inputs.append('||') - all_inputs.extend(order_only) + order_only = [escape_path(x) for x in serialize_paths(order_only)] + all_inputs.append("||") + all_inputs.extend(map(str, order_only)) if implicit_outputs: - implicit_outputs = [escape_path(x) - for x in as_list(implicit_outputs)] - out_outputs.append('|') - out_outputs.extend(implicit_outputs) + implicit_outputs = [ + escape_path(x) for x in serialize_paths(implicit_outputs) + ] + out_outputs.append("|") + out_outputs.extend(map(str, implicit_outputs)) - self._line('build %s: %s' % (' '.join(out_outputs), - ' '.join([rule] + all_inputs))) + self._line( + "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs)) + ) if pool is not None: - self._line(' pool = %s' % pool) + self._line(" pool = %s" % pool) if dyndep is not None: - self._line(' dyndep = %s' % dyndep) + self._line(" dyndep = %s" % serialize_path(dyndep)) if variables: if isinstance(variables, dict): @@ -111,89 +156,99 @@ class Writer(object): return outputs - def include(self, path): - self._line('include %s' % path) + def include(self, path: str) -> None: + self._line("include %s" % path) - def subninja(self, path): - self._line('subninja %s' % path) + def subninja(self, path: str) -> None: + self._line("subninja %s" % path) - def default(self, paths): - self._line('default %s' % ' '.join(as_list(paths))) + def default(self, paths: NinjaPathOrPaths) -> None: + self._line("default %s" % " ".join(serialize_paths(paths))) - def _count_dollars_before_index(self, s, i): + def _count_dollars_before_index(self, s: str, i: int) -> int: """Returns the number of '$' characters right in front of s[i].""" dollar_count = 0 dollar_index = i - 1 - while dollar_index > 0 and s[dollar_index] == '$': + while dollar_index > 0 and s[dollar_index] == "$": dollar_count += 1 dollar_index -= 1 return dollar_count - def _line(self, text, indent=0): + def _line(self, text: str, indent: int = 0) -> None: """Write 'text' word-wrapped at self.width characters.""" - leading_space = ' ' * indent + leading_space = " " * indent while len(leading_space) + len(text) > self.width: # The text is too wide; wrap if possible. # Find the rightmost space that would obey our width constraint and # that's not an escaped space. - available_space = self.width - len(leading_space) - len(' $') + available_space = self.width - len(leading_space) - len(" $") space = available_space while True: - space = text.rfind(' ', 0, space) - if (space < 0 or - self._count_dollars_before_index(text, space) % 2 == 0): + space = text.rfind(" ", 0, space) + if space < 0 or self._count_dollars_before_index(text, space) % 2 == 0: break if space < 0: # No such space; just use the first unescaped space we can find. space = available_space - 1 while True: - space = text.find(' ', space + 1) - if (space < 0 or - self._count_dollars_before_index(text, space) % 2 == 0): + space = text.find(" ", space + 1) + if ( + space < 0 + or self._count_dollars_before_index(text, space) % 2 == 0 + ): break if space < 0: # Give up on breaking. break - self.output.write(leading_space + text[0:space] + ' $\n') - text = text[space+1:] + self.output.write(leading_space + text[0:space] + " $\n") + text = text[space + 1 :] # Subsequent lines are continuations, so indent them. - leading_space = ' ' * (indent+2) + leading_space = " " * (indent + 2) - self.output.write(leading_space + text + '\n') + self.output.write(leading_space + text + "\n") - def close(self): + def close(self) -> None: self.output.close() -def as_list(input): - if input is None: - return [] +def serialize_path(input: Optional[NinjaPath]) -> str: + if not input: + return "" + if isinstance(input, Path): + return str(input).replace("/", os.sep) + else: + return str(input) + + +def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: if isinstance(input, list): - return input - return [input] + return [serialize_path(path) for path in input if path] + return [serialize_path(input)] if input else [] -def escape(string): +def escape(string: str) -> str: """Escape a string such that it can be embedded into a Ninja file without further interpretation.""" - assert '\n' not in string, 'Ninja syntax does not allow newlines' + assert "\n" not in string, "Ninja syntax does not allow newlines" # We only have one special metacharacter: '$'. - return string.replace('$', '$$') + return string.replace("$", "$$") -def expand(string, vars, local_vars={}): +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: """Expand a string containing $vars as Ninja would. Note: doesn't handle the full Ninja variable syntax, but it's enough to make configure.py's use of it work. """ - def exp(m): + + def exp(m: Match[str]) -> str: var = m.group(1) - if var == '$': - return '$' - return local_vars.get(var, vars.get(var, '')) - return re.sub(r'\$(\$|\w*)', exp, string) \ No newline at end of file + if var == "$": + return "$" + return local_vars.get(var, vars.get(var, "")) + + return re.sub(r"\$(\$|\w*)", exp, string) diff --git a/tools/project.py b/tools/project.py index 8ccfde1a..4e2263d0 100644 --- a/tools/project.py +++ b/tools/project.py @@ -12,13 +12,15 @@ import io import json +import math import os import platform import sys -import math - from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + from . import ninja_syntax +from .ninja_syntax import serialize_path if sys.platform == "cygwin": sys.exit( @@ -28,51 +30,86 @@ if sys.platform == "cygwin": ) +class Object: + def __init__(self, completed: bool, name: str, **options: Any) -> None: + self.name = name + self.base_name = Path(name).with_suffix("") + self.completed = completed + self.options: Dict[str, Any] = { + "add_to_all": True, + "asflags": None, + "extra_asflags": None, + "cflags": None, + "extra_cflags": None, + "mw_version": None, + "shift_jis": None, + "source": name, + } + self.options.update(options) + + class ProjectConfig: - def __init__(self): + def __init__(self) -> None: # Paths - self.build_dir = Path("build") - self.src_dir = Path("src") - self.tools_dir = Path("tools") + self.build_dir: Path = Path("build") # Output build files + self.src_dir: Path = Path("src") # C/C++/asm source files + self.tools_dir: Path = Path("tools") # Python scripts + self.asm_dir: Optional[Path] = Path( + "asm" + ) # Override incomplete objects (for modding) # Tooling - self.dtk_tag = None # Git tag - self.build_dtk_path = None # If None, download - self.compilers_tag = None # 1 - self.compilers_path = None # If None, download - self.wibo_tag = None # Git tag - self.wrapper = None # If None, download wibo on Linux - self.sjiswrap_tag = None # Git tag - self.sjiswrap_path = None # If None, download + self.binutils_tag: Optional[str] = None # Git tag + self.binutils_path: Optional[Path] = None # If None, download + self.dtk_tag: Optional[str] = None # Git tag + self.dtk_path: Optional[Path] = None # If None, download + self.compilers_tag: Optional[str] = None # 1 + self.compilers_path: Optional[Path] = None # If None, download + self.wibo_tag: Optional[str] = None # Git tag + self.wrapper: Optional[Path] = None # If None, download wibo on Linux + self.sjiswrap_tag: Optional[str] = None # Git tag + self.sjiswrap_path: Optional[Path] = None # If None, download # Project config - self.build_rels = True # Build REL files - self.check_sha_path = None # Path to version.sha1 - self.config_path = None # Path to config.yml - self.debug = False # Build with debug info - self.generate_map = False # Generate map file(s) - self.ldflags = None # Linker flags - self.libs = None # List of libraries - self.linker_version = None # mwld version - self.version = None # Version name - self.warn_missing_config = False # Warn on missing unit configuration - self.warn_missing_source = False # Warn on missing source file - self.rel_strip_partial = True # Generate PLFs with -strip_partial - self.rel_empty_file = None # Path to empty.c for generating empty RELs + self.non_matching: bool = False + self.build_rels: bool = True # Build REL files + self.check_sha_path: Optional[Path] = None # Path to version.sha1 + self.config_path: Optional[Path] = None # Path to config.yml + self.debug: bool = False # Build with debug info + self.generate_map: bool = False # Generate map file(s) + self.asflags: Optional[List[str]] = None # Assembler flags + self.ldflags: Optional[List[str]] = None # Linker flags + self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.linker_version: Optional[str] = None # mwld version + self.version: Optional[str] = None # Version name + self.warn_missing_config: bool = False # Warn on missing unit configuration + self.warn_missing_source: bool = False # Warn on missing source file + self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial + self.rel_empty_file: Optional[ + str + ] = None # Object name for generating empty RELs + self.shift_jis = ( + True # Convert source files from UTF-8 to Shift JIS automatically + ) + self.reconfig_deps: Optional[List[Path]] = ( + None # Additional re-configuration dependency files + ) # Progress output and progress.json config - self.progress_all = True # Include combined "all" category - self.progress_modules = True # Include combined "modules" category - self.progress_each_module = True # Include individual modules, disable for large numbers of modules + self.progress_all: bool = True # Include combined "all" category + self.progress_modules: bool = True # Include combined "modules" category + self.progress_each_module: bool = ( + True # Include individual modules, disable for large numbers of modules + ) # Progress fancy printing - self.progress_use_fancy = False - self.progress_code_fancy_frac = 0 - self.progress_code_fancy_item = "" - self.progress_data_fancy_frac = 0 - self.progress_data_fancy_item = "" + self.progress_use_fancy: bool = False + self.progress_code_fancy_frac: int = 0 + self.progress_code_fancy_item: str = "" + self.progress_data_fancy_frac: int = 0 + self.progress_data_fancy_item: str = "" - def validate(self): + def validate(self) -> None: required_attrs = [ "build_dir", "src_dir", @@ -88,33 +125,18 @@ class ProjectConfig: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name): - for lib in self.libs: + def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + for lib in self.libs or {}: for obj in lib["objects"]: if obj.name == name: - return [lib, obj] + return lib, obj return None - def out_path(self): - return self.build_dir / self.version + def out_path(self) -> Path: + return self.build_dir / str(self.version) -class Object: - def __init__(self, completed, name, **options): - self.name = name - self.completed = completed - self.options = { - "add_to_all": True, - "cflags": None, - "extra_cflags": None, - "mw_version": None, - "shiftjis": True, - "source": name, - } - self.options.update(options) - - -def is_windows(): +def is_windows() -> bool: return os.name == "nt" @@ -124,36 +146,25 @@ CHAIN = "cmd /c " if is_windows() else "" EXE = ".exe" if is_windows() else "" -# Replace forward slashes with backslashes on Windows -def os_str(value): - return str(value).replace("/", os.sep) - - -# Replace backslashes with forward slashes on Windows -def unix_str(value): - return str(value).replace(os.sep, "/") - - -# Stringify paths for ninja_syntax -def path(value): - if value is None: - return None - elif isinstance(value, list): - return list(map(os_str, filter(lambda x: x is not None, value))) +def make_flags_str(cflags: Union[str, List[str]]) -> str: + if isinstance(cflags, list): + return " ".join(cflags) else: - return [os_str(value)] + return cflags # Load decomp-toolkit generated config.json -def load_build_config(config, build_config_path): +def load_build_config( + config: ProjectConfig, build_config_path: Path +) -> Optional[Dict[str, Any]]: if not build_config_path.is_file(): return None - def versiontuple(v): + def versiontuple(v: str) -> Tuple[int, ...]: return tuple(map(int, (v.split(".")))) f = open(build_config_path, "r", encoding="utf-8") - build_config = json.load(f) + build_config: Dict[str, Any] = json.load(f) config_version = build_config.get("version") if not config_version: # Invalid config.json @@ -161,7 +172,7 @@ def load_build_config(config, build_config_path): os.remove(build_config_path) return None - dtk_version = config.dtk_tag[1:] # Strip v + dtk_version = str(config.dtk_tag)[1:] # Strip v if versiontuple(config_version) < versiontuple(dtk_version): # Outdated config.json f.close() @@ -173,14 +184,16 @@ def load_build_config(config, build_config_path): # Generate build.ninja and objdiff.json -def generate_build(config): +def generate_build(config: ProjectConfig) -> None: build_config = load_build_config(config, config.out_path() / "config.json") generate_build_ninja(config, build_config) generate_objdiff_config(config, build_config) # Generate build.ninja -def generate_build_ninja(config, build_config): +def generate_build_ninja( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: config.validate() out = io.StringIO() @@ -188,9 +201,9 @@ def generate_build_ninja(config, build_config): n.variable("ninja_required_version", "1.3") n.newline() - configure_script = os.path.relpath(os.path.abspath(sys.argv[0])) - python_lib = os.path.relpath(__file__) - python_lib_dir = os.path.dirname(python_lib) + configure_script = Path(os.path.relpath(os.path.abspath(sys.argv[0]))) + python_lib = Path(os.path.relpath(__file__)) + python_lib_dir = python_lib.parent n.comment("The arguments passed to configure.py, for rerunning it.") n.variable("configure_args", sys.argv[1:]) n.variable("python", f'"{sys.executable}"') @@ -200,13 +213,15 @@ def generate_build_ninja(config, build_config): # Variables ### n.comment("Variables") - ldflags = " ".join(config.ldflags) + ldflags = " ".join(config.ldflags or []) if config.generate_map: ldflags += " -mapunused" if config.debug: ldflags += " -g" n.variable("ldflags", ldflags) - n.variable("mw_version", config.linker_version) + if not config.linker_version: + sys.exit("ProjectConfig.linker_version missing") + n.variable("mw_version", Path(config.linker_version)) n.newline() ### @@ -215,6 +230,7 @@ def generate_build_ninja(config, build_config): n.comment("Tooling") build_path = config.out_path() + progress_path = build_path / "progress.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -223,20 +239,31 @@ def generate_build_ninja(config, build_config): description="TOOL $out", ) - if config.build_dtk_path: + decompctx = config.tools_dir / "decompctx.py" + n.rule( + name="decompctx", + command=f"$python {decompctx} $in -o $out -d $out.d", + description="CTX $in", + depfile="$out.d", + deps="gcc", + ) + + if config.dtk_path is not None and config.dtk_path.is_file(): + dtk = config.dtk_path + elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" n.rule( name="cargo", command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", description="CARGO $bin", - depfile=path(Path("$target") / "release" / "$bin.d"), + depfile=Path("$target") / "release" / "$bin.d", deps="gcc", ) n.build( - outputs=path(dtk), + outputs=dtk, rule="cargo", - inputs=path(config.build_dtk_path / "Cargo.toml"), - implicit=path(config.build_dtk_path / "Cargo.lock"), + inputs=config.dtk_path / "Cargo.toml", + implicit=config.dtk_path / "Cargo.lock", variables={ "bin": "dtk", "target": build_tools_path, @@ -245,9 +272,9 @@ def generate_build_ninja(config, build_config): elif config.dtk_tag: dtk = build_tools_path / f"dtk{EXE}" n.build( - outputs=path(dtk), + outputs=dtk, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "dtk", "tag": config.dtk_tag, @@ -261,9 +288,9 @@ def generate_build_ninja(config, build_config): elif config.sjiswrap_tag: sjiswrap = build_tools_path / "sjiswrap.exe" n.build( - outputs=path(sjiswrap), + outputs=sjiswrap, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "sjiswrap", "tag": config.sjiswrap_tag, @@ -274,7 +301,7 @@ def generate_build_ninja(config, build_config): # Only add an implicit dependency on wibo if we download it wrapper = config.wrapper - wrapper_implicit = None + wrapper_implicit: Optional[Path] = None if ( config.wibo_tag is not None and sys.platform == "linux" @@ -284,33 +311,53 @@ def generate_build_ninja(config, build_config): wrapper = build_tools_path / "wibo" wrapper_implicit = wrapper n.build( - outputs=path(wrapper), + outputs=wrapper, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "wibo", "tag": config.wibo_tag, }, ) if not is_windows() and wrapper is None: - wrapper = "wine" + wrapper = Path("wine") wrapper_cmd = f"{wrapper} " if wrapper else "" - compilers_implicit = None + compilers_implicit: Optional[Path] = None if config.compilers_path: compilers = config.compilers_path elif config.compilers_tag: compilers = config.build_dir / "compilers" compilers_implicit = compilers n.build( - outputs=path(compilers), + outputs=compilers, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "compilers", "tag": config.compilers_tag, }, ) + else: + sys.exit("ProjectConfig.compilers_tag missing") + + binutils_implicit = None + if config.binutils_path: + binutils = config.binutils_path + elif config.binutils_tag: + binutils = config.build_dir / "binutils" + binutils_implicit = binutils + n.build( + outputs=binutils, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "binutils", + "tag": config.binutils_tag, + }, + ) + else: + sys.exit("ProjectConfig.binutils_tag missing") n.newline() @@ -322,16 +369,24 @@ def generate_build_ninja(config, build_config): # MWCC mwcc = compiler_path / "mwcceppc.exe" mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit] + mwcc_implicit: List[Optional[Path]] = [compilers_implicit or mwcc, wrapper_implicit] # MWCC with UTF-8 to Shift JIS wrapper mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap] + mwcc_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap] # MWLD mwld = compiler_path / "mwldeppc.exe" mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" - mwld_implicit = [compilers_implicit or mwld, wrapper_implicit] + mwld_implicit: List[Optional[Path]] = [compilers_implicit or mwld, wrapper_implicit] + + # GNU as + gnu_as = binutils / f"powerpc-eabi-as{EXE}" + gnu_as_cmd = ( + f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d" + + f" && {dtk} elf fixup $out $out" + ) + gnu_as_implicit = [binutils_implicit or gnu_as, dtk] if os.name != "nt": transform_dep = config.tools_dir / "transform_dep.py" @@ -358,17 +413,6 @@ def generate_build_ninja(config, build_config): ) n.newline() - n.comment("Generate REL(s)") - makerel_rsp = build_path / "makerel.rsp" - n.rule( - name="makerel", - command=f"{dtk} rel make -w -c $config @{makerel_rsp}", - description="REL", - rspfile=path(makerel_rsp), - rspfile_content="$in_newline", - ) - n.newline() - n.comment("MWCC build") n.rule( name="mwcc", @@ -389,6 +433,16 @@ def generate_build_ninja(config, build_config): ) n.newline() + n.comment("Assemble asm") + n.rule( + name="as", + command=gnu_as_cmd, + description="AS $out", + depfile="$out.d", + deps="gcc", + ) + n.newline() + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -411,66 +465,67 @@ def generate_build_ninja(config, build_config): # Source files ### n.comment("Source files") + build_asm_path = build_path / "mod" build_src_path = build_path / "src" build_host_path = build_path / "host" build_config_path = build_path / "config.json" - def map_path(path): + def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") class LinkStep: - def __init__(self, config): - self.name = config["name"] - self.module_id = config["module_id"] - self.ldscript = config["ldscript"] + def __init__(self, config: Dict[str, Any]) -> None: + self.name: str = config["name"] + self.module_id: int = config["module_id"] + self.ldscript: Optional[Path] = Path(config["ldscript"]) self.entry = config["entry"] - self.inputs = [] + self.inputs: List[str] = [] - def add(self, obj): - self.inputs.append(obj) + def add(self, obj: Path) -> None: + self.inputs.append(serialize_path(obj)) - def output(self): + def output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.dol" else: return build_path / self.name / f"{self.name}.rel" - def partial_output(self): + def partial_output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.elf" else: return build_path / self.name / f"{self.name}.plf" - def write(self, n): + def write(self, n: ninja_syntax.Writer) -> None: n.comment(f"Link {self.name}") if self.module_id == 0: elf_path = build_path / f"{self.name}.elf" dol_path = build_path / f"{self.name}.dol" - elf_ldflags = f"$ldflags -lcf {self.ldscript}" + elf_ldflags = f"$ldflags -lcf {serialize_path(self.ldscript)}" if config.generate_map: elf_map = map_path(elf_path) - elf_ldflags += f" -map {elf_map}" + elf_ldflags += f" -map {serialize_path(elf_map)}" else: elf_map = None n.build( - outputs=path(elf_path), + outputs=elf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, *mwld_implicit]), - implicit_outputs=path(elf_map), + inputs=self.inputs, + implicit=[self.ldscript, *mwld_implicit], + implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, ) n.build( - outputs=path(dol_path), + outputs=dol_path, rule="elf2dol", - inputs=path(elf_path), - implicit=path(dtk), + inputs=elf_path, + implicit=dtk, ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" - preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" - plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" + preplf_ldflags = "$ldflags -sdata 0 -sdata2 0 -r" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {serialize_path(self.ldscript)}" if self.entry: plf_ldflags += f" -m {self.entry}" # -strip_partial is only valid with -m @@ -478,44 +533,140 @@ def generate_build_ninja(config, build_config): plf_ldflags += " -strip_partial" if config.generate_map: preplf_map = map_path(preplf_path) - preplf_ldflags += f" -map {preplf_map}" + preplf_ldflags += f" -map {serialize_path(preplf_map)}" plf_map = map_path(plf_path) - plf_ldflags += f" -map {plf_map}" + plf_ldflags += f" -map {serialize_path(plf_map)}" else: preplf_map = None plf_map = None n.build( - outputs=path(preplf_path), + outputs=preplf_path, rule="link", - inputs=path(self.inputs), - implicit=path(mwld_implicit), - implicit_outputs=path(preplf_map), + inputs=self.inputs, + implicit=mwld_implicit, + implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, ) n.build( - outputs=path(plf_path), + outputs=plf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, preplf_path, *mwld_implicit]), - implicit_outputs=path(plf_map), + inputs=self.inputs, + implicit=[self.ldscript, preplf_path, *mwld_implicit], + implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, ) n.newline() + link_outputs: List[Path] = [] if build_config: - link_steps = [] - used_compiler_versions = set() - source_inputs = [] - host_source_inputs = [] - source_added = set() + link_steps: List[LinkStep] = [] + used_compiler_versions: Set[str] = set() + source_inputs: List[Path] = [] + host_source_inputs: List[Path] = [] + source_added: Set[Path] = set() - def make_cflags_str(cflags): - if isinstance(cflags, list): - return " ".join(cflags) - else: - return cflags + def c_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: + cflags_str = make_flags_str(options["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str + used_compiler_versions.add(options["mw_version"]) - def add_unit(build_obj, link_step): + src_obj_path = build_src_path / f"{obj.base_name}.o" + src_base_path = build_src_path / obj.base_name + + # Avoid creating duplicate build rules + if src_obj_path in source_added: + return src_obj_path + source_added.add(src_obj_path) + + shift_jis = options["shift_jis"] + if shift_jis is None: + shift_jis = config.shift_jis + + # Add MWCC build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=src_obj_path, + rule="mwcc_sjis" if shift_jis else "mwcc", + inputs=src_path, + variables={ + "mw_version": Path(options["mw_version"]), + "cflags": cflags_str, + "basedir": os.path.dirname(src_base_path), + "basefile": src_base_path, + }, + implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + ) + + # Add ctx build rule + ctx_path = build_src_path / f"{obj.base_name}.ctx" + n.build( + outputs=ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) + + # Add host build rule + if options.get("host", False): + host_obj_path = build_host_path / f"{obj.base_name}.o" + host_base_path = build_host_path / obj.base_name + n.build( + outputs=host_obj_path, + rule="host_cc" if src_path.suffix == ".c" else "host_cpp", + inputs=src_path, + variables={ + "basedir": os.path.dirname(host_base_path), + "basefile": host_base_path, + }, + ) + if options["add_to_all"]: + host_source_inputs.append(host_obj_path) + n.newline() + + if options["add_to_all"]: + source_inputs.append(src_obj_path) + + return src_obj_path + + def asm_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: + asflags = options["asflags"] or config.asflags + if asflags is None: + sys.exit("ProjectConfig.asflags missing") + asflags_str = make_flags_str(asflags) + if options["extra_asflags"] is not None: + extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str += " " + extra_asflags_str + + asm_obj_path = build_asm_path / f"{obj.base_name}.o" + + # Avoid creating duplicate build rules + if asm_obj_path in source_added: + return asm_obj_path + source_added.add(asm_obj_path) + + # Add assembler build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=asm_obj_path, + rule="as", + inputs=src_path, + variables={"asflags": asflags_str}, + implicit=gnu_as_implicit, + ) + n.newline() + + if options["add_to_all"]: + source_inputs.append(asm_obj_path) + + return asm_obj_path + + def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) if not result: @@ -526,71 +677,50 @@ def generate_build_ninja(config, build_config): lib, obj = result lib_name = lib["lib"] - src_dir = Path(lib.get("src_dir", config.src_dir)) - options = obj.options - completed = obj.completed + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value - unit_src_path = src_dir / options["source"] + unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - if not unit_src_path.exists(): - if config.warn_missing_source or completed: + unit_asm_path: Optional[Path] = None + if config.asm_dir is not None: + unit_asm_path = ( + Path(lib.get("asm_dir", config.asm_dir)) / options["source"] + ).with_suffix(".s") + + link_built_obj = obj.completed + built_obj_path: Optional[Path] = None + if unit_src_path.exists(): + if unit_src_path.suffix in (".c", ".cp", ".cpp"): + # Add MWCC & host build rules + built_obj_path = c_build(obj, options, lib_name, unit_src_path) + elif unit_src_path.suffix == ".s": + # Add assembler build rule + built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + else: + sys.exit(f"Unknown source file type {unit_src_path}") + else: + if config.warn_missing_source or obj.completed: print(f"Missing source file {unit_src_path}") + link_built_obj = False + + # Assembly overrides + if unit_asm_path is not None and unit_asm_path.exists(): + link_built_obj = True + built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + + if link_built_obj and built_obj_path is not None: + # Use the source-built object + link_step.add(built_obj_path) + elif obj_path is not None: + # Use the original (extracted) object link_step.add(obj_path) - return - - mw_version = options["mw_version"] or lib["mw_version"] - cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_cflags_str(options["extra_cflags"]) - cflags_str += " " + extra_cflags_str - used_compiler_versions.add(mw_version) - - base_object = Path(obj.name).with_suffix("") - src_obj_path = build_src_path / f"{base_object}.o" - src_base_path = build_src_path / base_object - - if src_obj_path not in source_added: - source_added.add(src_obj_path) - - n.comment(f"{obj_name}: {lib_name} (linked {completed})") - n.build( - outputs=path(src_obj_path), - rule="mwcc_sjis" if options["shiftjis"] else "mwcc", - inputs=path(unit_src_path), - variables={ - "mw_version": path(Path(mw_version)), - "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": path(src_base_path), - }, - implicit=path( - mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit - ), - ) - - if lib["host"]: - host_obj_path = build_host_path / f"{base_object}.o" - host_base_path = build_host_path / base_object - n.build( - outputs=path(host_obj_path), - rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp", - inputs=path(unit_src_path), - variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": path(host_base_path), - }, - ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) - n.newline() - - if options["add_to_all"]: - source_inputs.append(src_obj_path) - - if completed: - obj_path = src_obj_path - link_step.add(obj_path) + else: + sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") # Add DOL link step link_step = LinkStep(build_config) @@ -626,7 +756,7 @@ def generate_build_ninja(config, build_config): sys.exit(f"Compiler {mw_path} does not exist") # Check if linker exists - mw_path = compilers / config.linker_version / "mwldeppc.exe" + mw_path = compilers / str(config.linker_version) / "mwldeppc.exe" if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") @@ -635,13 +765,25 @@ def generate_build_ninja(config, build_config): ### for step in link_steps: step.write(n) + link_outputs.append(step.output()) n.newline() ### # Generate RELs ### + n.comment("Generate REL(s)") + flags = "-w" + if len(build_config["links"]) > 1: + flags += " -q" + n.rule( + name="makerel", + command=f"{dtk} rel make {flags} -c $config $names @$rspfile", + description="REL", + rspfile="$rspfile", + rspfile_content="$in_newline", + ) generated_rels = [] - for link in build_config["links"]: + for idx, link in enumerate(build_config["links"]): # Map module names to link steps link_steps_local = list( filter( @@ -655,7 +797,7 @@ def generate_build_ninja(config, build_config): rels_to_generate = list( filter( lambda step: step.module_id != 0 - and not step.name in generated_rels, + and step.name not in generated_rels, link_steps_local, ) ) @@ -668,15 +810,23 @@ def generate_build_ninja(config, build_config): rels_to_generate, ) ) - n.comment("Generate RELs") + rel_names = list( + map( + lambda step: step.name, + link_steps_local, + ) + ) + rel_names_arg = " ".join(map(lambda name: f"-n {name}", rel_names)) n.build( - outputs=path(rel_outputs), + outputs=rel_outputs, rule="makerel", - inputs=path( - list(map(lambda step: step.partial_output(), link_steps_local)) - ), - implicit=path([dtk, config.config_path]), - variables={"config": path(config.config_path)}, + inputs=list(map(lambda step: step.partial_output(), link_steps_local)), + implicit=[dtk, config.config_path], + variables={ + "config": config.config_path, + "rspfile": config.out_path() / f"rel{idx}.rsp", + "names": rel_names_arg, + }, ) n.newline() @@ -687,7 +837,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source", rule="phony", - inputs=path(source_inputs), + inputs=source_inputs, ) n.newline() @@ -698,7 +848,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source_host", rule="phony", - inputs=path(host_source_inputs), + inputs=host_source_inputs, ) n.newline() @@ -714,10 +864,10 @@ def generate_build_ninja(config, build_config): description="CHECK $in", ) n.build( - outputs=path(ok_path), + outputs=ok_path, rule="check", - inputs=path(config.check_sha_path), - implicit=path([dtk, *map(lambda step: step.output(), link_steps)]), + inputs=config.check_sha_path, + implicit=[dtk, *link_outputs], ) n.newline() @@ -725,16 +875,15 @@ def generate_build_ninja(config, build_config): # Calculate progress ### n.comment("Calculate progress") - progress_path = build_path / "progress.json" n.rule( name="progress", command=f"$python {configure_script} $configure_args progress", description="PROGRESS", ) n.build( - outputs=path(progress_path), + outputs=progress_path, rule="progress", - implicit=path([ok_path, configure_script, python_lib, config.config_path]), + implicit=[ok_path, configure_script, python_lib, config.config_path], ) ### @@ -750,7 +899,7 @@ def generate_build_ninja(config, build_config): description=f"DIFF {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_diff", rule="dol_diff", ) @@ -768,10 +917,10 @@ def generate_build_ninja(config, build_config): description=f"APPLY {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_apply", rule="dol_apply", - implicit=path([ok_path]), + implicit=[ok_path], ) n.build( outputs="apply", @@ -792,11 +941,11 @@ def generate_build_ninja(config, build_config): deps="gcc", ) n.build( - inputs=path(config.config_path), - outputs=path(build_config_path), + inputs=config.config_path, + outputs=build_config_path, rule="split", - implicit=path(dtk), - variables={"out_dir": path(build_path)}, + implicit=dtk, + variables={"out_dir": build_path}, ) n.newline() @@ -813,14 +962,13 @@ def generate_build_ninja(config, build_config): n.build( outputs="build.ninja", rule="configure", - implicit=path( - [ - build_config_path, - configure_script, - python_lib, - Path(python_lib_dir) / "ninja_syntax.py", - ] - ), + implicit=[ + build_config_path, + configure_script, + python_lib, + python_lib_dir / "ninja_syntax.py", + *(config.reconfig_deps or []) + ], ) n.newline() @@ -829,9 +977,12 @@ def generate_build_ninja(config, build_config): ### n.comment("Default rule") if build_config: - n.default(path(progress_path)) + if config.non_matching: + n.default(link_outputs) + else: + n.default(progress_path) else: - n.default(path(build_config_path)) + n.default(build_config_path) # Write build.ninja with open("build.ninja", "w", encoding="utf-8") as f: @@ -840,12 +991,14 @@ def generate_build_ninja(config, build_config): # Generate objdiff.json -def generate_objdiff_config(config, build_config): +def generate_objdiff_config( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: if not build_config: return - objdiff_config = { - "min_version": "0.4.3", + objdiff_config: Dict[str, Any] = { + "min_version": "1.0.0", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -863,18 +1016,50 @@ def generate_objdiff_config(config, build_config): "units": [], } + # decomp.me compiler name mapping + # Commented out versions have not been added to decomp.me yet + COMPILER_MAP = { + "GC/1.0": "mwcc_233_144", + "GC/1.1": "mwcc_233_159", + "GC/1.2.5": "mwcc_233_163", + "GC/1.2.5e": "mwcc_233_163e", + "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3.2": "mwcc_242_81", + "GC/1.3.2r": "mwcc_242_81r", + "GC/2.0": "mwcc_247_92", + "GC/2.5": "mwcc_247_105", + "GC/2.6": "mwcc_247_107", + "GC/2.7": "mwcc_247_108", + "GC/3.0": "mwcc_41_60831", + # "GC/3.0a3": "mwcc_41_51213", + "GC/3.0a3.2": "mwcc_41_60126", + # "GC/3.0a3.3": "mwcc_41_60209", + # "GC/3.0a3.4": "mwcc_42_60308", + # "GC/3.0a5": "mwcc_42_60422", + "GC/3.0a5.2": "mwcc_41_60831", + "Wii/0x4201_127": "mwcc_42_142", + # "Wii/1.0": "mwcc_43_145", + # "Wii/1.0RC1": "mwcc_42_140", + "Wii/1.0a": "mwcc_42_142", + "Wii/1.1": "mwcc_43_151", + "Wii/1.3": "mwcc_43_172", + # "Wii/1.5": "mwcc_43_188", + "Wii/1.6": "mwcc_43_202", + "Wii/1.7": "mwcc_43_213", + } + build_path = config.out_path() - def add_unit(build_obj, module_name): + def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: if build_obj["autogenerated"]: # Skip autogenerated objects return obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") - unit_config = { - "name": unix_str(Path(module_name) / base_object), - "target_path": unix_str(obj_path), + unit_config: Dict[str, Any] = { + "name": Path(module_name) / base_object, + "target_path": obj_path, } result = config.find_object(obj_name) @@ -885,14 +1070,21 @@ def generate_objdiff_config(config, build_config): lib, obj = result src_dir = Path(lib.get("src_dir", config.src_dir)) - unit_src_path = src_dir / obj.options["source"] + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value + + unit_src_path = src_dir / str(options["source"]) if not unit_src_path.exists(): objdiff_config["units"].append(unit_config) return - cflags = obj.options["cflags"] or lib["cflags"] - src_obj_path = build_path / "src" / f"{base_object}.o" + cflags = options["cflags"] + src_obj_path = build_path / "src" / f"{obj.base_name}.o" + src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" reverse_fn_order = False if type(cflags) is list: @@ -905,9 +1097,32 @@ def generate_objdiff_config(config, build_config): elif value == "nodeferred": reverse_fn_order = False - unit_config["base_path"] = unix_str(src_obj_path) + # Filter out include directories + def keep_flag(flag): + return not flag.startswith("-i ") and not flag.startswith("-I ") + + cflags = list(filter(keep_flag, cflags)) + + # Add appropriate lang flag + if unit_src_path.suffix in (".cp", ".cpp"): + cflags.insert(0, "-lang=c++") + else: + cflags.insert(0, "-lang=c") + + unit_config["base_path"] = src_obj_path unit_config["reverse_fn_order"] = reverse_fn_order unit_config["complete"] = obj.completed + compiler_version = COMPILER_MAP.get(options["mw_version"]) + if compiler_version is None: + print(f"Missing scratch compiler mapping for {options['mw_version']}") + else: + unit_config["scratch"] = { + "platform": "gc_wii", + "compiler": compiler_version, + "c_flags": make_flags_str(cflags), + "ctx_path": src_ctx_path, + "build_ctx": True, + } objdiff_config["units"].append(unit_config) # Add DOL units @@ -921,32 +1136,36 @@ def generate_objdiff_config(config, build_config): # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: - json.dump(objdiff_config, w, indent=4) + + def unix_path(input: Any) -> str: + return str(input).replace(os.sep, "/") if input else "" + + json.dump(objdiff_config, w, indent=4, default=unix_path) # Calculate, print and write progress to progress.json -def calculate_progress(config): +def calculate_progress(config: ProjectConfig) -> None: out_path = config.out_path() build_config = load_build_config(config, out_path / "config.json") if not build_config: return class ProgressUnit: - def __init__(self, name): - self.name = name - self.code_total = 0 - self.code_fancy_frac = config.progress_code_fancy_frac - self.code_fancy_item = config.progress_code_fancy_item - self.code_progress = 0 - self.data_total = 0 - self.data_fancy_frac = config.progress_data_fancy_frac - self.data_fancy_item = config.progress_data_fancy_item - self.data_progress = 0 - self.objects_progress = 0 - self.objects_total = 0 - self.objects = set() + def __init__(self, name: str) -> None: + self.name: str = name + self.code_total: int = 0 + self.code_fancy_frac: int = config.progress_code_fancy_frac + self.code_fancy_item: str = config.progress_code_fancy_item + self.code_progress: int = 0 + self.data_total: int = 0 + self.data_fancy_frac: int = config.progress_data_fancy_frac + self.data_fancy_item: str = config.progress_data_fancy_item + self.data_progress: int = 0 + self.objects_progress: int = 0 + self.objects_total: int = 0 + self.objects: Set[Object] = set() - def add(self, build_obj): + def add(self, build_obj: Dict[str, Any]) -> None: self.code_total += build_obj["code_size"] self.data_total += build_obj["data_size"] @@ -973,10 +1192,10 @@ def calculate_progress(config): if include_object: self.objects_progress += 1 - def code_frac(self): + def code_frac(self) -> float: return self.code_progress / self.code_total - def data_frac(self): + def data_frac(self) -> float: return self.data_progress / self.data_total # Add DOL units @@ -989,7 +1208,7 @@ def calculate_progress(config): # Add REL units rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress = [] + modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: progress = ProgressUnit(module["name"]) modules_progress.append(progress) @@ -1003,7 +1222,7 @@ def calculate_progress(config): # Print human-readable progress print("Progress:") - def print_category(unit): + def print_category(unit: Optional[ProgressUnit]) -> None: if unit is None: return @@ -1037,9 +1256,9 @@ def calculate_progress(config): print_category(progress) # Generate and write progress.json - progress_json = {} + progress_json: Dict[str, Any] = {} - def add_category(category, unit): + def add_category(category: str, unit: ProgressUnit) -> None: progress_json[category] = { "code": unit.code_progress, "code/total": unit.code_total, diff --git a/tools/transform_dep.py b/tools/transform_dep.py index 86bd2ecb..124de04b 100644 --- a/tools/transform_dep.py +++ b/tools/transform_dep.py @@ -25,7 +25,7 @@ def in_wsl() -> bool: return "microsoft-standard" in uname().release -def import_d_file(in_file) -> str: +def import_d_file(in_file: str) -> str: out_text = "" with open(in_file) as file: @@ -60,7 +60,7 @@ def import_d_file(in_file) -> str: return out_text -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="""Transform a .d file from Wine paths to normal paths""" ) @@ -81,4 +81,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()