Debug in elf (#491)

* git subrepo pull (merge) --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "e5838f0b06"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "e5838f0b06"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "2f68596"

* Allow storage of extra debug info in elf

* Fix string

* Add comment detailing purpose of genobjcopy

tools/build/configure.py

@@ -32,7 +32,7 @@ def exec_shell(command: List[str]) -> str:
     return ret.stdout
 
 def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str, cppflags: str, extra_cflags: str, use_ccache: bool,
-                      non_matching: bool):
+                      non_matching: bool, debug: bool):
     # platform-specific
     if sys.platform  == "darwin":
         iconv = "tools/iconv.py UTF-8 SHIFT-JIS"
@@ -78,9 +78,17 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str, cppflags: str, extra
         command=f"{cross}ld -T ver/$version/build/undefined_syms.txt -T ver/$version/undefined_syms_auto.txt -T ver/$version/undefined_funcs_auto.txt -Map $mapfile --no-check-sections -T $in -o $out",
     )
 
+    objcopy_sections = ""
+    if debug:
+        ninja.rule("genobjcopy",
+            description="generate $out",
+            command=f"$python {BUILD_TOOLS}/genobjcopy.py $in $out",
+        )
+        objcopy_sections = "@ver/$version/build/objcopy_sections.txt "
+
     ninja.rule("z64",
         description="rom $out",
-        command=f"{cross}objcopy $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out",
+        command=f"{cross}objcopy {objcopy_sections} $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out",
     )
 
     ninja.rule("sha1sum",
@@ -227,7 +235,7 @@ class Configure:
         self.version_path = ROOT / f"ver/{version}"
         self.linker_entries = None
 
-    def split(self, assets: bool, code: bool):
+    def split(self, assets: bool, code: bool, debug: bool):
         import split
 
         modes = ["ld"]
@@ -237,12 +245,16 @@ class Configure:
         if code:
             modes.extend(["code", "c", "data", "rodata"])
 
+        splat_file = [str(self.version_path / "splat.yaml")]
+        if debug:
+            splat_file += [str(self.version_path / "splat-debug.yaml")]
+
         split.main(
-            str(self.version_path / "splat.yaml"),
+            splat_file,
             None,
             str(self.version_path / "baserom.z64"),
             modes,
-            verbose=False,
+            verbose=True,
         )
         self.linker_entries = split.linker_writer.entries[:]
         self.asset_stack = split.config["asset_stack"]
@@ -250,6 +262,9 @@ class Configure:
     def build_path(self) -> Path:
         return Path(f"ver/{self.version}/build")
 
+    def objcopy_sections_path(self) -> Path:
+        return self.build_path() / "objcopy_sections.txt"
+
     def undefined_syms_path(self) -> Path:
         return self.build_path() / "undefined_syms.txt"
 
@@ -296,7 +311,7 @@ class Configure:
         # ¯\_(ツ)_/¯
         return path
 
-    def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str], non_matching: bool):
+    def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str], non_matching: bool, debug: bool):
         import segtypes
         import segtypes.common.data
         import segtypes.n64.Yay0
@@ -615,6 +630,14 @@ class Configure:
             else:
                 raise Exception(f"don't know how to build {seg.__class__.__name__} '{seg.name}'")
 
+        # Create objcopy section list
+        if debug:
+            ninja.build(
+                str(self.objcopy_sections_path()),
+                "genobjcopy",
+                str(self.build_path() / "elf_sections.txt"),
+            )
+
         # Run undefined_syms through cpp
         ninja.build(
             str(self.undefined_syms_path()),
@@ -623,11 +646,15 @@ class Configure:
         )
 
         # Build elf, z64, ok
+        additional_objects = [str(self.undefined_syms_path())]
+        if debug:
+            additional_objects += [str(self.objcopy_sections_path())]
+
         ninja.build(
             str(self.elf_path()),
             "ld",
             str(self.linker_script_path()),
-            implicit=[str(obj) for obj in built_objects] + [str(self.undefined_syms_path())],
+            implicit=[str(obj) for obj in built_objects] +  additional_objects,
             variables={ "version": self.version, "mapfile": str(self.map_path()) },
         )
         ninja.build(
@@ -635,6 +662,7 @@ class Configure:
             "z64",
             str(self.elf_path()),
             implicit=[CRC_TOOL],
+            variables={ "version": self.version },
         )
         ninja.build(
             str(self.rom_ok_path()),
@@ -731,14 +759,14 @@ if __name__ == "__main__":
         cflags += " -g1"
 
     if not args.no_warn:
-        cflags += "-Wuninitialized -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes"
+        cflags += " -Wuninitialized -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes"
 
     # add splat to python import path
     sys.path.append(str((ROOT / args.splat).resolve()))
 
     ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999)
 
-    write_ninja_rules(ninja, args.cpp or "cpp", cppflags, cflags, args.ccache, args.non_matching)
+    write_ninja_rules(ninja, args.cpp or "cpp", cppflags, cflags, args.ccache, args.non_matching, args.debug)
     write_ninja_for_tools(ninja)
 
     skip_files = set()
@@ -753,8 +781,8 @@ if __name__ == "__main__":
         if not first_configure:
             first_configure = configure
 
-        configure.split(not args.no_split_assets, args.split_code)
+        configure.split(not args.no_split_assets, args.split_code, args.debug)
-        configure.write_ninja(ninja, skip_files, args.non_matching)
+        configure.write_ninja(ninja, skip_files, args.non_matching, args.debug)
 
         all_rom_oks.append(str(configure.rom_ok_path()))
 

tools/build/genobjcopy.py

@@ -0,0 +1,23 @@
+#! /usr/bin/python3
+import sys, os
+
+#Under normal compilation we rely on splat to use a discard option in the ldscript
+#to not include sections in the elf then just output all sections, however under debug we want
+#to have debug sections.
+#In debugging mode splat is told to output a list of sections it is custom creating, which are
+#all of the sections we export to the z64 file with an objcopy. The below chunk of code is
+#responsible for adding -j to each of the names and outputting a file for objcopy to use
+#so we can still generate a elf file with all the extra debugging sections and still output
+# the required sections to the .z64 without outputting everything.
+
+if __name__ == "__main__":
+	infile, outfile = sys.argv[1:]
+
+	#generate output based on input
+	file_data = open(infile,"r").read().split("\n")
+	if len(file_data[-1]) == 0:
+		file_data.pop()
+
+	outdata = "-j " + " -j ".join(file_data)
+	with open(outfile, "w") as f:
+		f.write(outdata)

tools/splat/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/ethteck/splat.git
 	branch = master
-	commit = 0efa552c5d3cf866b6325486868714787261673d
+	commit = e5838f0b063425e843d44091b8654962995829bd
-	parent = 35fa67cd8de20bf6dacdb92f1ac8411d3e5f09cf
+	parent = a1965af227c11f4dde5a7114352db3d24f0bc6a9
 	method = merge
 	cmdver = 0.4.3

tools/splat/segtypes/linker_entry.py

@@ -66,6 +66,7 @@ class LinkerEntry:
 class LinkerWriter():
     def __init__(self):
         self.shiftable: bool = options.get("shiftable", False)
+        self.linker_discard_section: bool = options.get("linker_discard_section", True)
         self.entries: List[LinkerEntry] = []
 
         self.buffer: List[str] = []
@@ -154,10 +155,11 @@ class LinkerWriter():
         self._end_segment(segment)
 
     def save_linker_script(self):
-        self._writeln("/DISCARD/ :")
+        if self.linker_discard_section:
-        self._begin_block()
+            self._writeln("/DISCARD/ :")
-        self._writeln("*(*);")
+            self._begin_block()
-        self._end_block()
+            self._writeln("*(*);")
+            self._end_block()
 
         self._end_block() # SECTIONS
 

tools/splat/split.py

@@ -8,7 +8,7 @@ import yaml
 import pickle
 from colorama import Style, Fore
 from segtypes.segment import Segment
-from segtypes.linker_entry import LinkerWriter
+from segtypes.linker_entry import LinkerWriter, to_cname
 from util import log
 from util import options
 from util import symbols
@@ -17,7 +17,7 @@ from util import palettes
 VERSION = "0.7.10"
 
 parser = argparse.ArgumentParser(description="Split a rom given a rom, a config, and output directory")
-parser.add_argument("config", help="path to a compatible config .yaml file")
+parser.add_argument("config", help="path to a compatible config .yaml file", nargs='+')
 parser.add_argument("--target", help="path to a file to split (.z64 rom)")
 parser.add_argument("--basedir", help="a directory in which to extract the rom")
 parser.add_argument("--modes", nargs="+", default="all")
@@ -109,14 +109,42 @@ def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached):
             log.write(f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached")
     log.write(f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files")
 
+def merge_configs(main_config, additional_config):
+    # Merge rules are simple
+    # For each key in the dictionary
+    # - If list then append to list
+    # - If a dictionary then repeat merge on sub dictionary entries
+    # - Else assume string or number and replace entry
+
+    for curkey in additional_config:
+        if curkey not in main_config:
+            main_config[curkey] = additional_config[curkey]
+        elif type(main_config[curkey]) != type(additional_config[curkey]):
+            log.error(f"Type for key {curkey} in configs does not match")
+        else:
+            # keys exist and match, see if a list to append
+            if type(main_config[curkey]) == list:
+                main_config[curkey] += additional_config[curkey]
+            elif type(main_config[curkey]) == dict:
+                #need to merge sub areas
+                main_config[curkey] = merge_configs(main_config[curkey], additional_config[curkey])
+            else:
+                #not a list or dictionary, must be a number or string, overwrite
+                main_config[curkey] = additional_config[curkey]
+
+    return main_config
+
 def main(config_path, base_dir, target_path, modes, verbose, use_cache=True):
     global config
 
     log.write(f"splat {VERSION}")
 
     # Load config
-    with open(config_path) as f:
+    config = {}
-        config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+    for entry in config_path:
+        with open(entry) as f:
+            additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+        config = merge_configs(config, additional_config)
 
     options.initialize(config, config_path, base_dir, target_path)
     options.set("modes", modes)
@@ -236,6 +264,15 @@ def main(config_path, base_dir, target_path, modes, verbose, use_cache=True):
         linker_writer.save_linker_script()
         linker_writer.save_symbol_header()
 
+        # write elf_sections.txt - this only lists the generated sections in the elf, not sub sections
+        # that the elf combines into one section
+        if options.get_create_elf_section_list_auto():
+            section_list = ""
+            for segment in all_segments:
+                section_list += "." + to_cname(segment.name) + "\n"
+            with open(options.get_elf_section_list_path(), "w", newline="\n") as f:
+                f.write(section_list)
+
     # Write undefined_funcs_auto.txt
     if options.get_create_undefined_funcs_auto():
         to_write = [s for s in symbols.all_symbols if s.referenced and not s.defined and not s.dead and s.type == "func"]

tools/splat/util/options.py

@@ -4,7 +4,7 @@ from util import log
 
 opts = {}
 
-def initialize(config: Dict, config_path: str, base_path=None, target_path=None):
+def initialize(config: Dict, config_path, base_path=None, target_path=None):
     global opts
     opts = dict(config.get("options", {}))
 
@@ -14,7 +14,7 @@ def initialize(config: Dict, config_path: str, base_path=None, target_path=None)
         if not "base_path" in opts:
             log.error("Error: Base output dir not specified as a command line arg or via the config yaml (base_path)")
 
-        opts["base_path"] = Path(config_path).parent / opts["base_path"]
+        opts["base_path"] = Path(config_path[0]).parent / opts["base_path"]
 
     if not target_path:
         if "target_path" not in opts:
@@ -77,6 +77,12 @@ def get_create_undefined_syms_auto() -> bool:
 def get_undefined_syms_auto_path():
     return get_base_path() / opts.get("undefined_syms_auto_path", "undefined_syms_auto.txt")
 
+def get_create_elf_section_list_auto():
+    return opts.get("create_elf_section_list_auto", False)
+
+def get_elf_section_list_path():
+    return get_base_path() / opts.get("elf_section_list_path", "elf_sections.txt")
+
 def get_symbol_addrs_path():
     return get_base_path() / opts.get("symbol_addrs_path", "symbol_addrs.txt")
 

ver/jp/splat-debug.yaml

@@ -0,0 +1,4 @@
+options:
+  linker_discard_section: False
+  create_elf_section_list_auto: True
+  elf_section_list_path: ver/jp/build/elf_sections.txt

ver/us/splat-debug.yaml

@@ -0,0 +1,4 @@
+options:
+  linker_discard_section: False
+  create_elf_section_list_auto: True
+  elf_section_list_path: ver/us/build/elf_sections.txt