Regenerate orphaned symbols in the CI (#625)

[As
mentioned](https://discord.com/channels/1079389589950705684/1135205782703570994/1154948845638254672),
since #595 some tools are not able to pick up some function names as the
symbol names are gone. This PR adds a quick tool to re-generate that
list from a map file and integrates it in the CI flow.

EDIT: CI is failing because the running Linter is from `master`, where
`mapfile-parser` is not installed.

.github/workflows/format.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install black
-        run: pip install black pyyaml
+        run: pip install black pyyaml mapfile-parser==2.1.4
       - name: Clone main repository
         uses: actions/checkout@v3
         with:

.github/workflows/linter.yaml

@@ -18,7 +18,7 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install black
-        run: pip install black pyyaml
+        run: pip install black pyyaml mapfile-parser==2.1.4
       - name: Clone main repo (PR)
         if: github.event_name == 'pull_request_target'
         uses: actions/checkout@v3

.github/workflows/validate-and-report.yml

@@ -85,7 +85,9 @@ jobs:
         run: make check
       - name: Analyze calls dry run
         if: matrix.version == 'us'
-        run: make force_extract && ./tools/analyze_calls.py --ultradry
+        run: |
+          make force_extract
+          ./tools/analyze_calls.py --ultradry
       - name: Remove clutter from build folder
         run: rm -rf build/${{ matrix.version }}/asm build/${{ matrix.version }}/src build/${{ matrix.version }}/assets
       - name: Export build folder
@@ -191,17 +193,23 @@ jobs:
             wait
       - name: Generate function calls chart
         run: |
+            make force_symbols
             make force_extract
             python3 tools/analyze_calls.py
             git clean -fdx asm/ 
       - name: Generate function report
         run: |
+            git checkout config/
             make -j extract
             python3 tools/function_finder/function_finder_psx.py --use-call-trees > gh-duplicates/functions.md
             rm -rf gh-duplicates/function_calls/ || true
             mv function_calls gh-duplicates/
       - name: Generate duplicates report
-        run: make force_extract && cd tools/dups && cargo run --release -- --threshold .90 --output-file ../../gh-duplicates/duplicates.txt
+        run: |
+          make force_symbols
+          make force_extract
+          cd tools/dups
+          cargo run --release -- --threshold .90 --output-file ../../gh-duplicates/duplicates.txt
       - name: Commit all reports
         run: |
             git config --global user.name 'GitHub Action' 

Makefile

@@ -395,6 +395,21 @@ force_extract:
 	rm -rf src/
 	mv src_tmp src
 
+# Rewrites symbol list from a successful build
+force_symbols:
+	$(PYTHON) ./tools/symbols.py map build/us/dra.map > config/symbols.us.dra.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stcen.map > config/symbols.us.stcen.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stdre.map > config/symbols.us.stdre.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stmad.map > config/symbols.us.stmad.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stno3.map > config/symbols.us.stno3.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stnp3.map > config/symbols.us.stnp3.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stnz0.map > config/symbols.us.stnz0.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stsel.map > config/symbols.us.stsel.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stst0.map > config/symbols.us.stst0.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/stwrp.map > config/symbols.us.stwrp.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/strwrp.map > config/symbols.us.strwrp.txt --no-default
+	$(PYTHON) ./tools/symbols.py map build/us/tt_000.map > config/symbols.us.tt_000.txt --no-default
+
 context:
 	$(M2CTX) $(SOURCE)
 	@echo ctx.c has been updated.

tools/requirements-python.txt

@@ -7,7 +7,7 @@ intervaltree
 colorama
 python-Levenshtein
 cxxfilt
-mapfile-parser==2.1.1
+mapfile-parser==2.1.4
 tabulate
 requests
 graphviz

tools/symbols.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 
 import argparse
+import mapfile_parser
 import os
+from pathlib import Path
 import re
 import sys
 import yaml
@@ -13,6 +15,7 @@ subparsers = parser.add_subparsers(dest="command")
 sort_parser = subparsers.add_parser(
     "sort", description="Sort all the symbols of a given GNU LD script by their offset"
 )
+
 cross_parser = subparsers.add_parser(
     "cross",
     description="Cross-reference the symbols between two assembly files and print the result to stdout for GNU LD. Useful to cross-reference symbols between different overlays or game revisions. The assemblies must be identical.",
@@ -25,6 +28,7 @@ cross_parser.add_argument(
     "to_cross",
     help="Assembly source file to be cross-referenced to",
 )
+
 orphan_parser = subparsers.add_parser(
     "remove-orphans",
     description="Remove all symbols that are not referenced from a specific group of assembly code",
@@ -34,6 +38,21 @@ orphan_parser.add_argument(
     help="The Splat YAML config of the overlay to remove the orphan symbols from",
 )
 
+map_parser = subparsers.add_parser(
+    "map",
+    description="Print the list of symbols from a map file",
+)
+map_parser.add_argument(
+    "map_file_name",
+    help="The map file to extract the symbols from",
+)
+map_parser.add_argument(
+    "--no-default",
+    required=False,
+    action="store_true",
+    help="Do not include Splat default symbols that starts with D_ or func_",
+)
+
 args = parser.parse_args()
 if args.version == None:
     args.version = os.getenv("VERSION")
@@ -41,6 +60,15 @@ if args.version == None:
         args.version = "us"
 
 
+def is_splat_symbol_name(name):
+    return (
+        name.startswith("D_")
+        or name.startswith("func_")
+        or name.startswith("jpt_")
+        or name.startswith("jtbl_")
+    )
+
+
 def add_newline_if_missing(list):
     if not list[-1].endswith("\n"):
         list[-1] += "\n"
@@ -315,6 +343,26 @@ def remove_orphans_from_config(config_yaml):
     remove_orphans(symbol_file_name, symbols_found)
 
 
+def print_map_symbols(map_file_name, no_default):
+    map_file = mapfile_parser.MapFile()
+    map_file.readMapFile(Path(map_file_name))
+
+    filter = (
+        (lambda name: not is_splat_symbol_name(name))
+        if no_default
+        else (lambda _: True)
+    )
+
+    syms = dict()
+    for segment in map_file:
+        for file in segment:
+            for sym in file:
+                if sym.vram not in syms and filter(sym.name):
+                    syms[sym.vram] = sym.name
+    for vram in syms:
+        print(f"{syms[vram]} = 0x{vram:08X};")
+
+
 if __name__ == "__main__":
     if args.command == "sort":
         sort("config/")
@@ -322,3 +370,5 @@ if __name__ == "__main__":
         cross(args.ref, args.to_cross)
     elif args.command == "remove-orphans":
         remove_orphans_from_config(args.config_yaml)
+    elif args.command == "map":
+        print_map_symbols(args.map_file_name, args.no_default)