diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 62ecd652a..d1ced86ff 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -26,7 +26,7 @@ jobs: with: python-version: '3.10' - name: Install black - run: pip install black pyyaml + run: pip install black pyyaml mapfile-parser==2.1.4 - name: Clone main repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 7915a63e0..9191f09d9 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -18,7 +18,7 @@ jobs: with: python-version: '3.10' - name: Install black - run: pip install black pyyaml + run: pip install black pyyaml mapfile-parser==2.1.4 - name: Clone main repo (PR) if: github.event_name == 'pull_request_target' uses: actions/checkout@v3 diff --git a/.github/workflows/validate-and-report.yml b/.github/workflows/validate-and-report.yml index 94494fdf0..a3c33e23b 100644 --- a/.github/workflows/validate-and-report.yml +++ b/.github/workflows/validate-and-report.yml @@ -85,7 +85,9 @@ jobs: run: make check - name: Analyze calls dry run if: matrix.version == 'us' - run: make force_extract && ./tools/analyze_calls.py --ultradry + run: | + make force_extract + ./tools/analyze_calls.py --ultradry - name: Remove clutter from build folder run: rm -rf build/${{ matrix.version }}/asm build/${{ matrix.version }}/src build/${{ matrix.version }}/assets - name: Export build folder @@ -191,17 +193,23 @@ jobs: wait - name: Generate function calls chart run: | + make force_symbols make force_extract python3 tools/analyze_calls.py git clean -fdx asm/ - name: Generate function report run: | + git checkout config/ make -j extract python3 tools/function_finder/function_finder_psx.py --use-call-trees > gh-duplicates/functions.md rm -rf gh-duplicates/function_calls/ || true mv function_calls gh-duplicates/ - name: Generate duplicates report - run: make force_extract && cd tools/dups && cargo run --release -- --threshold .90 --output-file ../../gh-duplicates/duplicates.txt + run: | + make force_symbols + make force_extract + cd tools/dups + cargo run --release -- --threshold .90 --output-file ../../gh-duplicates/duplicates.txt - name: Commit all reports run: | git config --global user.name 'GitHub Action' diff --git a/Makefile b/Makefile index 245113e39..f2ad9c15f 100644 --- a/Makefile +++ b/Makefile @@ -395,6 +395,21 @@ force_extract: rm -rf src/ mv src_tmp src +# Rewrites symbol list from a successful build +force_symbols: + $(PYTHON) ./tools/symbols.py map build/us/dra.map > config/symbols.us.dra.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stcen.map > config/symbols.us.stcen.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stdre.map > config/symbols.us.stdre.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stmad.map > config/symbols.us.stmad.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stno3.map > config/symbols.us.stno3.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stnp3.map > config/symbols.us.stnp3.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stnz0.map > config/symbols.us.stnz0.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stsel.map > config/symbols.us.stsel.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stst0.map > config/symbols.us.stst0.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/stwrp.map > config/symbols.us.stwrp.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/strwrp.map > config/symbols.us.strwrp.txt --no-default + $(PYTHON) ./tools/symbols.py map build/us/tt_000.map > config/symbols.us.tt_000.txt --no-default + context: $(M2CTX) $(SOURCE) @echo ctx.c has been updated. diff --git a/tools/requirements-python.txt b/tools/requirements-python.txt index f0c30e67a..e2d82c567 100644 --- a/tools/requirements-python.txt +++ b/tools/requirements-python.txt @@ -7,7 +7,7 @@ intervaltree colorama python-Levenshtein cxxfilt -mapfile-parser==2.1.1 +mapfile-parser==2.1.4 tabulate requests graphviz diff --git a/tools/symbols.py b/tools/symbols.py index 268906dc8..cd39a96fa 100755 --- a/tools/symbols.py +++ b/tools/symbols.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import argparse +import mapfile_parser import os +from pathlib import Path import re import sys import yaml @@ -13,6 +15,7 @@ subparsers = parser.add_subparsers(dest="command") sort_parser = subparsers.add_parser( "sort", description="Sort all the symbols of a given GNU LD script by their offset" ) + cross_parser = subparsers.add_parser( "cross", description="Cross-reference the symbols between two assembly files and print the result to stdout for GNU LD. Useful to cross-reference symbols between different overlays or game revisions. The assemblies must be identical.", @@ -25,6 +28,7 @@ cross_parser.add_argument( "to_cross", help="Assembly source file to be cross-referenced to", ) + orphan_parser = subparsers.add_parser( "remove-orphans", description="Remove all symbols that are not referenced from a specific group of assembly code", @@ -34,6 +38,21 @@ orphan_parser.add_argument( help="The Splat YAML config of the overlay to remove the orphan symbols from", ) +map_parser = subparsers.add_parser( + "map", + description="Print the list of symbols from a map file", +) +map_parser.add_argument( + "map_file_name", + help="The map file to extract the symbols from", +) +map_parser.add_argument( + "--no-default", + required=False, + action="store_true", + help="Do not include Splat default symbols that starts with D_ or func_", +) + args = parser.parse_args() if args.version == None: args.version = os.getenv("VERSION") @@ -41,6 +60,15 @@ if args.version == None: args.version = "us" +def is_splat_symbol_name(name): + return ( + name.startswith("D_") + or name.startswith("func_") + or name.startswith("jpt_") + or name.startswith("jtbl_") + ) + + def add_newline_if_missing(list): if not list[-1].endswith("\n"): list[-1] += "\n" @@ -315,6 +343,26 @@ def remove_orphans_from_config(config_yaml): remove_orphans(symbol_file_name, symbols_found) +def print_map_symbols(map_file_name, no_default): + map_file = mapfile_parser.MapFile() + map_file.readMapFile(Path(map_file_name)) + + filter = ( + (lambda name: not is_splat_symbol_name(name)) + if no_default + else (lambda _: True) + ) + + syms = dict() + for segment in map_file: + for file in segment: + for sym in file: + if sym.vram not in syms and filter(sym.name): + syms[sym.vram] = sym.name + for vram in syms: + print(f"{syms[vram]} = 0x{vram:08X};") + + if __name__ == "__main__": if args.command == "sort": sort("config/") @@ -322,3 +370,5 @@ if __name__ == "__main__": cross(args.ref, args.to_cross) elif args.command == "remove-orphans": remove_orphans_from_config(args.config_yaml) + elif args.command == "map": + print_map_symbols(args.map_file_name, args.no_default)