mirror of
https://github.com/zeldaret/mm.git
synced 2024-11-22 20:39:49 +00:00
Bss script update (#1601)
* port back AF's version * Update to use mapfile_parser 2 * Allow to specify other sections * hardcode map path * eprint * -r * review * review * Use fixed version * Update docs/tutorial/vscode.md Co-authored-by: Derek Hensley <hensley.derek58@gmail.com> * review * Update tools/check_reordering.py Co-authored-by: Derek Hensley <hensley.derek58@gmail.com> * update build path --------- Co-authored-by: Derek Hensley <hensley.derek58@gmail.com>
This commit is contained in:
Anghelo Carvajal
GitHub
parent
ef6323b9cd
commit
f4e8ec14e9
@ -12,7 +12,7 @@ pyelftools>=0.26
|
||||
# diff
|
||||
watchdog>=0.10.2
|
||||
argcomplete
|
||||
python-Levenshtein
|
||||
Levenshtein
|
||||
cxxfilt
|
||||
|
||||
# permuter
|
||||
@ -20,4 +20,4 @@ pycparser
|
||||
toml
|
||||
|
||||
# map parsing
|
||||
mapfile-parser>=1.2.1,<2.0.0
|
||||
mapfile-parser>=2.3.5,<3.0.0
|
||||
|
||||
168
tools/check_reordering.py
Executable file
168
tools/check_reordering.py
Executable file
@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: © 2024 ZeldaRET
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import colorama
|
||||
colorama.init()
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import mapfile_parser
|
||||
from pathlib import Path
|
||||
|
||||
def eprint(*args, **kwargs):
|
||||
print(*args, file=sys.stderr, **kwargs)
|
||||
|
||||
def mapPathToSource(origName: Path) -> Path:
|
||||
# Try to map built path to the source path
|
||||
parts = origName.parts
|
||||
if parts[0] == "build":
|
||||
parts = parts[1:]
|
||||
|
||||
path = Path(*parts)
|
||||
# Assume every file in the asm folder has .s extension, while everything else has .c extension
|
||||
if path.parts[0] == "asm":
|
||||
path = path.with_suffix(".s")
|
||||
else:
|
||||
path = path.with_suffix(".c")
|
||||
return path
|
||||
|
||||
def compareMapFiles(mapFileBuild: Path, mapFileExpected: Path, section: str=".bss", reverseCheck: bool=True) -> mapfile_parser.MapsComparisonInfo:
|
||||
eprint(f"Build mapfile: {mapFileBuild}")
|
||||
eprint(f"Expected mapfile: {mapFileExpected}")
|
||||
eprint("")
|
||||
|
||||
if not mapFileBuild.exists():
|
||||
eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you forget to build the rom?")
|
||||
exit(1)
|
||||
|
||||
if not mapFileExpected.exists():
|
||||
eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. A mapfile under the 'expected' folder is required")
|
||||
exit(1)
|
||||
|
||||
buildMap = mapfile_parser.MapFile()
|
||||
buildMap.readMapFile(mapFileBuild)
|
||||
if section != "all":
|
||||
buildMap = buildMap.filterBySectionType(section)
|
||||
|
||||
expectedMap = mapfile_parser.MapFile()
|
||||
expectedMap.readMapFile(mapFileExpected)
|
||||
if section != "all":
|
||||
expectedMap = expectedMap.filterBySectionType(section)
|
||||
|
||||
return buildMap.compareFilesAndSymbols(expectedMap, checkOtherOnSelf=reverseCheck)
|
||||
|
||||
|
||||
def printSymbolComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, printAll = True):
|
||||
print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
|
||||
|
||||
# If it's bad or missing, don't need to do anything special.
|
||||
# If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
|
||||
|
||||
for symbolInfo in comparisonInfo.comparedList:
|
||||
buildFile = symbolInfo.buildFile.filepath if symbolInfo.buildFile is not None else None
|
||||
expectedFile = symbolInfo.expectedFile.filepath if symbolInfo.expectedFile is not None else None
|
||||
|
||||
buildFileName = ""
|
||||
if buildFile is not None:
|
||||
buildFileName = mapPathToSource(buildFile)
|
||||
|
||||
expectedFileName = ""
|
||||
if expectedFile is not None:
|
||||
expectedFileName = mapPathToSource(expectedFile)
|
||||
|
||||
symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET
|
||||
if symbolInfo.diff is None:
|
||||
symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET
|
||||
print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff},{symbolGood}")
|
||||
continue
|
||||
|
||||
if symbolInfo.diff == 0:
|
||||
symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET
|
||||
if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
|
||||
if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
|
||||
if not printAll:
|
||||
continue
|
||||
|
||||
if buildFile != expectedFile:
|
||||
symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET
|
||||
print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff:X},{symbolGood}")
|
||||
|
||||
|
||||
def printFileComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, fun_allowed: bool = True):
|
||||
eprint("")
|
||||
|
||||
if len(comparisonInfo.badFiles) != 0:
|
||||
print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL)
|
||||
|
||||
for file in comparisonInfo.badFiles:
|
||||
eprint(f"Symbol reordering in {mapPathToSource(file.filepath)}")
|
||||
eprint("")
|
||||
|
||||
if fun_allowed:
|
||||
eprint(colorama.Fore.LIGHTWHITE_EX +
|
||||
" Symbols are REORDERED!!\n"
|
||||
" Oh! MY GOD!!"
|
||||
+ colorama.Style.RESET_ALL)
|
||||
eprint("")
|
||||
|
||||
if len(comparisonInfo.missingFiles) != 0:
|
||||
print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL)
|
||||
|
||||
for file in comparisonInfo.missingFiles:
|
||||
eprint(f"Symbols missing from {mapPathToSource(file.filepath)}")
|
||||
eprint("")
|
||||
|
||||
if fun_allowed:
|
||||
eprint(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL)
|
||||
eprint("")
|
||||
|
||||
eprint("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'")
|
||||
|
||||
|
||||
def main():
|
||||
description = "Check that globally visible symbols has not been reordered."
|
||||
epilog = """\
|
||||
N.B. Since this script reads the map files, it can only see globally visible symbols; in-function static symbols must be examined with other tools.
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument("-v", "--version", help="MM version to check. Defaults to n64-us", default="n64-us")
|
||||
parser.add_argument("-a", "--print-all", help="Print all symbols of the section, not just non-matching.", action="store_true")
|
||||
parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true")
|
||||
parser.add_argument("-r", "--no-reverse-check", help="Disable looking for symbols on the expected map that are missing on the built map file.", action="store_true")
|
||||
parser.add_argument("-s", "--section", help="Specify which section should be checked for reordered symbols. Use `all` to check all sections. Defaults to .bss", default=".bss", choices=[".text", ".data", ".rodata", ".bss", "all"])
|
||||
args = parser.parse_args()
|
||||
|
||||
mapfilePath = Path(f"build/{args.version}/mm-{args.version}.map")
|
||||
mapfileExpectedPath = "expected" / mapfilePath
|
||||
reverseCheck: bool = not args.no_reverse_check
|
||||
section: str = args.section
|
||||
|
||||
comparisonInfo = compareMapFiles(mapfilePath, mapfileExpectedPath, section, reverseCheck)
|
||||
printSymbolComparison(comparisonInfo, args.print_all)
|
||||
|
||||
if len(comparisonInfo.badFiles) + len(comparisonInfo.missingFiles) != 0:
|
||||
printFileComparison(comparisonInfo, not args.no_fun_allowed)
|
||||
return 1
|
||||
|
||||
eprint("")
|
||||
eprint(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL)
|
||||
|
||||
if args.no_fun_allowed:
|
||||
return 0
|
||||
|
||||
eprint("\n" + colorama.Fore.LIGHTWHITE_EX +
|
||||
colorama.Back.RED + f" " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + f" CONGRATURATIONS! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + "{:^34}".format(f"All Global {section if section != 'all' else 'Symbols'} is correct.") + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + f" THANK YOU! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + f" You are great decomper! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + f" " + colorama.Style.RESET_ALL )
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
ret = main()
|
||||
exit(ret)
|
||||
@ -1,207 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import colorama
|
||||
colorama.init()
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import collections
|
||||
|
||||
regex_fileDataEntry = re.compile(r"^\s+(?P<section>[^\s]+)\s+(?P<vram>0x[^\s]+)\s+(?P<size>0x[^\s]+)\s+(?P<name>[^\s]+)$")
|
||||
regex_bssEntry = re.compile(r"^\s+(?P<vram>0x[^\s]+)\s+(?P<name>[^\s]+)$")
|
||||
regex_label = re.compile(r"^(?P<name>L[0-9A-F]{8})$")
|
||||
|
||||
VarInfo = collections.namedtuple("VarInfo", ["file", "vram"])
|
||||
|
||||
def parseMapFile(mapPath: str):
|
||||
with open(mapPath) as f:
|
||||
mapData = f.read()
|
||||
startIndex = mapData.find("..makerom")
|
||||
mapData = mapData[startIndex:]
|
||||
# print(len(mapData))
|
||||
|
||||
symbolsDict = collections.OrderedDict()
|
||||
|
||||
inFile = False
|
||||
currentFile = ""
|
||||
|
||||
mapLines = mapData.split("\n")
|
||||
for line in mapLines:
|
||||
if inFile:
|
||||
if line.startswith(" "):
|
||||
entryMatch = regex_bssEntry.search(line)
|
||||
|
||||
# Find variable
|
||||
if entryMatch is not None:
|
||||
varName = entryMatch["name"]
|
||||
varVram = int(entryMatch["vram"], 16)
|
||||
|
||||
# Filter out jump table labels
|
||||
labelMatch = regex_label.search(varName)
|
||||
if labelMatch is None:
|
||||
symbolsDict[varName] = VarInfo( currentFile, varVram )
|
||||
# print( symbolsDict[varName] )
|
||||
|
||||
else:
|
||||
inFile = False
|
||||
else:
|
||||
if line.startswith(" .bss "):
|
||||
inFile = False
|
||||
entryMatch = regex_fileDataEntry.search(line)
|
||||
|
||||
# Find file
|
||||
if entryMatch is not None:
|
||||
name = "/".join(entryMatch["name"].split("/")[1:])
|
||||
|
||||
# mapfile only contains .o files, so just strip the last character to replace it
|
||||
# we assume all the .c files are in the src folder, and all others are .s (true for OoT/MM)
|
||||
if name.split("/")[0] == "src":
|
||||
name = name[:-1] + "c"
|
||||
else:
|
||||
name = name[:-1] + "s"
|
||||
|
||||
size = int(entryMatch["size"], 16)
|
||||
vram = int(entryMatch["vram"], 16)
|
||||
|
||||
if size > 0:
|
||||
inFile = True
|
||||
currentFile = name
|
||||
|
||||
return symbolsDict
|
||||
|
||||
|
||||
Compared = collections.namedtuple("Compared", [ "buildAddress", "buildFile", "expectedAddress", "expectedFile", "diff"])
|
||||
|
||||
def compareMapFiles(mapFileBuild: str, mapFileExpected: str):
|
||||
badFiles = set()
|
||||
missingFiles = set()
|
||||
|
||||
print("Build mapfile: " + mapFileBuild, file=os.sys.stderr)
|
||||
print("Expected mapfile: " + mapFileExpected, file=os.sys.stderr)
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
if not os.path.exists(mapFileBuild):
|
||||
print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you enter the correct path?", file=os.sys.stderr)
|
||||
exit(1)
|
||||
|
||||
if not os.path.exists(mapFileExpected):
|
||||
print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. Is 'expected' missing or in a different folder?", file=os.sys.stderr)
|
||||
exit(1)
|
||||
|
||||
buildMap = parseMapFile(mapFileBuild)
|
||||
expectedMap = parseMapFile(mapFileExpected)
|
||||
|
||||
comparedDict = collections.OrderedDict()
|
||||
|
||||
for symbol in buildMap:
|
||||
if symbol in expectedMap:
|
||||
comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, expectedMap[symbol].vram, expectedMap[symbol].file, buildMap[symbol].vram - expectedMap[symbol].vram )
|
||||
if comparedDict[symbol].diff != 0:
|
||||
badFiles.add(buildMap[symbol].file)
|
||||
|
||||
else:
|
||||
missingFiles.add(buildMap[symbol].file)
|
||||
comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, -1, "", "Unknown" )
|
||||
|
||||
for symbol in expectedMap:
|
||||
if not symbol in buildMap:
|
||||
missingFiles.add(expectedMap[symbol].file)
|
||||
comparedDict[symbol] = Compared( -1, "", expectedMap[symbol].vram, expectedMap[symbol].file, "Unknown" )
|
||||
|
||||
return badFiles, missingFiles, comparedDict
|
||||
|
||||
|
||||
def printCsv(badFiles, missingFiles, comparedDict, printAll = True):
|
||||
print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
|
||||
|
||||
# If it's bad or missing, don't need to do anything special.
|
||||
# If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
|
||||
|
||||
for symbol in comparedDict:
|
||||
symbolInfo = comparedDict[symbol]
|
||||
symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET
|
||||
if type(symbolInfo.diff) != int:
|
||||
symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET
|
||||
print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff},{symbolGood}")
|
||||
continue
|
||||
|
||||
if symbolInfo.diff == 0:
|
||||
symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET
|
||||
if (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and not printAll:
|
||||
continue
|
||||
|
||||
if symbolInfo.buildFile != symbolInfo.expectedFile:
|
||||
symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET
|
||||
print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff:X},{symbolGood}")
|
||||
|
||||
|
||||
def main():
|
||||
description = "Check that globally visible bss has not been reordered."
|
||||
epilog = """\
|
||||
N.B. Since this script reads the map files, it can only see globally visible bss; in-function static bss must be examined with other tools.
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument("mapFile", help="Path to a map file.")
|
||||
parser.add_argument("mapFileExpected", help="Path to the expected map file. Optional, default is 'expected/mapFile'.", nargs="?", default="")
|
||||
parser.add_argument("-a", "--print-all", help="Print all bss, not just non-matching.", action="store_true")
|
||||
parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.mapFileExpected == "":
|
||||
args.mapFileExpected = os.path.join("expected", args.mapFile)
|
||||
|
||||
badFiles, missingFiles, comparedDict = compareMapFiles(args.mapFile, args.mapFileExpected)
|
||||
printCsv(badFiles, missingFiles, comparedDict, args.print_all)
|
||||
|
||||
if len(badFiles) + len(missingFiles) != 0:
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
if len(badFiles) != 0:
|
||||
print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL)
|
||||
|
||||
for file in badFiles:
|
||||
print(f"bss reordering in {file}", file=os.sys.stderr)
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
if not args.no_fun_allowed:
|
||||
print(colorama.Fore.LIGHTWHITE_EX +
|
||||
" BSS is REORDERED!!\n"
|
||||
" Oh! MY GOD!!"
|
||||
+ colorama.Style.RESET_ALL, file=os.sys.stderr)
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
if len(missingFiles) != 0:
|
||||
print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL)
|
||||
|
||||
for file in missingFiles:
|
||||
print(f"Symbols missing from {file}", file=os.sys.stderr)
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
if not args.no_fun_allowed:
|
||||
print(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL, file=os.sys.stderr)
|
||||
print("", file=os.sys.stderr)
|
||||
|
||||
print("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'", file=os.sys.stderr)
|
||||
|
||||
return 1
|
||||
|
||||
print("", file=os.sys.stderr)
|
||||
print(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL, file=os.sys.stderr)
|
||||
|
||||
if args.no_fun_allowed:
|
||||
return 0
|
||||
|
||||
print("\n" + colorama.Fore.LIGHTWHITE_EX +
|
||||
colorama.Back.RED + " " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + " CONGRATURATIONS! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + " All Global BSS is correct. " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + " THANK YOU! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + " You are great decomper! " + colorama.Back.RESET + "\n" +
|
||||
colorama.Back.RED + " " + colorama.Style.RESET_ALL , file=os.sys.stderr)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
ret = main()
|
||||
exit(ret)
|
||||
Loading…
Reference in New Issue
Block a user