diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..3dd31e6 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +# E203: whitespace before ':' +# E501: line too long +extend-ignore = E203,E501 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..47d4301 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Explicitly declare text files +*.py text + +# Enforce platform-specific encodings +*.bat text eol=crlf +*.sh text eol=lf +*.sha1 text eol=lf + +# decomp-toolkit writes files with LF +config/**/*.txt text eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d793615..6a18c00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,11 +15,10 @@ jobs: version: [MarioClub_us] steps: - # Checkout the repository (shallow clone) + # Checkout the repository - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: 0 submodules: recursive # Set Git config @@ -33,17 +32,33 @@ jobs: # Build the project - name: Build run: | - python configure.py --version ${{ matrix.version }} \ + python configure.py --map --version ${{ matrix.version }} \ --binutils /binutils --compilers /compilers - ninja build/${{ matrix.version }}/progress.json + ninja all_source build/${{ matrix.version }}/progress.json \ + build/${{ matrix.version }}/report.json # Upload progress if we're on the main branch - name: Upload progress if: github.ref == 'refs/heads/main' continue-on-error: true env: + PROGRESS_SLUG: mkdd PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} run: | python tools/upload_progress.py -b https://progress.decomp.club/ \ - -p mkdd -v ${{ matrix.version }} \ + -p $PROGRESS_SLUG -v ${{ matrix.version }} \ build/${{ matrix.version }}/progress.json + + # Upload map files + - name: Upload map + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.version }}_maps + path: build/${{ matrix.version }}/**/*.MAP + + # Upload progress report + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.version }}_report + path: build/${{ matrix.version }}/report.json diff --git a/.gitignore b/.gitignore index 2351196..30bfdb4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,43 @@ +# IDE folders +.idea/ +.vs/ + +# Caches __pycache__ -.vs -.vscode -build -devkitppc -expected -out +.mypy_cache +.cache/ + +# Original files +orig/*/* +!orig/*/.gitkeep +*.dol +*.rel +*.elf +*.o +*.map +*.MAP + +# Build files +build/ +.ninja_* +build.ninja + +# decompctx output +ctx.* +*.ctx + +# Generated configs +objdiff.json +compile_commands.json + +# Miscellaneous +/*.txt +*.exe +*.dll *.bin *.bfn *.arc -*.d -*.o -*.dol -*.elf -*.MAP -*.exe -*.dll -build.ninja build_opts.yml -.ninja_log -.ninja_deps -objdiff.json -ctx.c \ No newline at end of file +devkitppc +expected +out diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..c595055 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "llvm-vs-code-extensions.vscode-clangd", + ], + "unwantedRecommendations": [ + "ms-vscode.cmake-tools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..18caba5 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,21 @@ +{ + "[c]": { + "files.encoding": "shiftjis", + "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd" + }, + "[cpp]": { + "files.encoding": "shiftjis", + "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.associations": { + "*.inc": "c", + ".clangd": "yaml" + }, + // Disable C/C++ IntelliSense, use clangd instead + "C_Cpp.intelliSenseEngine": "disabled", +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..6dc1213 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,16 @@ +{ + // Use Ctrl+Shift+B to run build tasks. + // Or "Run Build Task" in the Command Palette. + "version": "2.0.0", + "tasks": [ + { + "label": "ninja", + "type": "shell", + "command": "ninja", + "group": { + "kind": "build", + "isDefault": true + } + }, + ] +} diff --git a/README.md b/README.md index 619e78d..1356fb1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Currently only the debug version is supported - Install python, 3.10.4 is recommended as it gets installed by default on most recent Linux distros - Add the path where ninja is located to your environment variables under `PATH` - Clone the repository using `git clone https://github.com/SwareJonge/mkdd` -- Dump a copy of the Debug version and extract `main.dol` and put it in `orig/MarioClub_us/` +- Dump a copy of the Debug version and copy the disc image to `orig/MarioClub_us` +- After the initial build, the disc image can be removed to save space ### Instructions - run `python configure.py` diff --git a/config/MarioClub_us/config_dtk.yml b/config/MarioClub_us/config_dtk.yml index f59def3..61b3476 100644 --- a/config/MarioClub_us/config_dtk.yml +++ b/config/MarioClub_us/config_dtk.yml @@ -1,6 +1,7 @@ -object: orig/MarioClub_us/main.dol +object_base: orig/MarioClub_us +object: sys/main.dol hash: db87a9ec1a34275efc45d965dcdcb1a9eb131885 -#map: orig/MarioClub_us/debugInfoM.MAP +#map: orig/MarioClub_us/files/debugInfoM.MAP symbols: config/MarioClub_us/symbols.txt splits: config/MarioClub_us/splits.txt mw_comment_version: 8 @@ -14,4 +15,4 @@ extract: binary: system.arc.bin - symbol: lbl_803608E0 - binary: buildTimeDat.bin \ No newline at end of file + binary: buildTimeDat.bin diff --git a/config/MarioClub_us/symbols.txt b/config/MarioClub_us/symbols.txt index c269dcb..19b726d 100644 --- a/config/MarioClub_us/symbols.txt +++ b/config/MarioClub_us/symbols.txt @@ -5128,7 +5128,7 @@ __ptmf_test = .text:0x80105EA4; // type:function size:0x30 scope:global align:4 __ptmf_scall = .text:0x80105ED4; // type:function size:0x28 scope:global align:4 __ptmf_scall4 = .text:0x80105EFC; // type:function size:0x28 scope:global align:4 __cvt_fp2unsigned = .text:0x80105F24; // type:function size:0x5C scope:global align:4 -__save_fpr = .text:0x80105F80; // type:function size:0x4C scope:local align:4 +__save_fpr = .text:0x80105F80; // type:function size:0x4C scope:global align:4 _savefpr_14 = .text:0x80105F80; // type:label scope:global _savefpr_15 = .text:0x80105F84; // type:label scope:global _savefpr_16 = .text:0x80105F88; // type:label scope:global @@ -5147,7 +5147,7 @@ _savefpr_28 = .text:0x80105FB8; // type:label scope:global _savefpr_29 = .text:0x80105FBC; // type:label scope:global _savefpr_30 = .text:0x80105FC0; // type:label scope:global _savefpr_31 = .text:0x80105FC4; // type:label scope:global -__restore_fpr = .text:0x80105FCC; // type:function size:0x4C scope:local align:4 +__restore_fpr = .text:0x80105FCC; // type:function size:0x4C scope:global align:4 _restfpr_14 = .text:0x80105FCC; // type:label scope:global _restfpr_15 = .text:0x80105FD0; // type:label scope:global _restfpr_16 = .text:0x80105FD4; // type:label scope:global @@ -5166,7 +5166,7 @@ _restfpr_28 = .text:0x80106004; // type:label scope:global _restfpr_29 = .text:0x80106008; // type:label scope:global _restfpr_30 = .text:0x8010600C; // type:label scope:global _restfpr_31 = .text:0x80106010; // type:label scope:global -__save_gpr = .text:0x80106018; // type:function size:0x4C scope:local align:4 +__save_gpr = .text:0x80106018; // type:function size:0x4C scope:global align:4 _savegpr_14 = .text:0x80106018; // type:label scope:global _savegpr_15 = .text:0x8010601C; // type:label scope:global _savegpr_16 = .text:0x80106020; // type:label scope:global @@ -5185,7 +5185,7 @@ _savegpr_28 = .text:0x80106050; // type:label scope:global _savegpr_29 = .text:0x80106054; // type:label scope:global _savegpr_30 = .text:0x80106058; // type:label scope:global _savegpr_31 = .text:0x8010605C; // type:label scope:global -__restore_gpr = .text:0x80106064; // type:function size:0x4C scope:local align:4 +__restore_gpr = .text:0x80106064; // type:function size:0x4C scope:global align:4 _restgpr_14 = .text:0x80106064; // type:label scope:global _restgpr_15 = .text:0x80106068; // type:label scope:global _restgpr_16 = .text:0x8010606C; // type:label scope:global diff --git a/config/Release_eu/config_dtk.yml b/config/Release_eu/config_dtk.yml index 1db884f..8a9dcc8 100644 --- a/config/Release_eu/config_dtk.yml +++ b/config/Release_eu/config_dtk.yml @@ -1,5 +1,6 @@ # doesn't work yet! -object: orig/Release_eu/main.dol +object_base: orig/Release_eu +object: sys/main.dol hash: f3bf225dd81cd9eb094fa9f8415f95f6bbcb9d10 symbols: config/Release_eu/symbols.txt splits: config/Release_eu/splits.txt diff --git a/configure.py b/configure.py old mode 100644 new mode 100755 index 1569945..fec2e73 --- a/configure.py +++ b/configure.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + ### # Generates build files for the project. # This file also includes the project configuration, @@ -70,11 +72,6 @@ parser.add_argument( action="store_true", help="generate map file(s)", ) -parser.add_argument( - "--no-asm", - action="store_true", - help="don't incorporate .s files from asm directory", -) parser.add_argument( "--debug", action="store_true", @@ -93,6 +90,12 @@ parser.add_argument( type=Path, help="path to decomp-toolkit binary or source (optional)", ) +parser.add_argument( + "--objdiff", + metavar="BINARY | DIR", + type=Path, + help="path to objdiff-cli binary or source (optional)", +) parser.add_argument( "--sjiswrap", metavar="EXE", @@ -110,32 +113,46 @@ parser.add_argument( action="store_true", help="builds equivalent (but non-matching) or modded objects", ) +parser.add_argument( + "--no-progress", + dest="progress", + action="store_false", + help="disable progress calculation", +) args = parser.parse_args() config = ProjectConfig() config.version = str(args.version) version_num = VERSIONS.index(config.version) +debug = args.debug +if config.version == "MarioClub_us": + debug = True + # Apply arguments config.build_dir = args.build_dir config.dtk_path = args.dtk +config.objdiff_path = args.objdiff config.binutils_path = args.binutils config.compilers_path = args.compilers -config.debug = args.debug config.generate_map = args.map config.non_matching = args.non_matching config.sjiswrap_path = args.sjiswrap +config.progress = args.progress if not is_windows(): config.wrapper = args.wrapper -if args.no_asm: +# Don't build asm unless we're --non-matching +if not config.non_matching: config.asm_dir = None +# Disable sjiswrap config.shift_jis = False # Tool versions config.binutils_tag = "2.42-1" -config.compilers_tag = "20231018" -config.dtk_tag = "v0.9.0" +config.compilers_tag = "20240706" +config.dtk_tag = "v1.1.4" +config.objdiff_tag = "v2.3.2" config.sjiswrap_tag = "v1.1.1" config.wibo_tag = "0.6.11" @@ -152,9 +169,12 @@ config.asflags = [ config.ldflags = [ "-fp hardware", "-nodefaults", - "-warn off", - # "-listclosure", # Uncomment for Wii linkers ] +if debug: + config.ldflags.append("-g") +if args.map: + config.ldflags.append("-mapunused") + # Use for any additional files that should cause a re-configure when modified config.reconfig_deps = [] @@ -165,7 +185,7 @@ cflags_base = [ "-enum int", "-fp hardware", "-Cpp_exceptions off", - #"-W all", + # "-W all", "-inline auto", '-pragma "cats off"', '-pragma "warn_notinlined off"', @@ -181,12 +201,8 @@ cflags_base = [ f"-i build/{config.version}/include" ] -if config.version == "MarioClub_us": - config.debug = True - - # Debug flags -if config.debug: +if debug: cflags_base.extend(["-DDEBUG=1", "-DHIO_SCREENSHOT=1"]) else: cflags_base.append("-DNDEBUG=1") @@ -198,7 +214,7 @@ elif "_us" in config.version: elif "_jp" in config.version: cflags_base.extend(["-DREGION_JP=1",]) -if config.non_matching == False: +if not config.non_matching: cflags_base.extend(["-DMATCHING=1"]) # Metrowerks library flags @@ -256,17 +272,19 @@ def DolphinLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "objects": objects, } + # Helper function for MSL libraries -def mslLib(lib_name: str, extra_cflags: str, objects: List[Object]) -> Dict[str, Any]: +def mslLib(lib_name: str, extra_cflags: List[str], objects: List[Object]) -> Dict[str, Any]: return { "lib": lib_name, "src_dir": "libs/PowerPC_EABI_Support/src", "mw_version": "GC/2.6", - "cflags": cflags_runtime + [ f"{extra_cflags}" ], + "cflags": cflags_runtime + extra_cflags, "host": True, "objects": objects, } + def trkLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: return { "lib": lib_name, @@ -277,23 +295,25 @@ def trkLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "objects": objects, } + # Helper function for JSystem libraries -def JSystemLib(lib_name: str, extra_cflags: str, objects: List[Object]) -> Dict[str, Any]: +def JSystemLib(lib_name: str, extra_cflags: List[str], objects: List[Object]) -> Dict[str, Any]: return { "lib": lib_name, "src_dir": "libs", "mw_version": "GC/2.6", - "cflags": cflags_jsystem + [ f"{extra_cflags}" ], + "cflags": cflags_jsystem + extra_cflags, "host": True, "objects": objects, } + # Helper function for Game libraries -def GameLib(lib_name: str, extra_cflags: str, objects: List[Object]) -> Dict[str, Any]: +def GameLib(lib_name: str, extra_cflags: List[str], objects: List[Object]) -> Dict[str, Any]: return { "lib": lib_name, "mw_version": "GC/2.6", - "cflags": cflags_game + [ f"{extra_cflags}" ], + "cflags": cflags_game + extra_cflags, "host": True, "objects": objects, } @@ -303,23 +323,26 @@ Matching = True # Object matches and should be linked NonMatching = False # Object does not match and should not be linked Equivalent = config.non_matching # Object should be linked when configured with --non-matching + +# Object is only matching for specific versions +def MatchingFor(*versions): + return config.version in versions + + config.warn_missing_config = True config.warn_missing_source = True config.progress_all = False - -#Object(Matching, "kartLocale.cpp") - config.libs = [ GameLib( "Localize", - "", + [], [ - Object(Matching, "kartLocale.cpp") + Object(Matching, "kartLocale.cpp"), ] ), mslLib( "Runtime.PPCEABI.H", - "", + [], [ Object(Matching, "Runtime/__mem.c"), Object(Matching, "Runtime/__va_arg.c"), @@ -335,7 +358,7 @@ config.libs = [ ), mslLib( "MSL_C.PPCEABI.H", - "-str pool -opt level=0, peephole, schedule, nospace -inline off -sym on", + ["-str pool", "-opt level=0, peephole, schedule, nospace", "-inline off", "-sym on"], [ Object(Matching, "MSL_C/PPC_EABI/abort_exit.c"), Object(Matching, "MSL_C/MSL_Common/alloc.c"), @@ -368,7 +391,7 @@ config.libs = [ ), mslLib( "fdlibm.PPCEABI.H", - "", + [], [ Object(Matching, "MSL_C/MSL_Common_Embedded/Math/Double_precision/e_asin.c"), Object(Matching, "MSL_C/MSL_Common_Embedded/Math/Double_precision/e_atan2.c"), @@ -400,7 +423,7 @@ config.libs = [ ), mslLib( "MSL_C.PPCEABI.bare.H", - "", + [], [ Object(Matching, "MSL_C/MSL_Common/extras.c") ] @@ -598,7 +621,7 @@ config.libs = [ ] ), DolphinLib( - "lg", # unofficial name + "lg", # unofficial name [ Object(NonMatching, "dolphin/lg/allsrc.c") ] @@ -698,7 +721,7 @@ config.libs = [ ), JSystemLib( "J2DGraph", - "-O4,s", + ["-O4,s"], [ Object(Matching, "JSystem/J2DGraph/J2DAnimation.cpp"), Object(NonMatching, "JSystem/J2DGraph/J2DAnmLoader.cpp"), @@ -724,7 +747,7 @@ config.libs = [ ), JSystemLib( "J3D", - "-O4,p", + ["-O4,p"], [ Object(NonMatching, "JSystem/J3D/GraphAnimator/J3DAnimation.cpp"), Object(NonMatching, "JSystem/J3D/GraphAnimator/J3DCluster.cpp"), @@ -754,20 +777,20 @@ config.libs = [ Object(NonMatching, "JSystem/J3D/GraphBase/J3DVertex.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DAnmLoader.cpp"), - Object(NonMatching, "JSystem/J3D/GraphLoader/J3DBinaryFormat.cpp"), # might not be in GraphLoader + Object(NonMatching, "JSystem/J3D/GraphLoader/J3DBinaryFormat.cpp"), # might not be in GraphLoader Object(NonMatching, "JSystem/J3D/GraphLoader/J3DClusterLoader.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DJointFactory.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DMaterialFactory.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DMaterialFactory_v21.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DModelLoader.cpp"), Object(NonMatching, "JSystem/J3D/GraphLoader/J3DModelLoaderCalcSize.cpp"), - Object(NonMatching, "JSystem/J3D/GraphLoader/J3DModelSaver.cpp"), # might not be in GraphLoader + Object(NonMatching, "JSystem/J3D/GraphLoader/J3DModelSaver.cpp"), # might not be in GraphLoader Object(NonMatching, "JSystem/J3D/GraphLoader/J3DShapeFactory.cpp") ] ), JSystemLib( "J3DU", - "-O4,p", + ["-O4,p"], [ Object(NonMatching, "JSystem/J3DU/J3DUPerf.cpp"), Object(NonMatching, "JSystem/J3DU/J3DUDL.cpp"), @@ -777,7 +800,7 @@ config.libs = [ ), JSystemLib( "JAudio", - "-O4,p", + ["-O4,p"], [ Object(Matching, "JSystem/JAudio/Interface/JAIAudience.cpp"), Object(Matching, "JSystem/JAudio/Interface/JAISe.cpp"), @@ -835,10 +858,10 @@ config.libs = [ Object(NonMatching, "JSystem/JAudio/System/JASTrack.cpp"), Object(NonMatching, "JSystem/JAudio/System/JASTrackPort.cpp"), - Object(Matching, "JSystem/JAudio/Task/dspproc.c", extra_cflags="-O4,s, -inline noauto -use_lmw_stmw off -func_align 32"), - Object(Matching, "JSystem/JAudio/Task/dsptask.c", extra_cflags="-O4,s, -inline noauto -use_lmw_stmw off -func_align 32"), - Object(Matching, "JSystem/JAudio/Task/osdsp.c", extra_cflags="-O4,s, -inline noauto -use_lmw_stmw off -func_align 32"), - Object(Matching, "JSystem/JAudio/Task/osdsp_task.c", extra_cflags="-O4,s, -inline noauto -use_lmw_stmw off -func_align 32"), + Object(Matching, "JSystem/JAudio/Task/dspproc.c", extra_cflags=["-O4,s", "-inline noauto", "-use_lmw_stmw off", "-func_align 32"]), + Object(Matching, "JSystem/JAudio/Task/dsptask.c", extra_cflags=["-O4,s", "-inline noauto", "-use_lmw_stmw off", "-func_align 32"]), + Object(Matching, "JSystem/JAudio/Task/osdsp.c", extra_cflags=["-O4,s", "-inline noauto", "-use_lmw_stmw off", "-func_align 32"]), + Object(Matching, "JSystem/JAudio/Task/osdsp_task.c", extra_cflags=["-O4,s", "-inline noauto", "-use_lmw_stmw off", "-func_align 32"]), Object(Matching, "JSystem/JAudio/Utility/JAUAudioArcInterpreter.cpp"), Object(Matching, "JSystem/JAudio/Utility/JAUAudioArcLoader.cpp"), @@ -859,7 +882,7 @@ config.libs = [ ), JSystemLib( "JFramework", - "-O4,p", + ["-O4,p"], [ Object(Matching, "JSystem/JFramework/JFWSystem.cpp"), Object(Matching, "JSystem/JFramework/JFWDisplay.cpp") @@ -867,7 +890,7 @@ config.libs = [ ), JSystemLib( "JGadget", - "-O4,p", + ["-O4,p"], [ Object(Matching, "JSystem/JGadget/hashcode.cpp"), Object(Matching, "JSystem/JGadget/linklist.cpp") @@ -875,7 +898,7 @@ config.libs = [ ), JSystemLib( "JKernel", - "-O4,s", + ["-O4,s"], [ Object(Matching, "JSystem/JKernel/JKRAram.cpp"), Object(Matching, "JSystem/JKernel/JKRAramArchive.cpp"), @@ -905,7 +928,7 @@ config.libs = [ ), JSystemLib( "JMath", - "-O4,p", + ["-O4,p"], [ Object(Matching, "JSystem/JMath/JMath.cpp"), Object(Matching, "JSystem/JMath/random.cpp"), @@ -914,7 +937,7 @@ config.libs = [ ), JSystemLib( "JParticle", - "-O4,p", + ["-O4,p"], [ Object(Equivalent, "JSystem/JParticle/JPABaseShape.cpp"), Object(NonMatching, "JSystem/JParticle/JPAChildShape.cpp"), @@ -935,7 +958,7 @@ config.libs = [ ), JSystemLib( "JSupport", - "-O4,p", + ["-O4,p"], [ Object(Matching, "JSystem/JSupport/JSUInputStream.cpp"), Object(Matching, "JSystem/JSupport/JSUList.cpp"), @@ -945,7 +968,7 @@ config.libs = [ ), JSystemLib( "JUtility", - "-O4,s", + ["-O4,s"], [ Object(Matching, "JSystem/JUtility/JUTAssert.cpp"), Object(Matching, "JSystem/JUtility/JUTConsole.cpp"), @@ -965,12 +988,12 @@ config.libs = [ Object(Matching, "JSystem/JUtility/JUTTexture.cpp"), Object(Matching, "JSystem/JUtility/JUTVideo.cpp"), Object(Matching, "JSystem/JUtility/JUTXfb.cpp"), - Object(Matching, "JSystem/JUtility/JUTFontData_Ascfont_fix12.s", extra_asflags=f"-I build/{config.version}/bin",) + Object(Matching, "JSystem/JUtility/JUTFontData_Ascfont_fix12.s", extra_asflags=[f"-I build/{config.version}/bin"],) ] ), JSystemLib( "Logitech", - "-O4,p", + ["-O4,p"], [ Object(NonMatching, "JSystem/Logitech/Conditon.cpp"), Object(NonMatching, "JSystem/Logitech/Constant.cpp"), @@ -983,14 +1006,14 @@ config.libs = [ ), GameLib( "Bando", - "", + [], [ Object(Matching, "Bando/EngineSound.cpp") ] ), GameLib( "Inagaki", - "", + [], [ Object(NonMatching, "Inagaki/CharacterSoundMgr.cpp"), Object(NonMatching, "Inagaki/GameAudioCamera.cpp"), @@ -1014,7 +1037,7 @@ config.libs = [ ), GameLib( "Kameda", - "", + [], [ Object(NonMatching, "Kameda/Motor.cpp"), Object(NonMatching, "Kameda/MotorManager.cpp"), @@ -1064,7 +1087,7 @@ config.libs = [ ), GameLib( "Kaneshige", - "-inline off", + ["-inline off"], [ Object(Matching, "Kaneshige/DrawBuffer.cpp"), Object(Equivalent, "Kaneshige/Course/Course.cpp"), @@ -1142,7 +1165,7 @@ config.libs = [ ), GameLib( "Kawano", - "", + [], [ Object(NonMatching, "Kawano/driverData.cpp"), Object(NonMatching, "Kawano/driver.cpp"), @@ -1165,7 +1188,7 @@ config.libs = [ ), GameLib( "Osako", - "", + [], [ Object(Matching, "Osako/clock.cpp"), Object(NonMatching, "Osako/animator.cpp"), @@ -1222,7 +1245,7 @@ config.libs = [ ), GameLib( "Sato", - "", + [], [ Object(NonMatching, "Sato/logbridge.cpp"), Object(Matching, "Sato/stMath.cpp"), @@ -1278,7 +1301,7 @@ config.libs = [ ), GameLib( "Shiraiwa", - "", + [], [ Object(NonMatching, "Shiraiwa/Coord3D.cpp"), Object(NonMatching, "Shiraiwa/MapObjDossun.cpp"), @@ -1367,7 +1390,7 @@ config.libs = [ ), GameLib( "Yamamoto", - "-inline off", + ["-inline off"], [ Object(NonMatching, "Yamamoto/kartBody.cpp"), Object(Matching, "Yamamoto/kartCamera.cpp"), @@ -1407,12 +1430,18 @@ config.libs = [ ), ] +# Optional extra categories for progress tracking +config.progress_categories = [ + # ProgressCategory("game", "Game Code"), + # ProgressCategory("sdk", "SDK Code"), +] +config.progress_each_module = args.verbose + if args.mode == "configure": # Write build.ninja and objdiff.json generate_build(config) elif args.mode == "progress": # Print progress and write progress.json - config.progress_each_module = args.verbose calculate_progress(config) else: - sys.exit("Unknown mode: " + args.mode) \ No newline at end of file + sys.exit("Unknown mode: " + args.mode) diff --git a/include/JSystem/JAudio/JASFakeMatch.h b/include/JSystem/JAudio/JASFakeMatch.h index 6fe1885..d717ec5 100644 --- a/include/JSystem/JAudio/JASFakeMatch.h +++ b/include/JSystem/JAudio/JASFakeMatch.h @@ -8,12 +8,15 @@ #include "JSystem/JAudio/System/JASHeapCtrl.h" #include "JSystem/JAudio/System/JASTrack.h" +template <> JASMemPool_MultiThreaded JASPoolAllocObject_MultiThreaded::memPool_; +template <> JASMemPool_MultiThreaded JASPoolAllocObject_MultiThreaded::memPool_; +template <> JASMemPool_MultiThreaded JASPoolAllocObject_MultiThreaded::memPool_; #endif -#endif \ No newline at end of file +#endif diff --git a/include/JSystem/JAudio/JASFakeMatch2.h b/include/JSystem/JAudio/JASFakeMatch2.h index 2e436ce..d6c3efd 100644 --- a/include/JSystem/JAudio/JASFakeMatch2.h +++ b/include/JSystem/JAudio/JASFakeMatch2.h @@ -7,8 +7,9 @@ class JAISe; #include "JSystem/JAudio/JASFakeMatch.h" +template <> JASMemPool JASPoolAllocObject::memPool_; #endif -#endif \ No newline at end of file +#endif diff --git a/include/JSystem/JAudio/JASFakeMatch3.h b/include/JSystem/JAudio/JASFakeMatch3.h index 44716ae..f59828d 100644 --- a/include/JSystem/JAudio/JASFakeMatch3.h +++ b/include/JSystem/JAudio/JASFakeMatch3.h @@ -10,11 +10,15 @@ class JAISeq; class JAISe; #include "JSystem/JAudio/JASFakeMatch.h" +template <> JASMemPool JASPoolAllocObject::memPool_; +template <> JASMemPool JASPoolAllocObject::memPool_; +template <> JASMemPool JASPoolAllocObject::memPool_; +template <> JASMemPool JASPoolAllocObject::memPool_; #endif -#endif \ No newline at end of file +#endif diff --git a/include/JSystem/JAudio/JAUSectionHeap.h b/include/JSystem/JAudio/JAUSectionHeap.h index ef200e3..011708c 100644 --- a/include/JSystem/JAudio/JAUSectionHeap.h +++ b/include/JSystem/JAudio/JAUSectionHeap.h @@ -136,10 +136,11 @@ JAUSectionHeap *JAUNewSectionHeap(bool); // TODO: probably move this to a different file(JASFakeMatch?) #define createJASInstance(TYPE) \ + template <> \ DECL_WEAK TYPE *JASGlobalInstance::sInstance createJASInstance(JAUSectionHeap); createJASInstance(JAUSoundNameTable); createJASInstance(JAUSoundTable); -#endif \ No newline at end of file +#endif diff --git a/include/JSystem/JGeometry/Box.h b/include/JSystem/JGeometry/Box.h index 62b5307..05bde23 100644 --- a/include/JSystem/JGeometry/Box.h +++ b/include/JSystem/JGeometry/Box.h @@ -54,10 +54,10 @@ struct TBox2 : TBox > { TBox2(f32 x0, f32 y0, const TVec2& f) { set(x0, y0, x0 + f.x, y0 + f.y); } TBox2(f32 val) { - f.y = val; - f.x = val; - i.y = val; - i.x = val; + this->f.y = val; + this->f.x = val; + this->i.y = val; + this->i.x = val; } void absolute() { @@ -77,7 +77,7 @@ struct TBox2 : TBox > { void set(const TBox2& other) { set(other.i, other.f); } void set(const TVec2& i, const TVec2& f) { this->i.set(i), this->f.set(f); } - void set(f32 x0, f32 y0, f32 x1, f32 y1) { i.set(x0, y0); f.set(x1, y1); } + void set(f32 x0, f32 y0, f32 x1, f32 y1) { this->i.set(x0, y0); this->f.set(x1, y1); } }; // clang-format on diff --git a/include/JSystem/JGeometry/Matrix.h b/include/JSystem/JGeometry/Matrix.h index 546cc57..7d40d7b 100644 --- a/include/JSystem/JGeometry/Matrix.h +++ b/include/JSystem/JGeometry/Matrix.h @@ -139,22 +139,22 @@ namespace JGeometry { void identity() // TODO: OK? { const T zero = (T)0; - ref(2, 3) = zero; - ref(1, 3) = zero; - ref(0, 3) = zero; + this->ref(2, 3) = zero; + this->ref(1, 3) = zero; + this->ref(0, 3) = zero; - ref(1, 2) = zero; - ref(0, 2) = zero; + this->ref(1, 2) = zero; + this->ref(0, 2) = zero; - ref(2, 1) = zero; - ref(0, 1) = zero; - ref(2, 0) = zero; - ref(1, 0) = zero; + this->ref(2, 1) = zero; + this->ref(0, 1) = zero; + this->ref(2, 0) = zero; + this->ref(1, 0) = zero; const T one = TUtil::one(); - ref(2, 2) = one; - ref(1, 1) = one; - ref(0, 0) = one; + this->ref(2, 2) = one; + this->ref(1, 1) = one; + this->ref(0, 0) = one; } void concat(const T &rSrcA, const T &rSrcB); void concat(const T &rSrc); @@ -174,17 +174,17 @@ namespace JGeometry { void getXDir(TVec3f &rDest) const { - rDest.set(mMtx[0][0], mMtx[1][0], mMtx[2][0]); + rDest.set(this->mMtx[0][0], this->mMtx[1][0], this->mMtx[2][0]); }; void getYDir(TVec3f &rDest) const { - rDest.set(mMtx[0][1], mMtx[1][1], mMtx[2][1]); + rDest.set(this->mMtx[0][1], this->mMtx[1][1], this->mMtx[2][1]); }; void getZDir(TVec3f &rDest) const { - rDest.set(mMtx[0][2], mMtx[1][2], mMtx[2][2]); + rDest.set(this->mMtx[0][2], this->mMtx[1][2], this->mMtx[2][2]); }; void getXYZDir(TVec3f &rDestX, TVec3f &rDestY, TVec3f &rDestZ) const; @@ -196,15 +196,15 @@ namespace JGeometry { void setZDir(f32 x, f32 y, f32 z); void setXYZDir(const TVec3f &rSrcX, const TVec3f &rSrcY, const TVec3f &rSrcZ) { - ref(0, 0) = rSrcX.x; - ref(1, 0) = rSrcX.y; - ref(2, 0) = rSrcX.z; - ref(0, 1) = rSrcY.x; - ref(1, 1) = rSrcY.y; - ref(2, 1) = rSrcY.z; - ref(0, 2) = rSrcZ.x; - ref(1, 2) = rSrcZ.y; - ref(2, 2) = rSrcZ.z; + this->ref(0, 0) = rSrcX.x; + this->ref(1, 0) = rSrcX.y; + this->ref(2, 0) = rSrcX.z; + this->ref(0, 1) = rSrcY.x; + this->ref(1, 1) = rSrcY.y; + this->ref(2, 1) = rSrcY.z; + this->ref(0, 2) = rSrcZ.x; + this->ref(1, 2) = rSrcZ.y; + this->ref(2, 2) = rSrcZ.z; } void getEuler(TVec3f &rDest) const; @@ -226,9 +226,9 @@ namespace JGeometry { inline void getXDirInline(TVec3f &rDest) const { - f32 z = mMtx[2][0]; - f32 y = mMtx[1][0]; - f32 x = mMtx[0][0]; + f32 z = this->mMtx[2][0]; + f32 y = this->mMtx[1][0]; + f32 x = this->mMtx[0][0]; rDest.set(x, y, z); } @@ -253,9 +253,9 @@ namespace JGeometry { void setTrans(f32 x, f32 y, f32 z); void zeroTrans() { - ref(0, 3) = 0.0f; - ref(1, 3) = 0.0f; - ref(2, 3) = 0.0f; + this->ref(0, 3) = 0.0f; + this->ref(1, 3) = 0.0f; + this->ref(2, 3) = 0.0f; } void makeRotate(const TVec3f &, f32); @@ -265,9 +265,9 @@ namespace JGeometry { inline void getTransInline(TVec3f &rDest) const { - f32 z = mMtx[2][3]; - f32 y = mMtx[1][3]; - f32 x = mMtx[0][3]; + f32 z = this->mMtx[2][3]; + f32 y = this->mMtx[1][3]; + f32 x = this->mMtx[0][3]; rDest.set(x, y, z); } }; diff --git a/include/JSystem/JGeometry/Quat.h b/include/JSystem/JGeometry/Quat.h index c8e8409..f9688f3 100644 --- a/include/JSystem/JGeometry/Quat.h +++ b/include/JSystem/JGeometry/Quat.h @@ -12,8 +12,8 @@ namespace JGeometry { /* Constructors */ inline TQuat4() {} - template - TQuat4(T _x, T _y, T _z, T _w) + template + TQuat4(A _x, A _y, A _z, A _w) { x = _x; y = _y; diff --git a/include/JSystem/JGeometry/Util.h b/include/JSystem/JGeometry/Util.h index bac4c7f..10f0214 100644 --- a/include/JSystem/JGeometry/Util.h +++ b/include/JSystem/JGeometry/Util.h @@ -28,7 +28,7 @@ namespace JGeometry { return equal; } - f32 invert(f32 x) { return 1.0f/x; } + static f32 invert(f32 x) { return 1.0f/x; } static f32 sqrt(f32 x) { diff --git a/include/JSystem/JGeometry/Vec.h b/include/JSystem/JGeometry/Vec.h index c9a5080..2400296 100644 --- a/include/JSystem/JGeometry/Vec.h +++ b/include/JSystem/JGeometry/Vec.h @@ -7,6 +7,7 @@ namespace JGeometry { inline void setTVec3f(register const f32 *src, register f32 *dst) { +#ifdef __MWERKS__ register f32 xy, z; __asm { // clang-format off @@ -16,10 +17,12 @@ namespace JGeometry { stfs z, 8(dst) // clang-format on } +#endif } inline void negateInternal(register const f32 *src, register f32 *dst) { +#ifdef __MWERKS__ register f32 xy; __asm { // clang-format off @@ -29,10 +32,12 @@ namespace JGeometry { // clang-format on } dst[2] = -src[2]; +#endif } inline void mulInternal(register const f32 *vec1, register const f32 *vec2, register f32 *dst) { +#ifdef __MWERKS__ register f32 xy1, xy2, res; __asm { // clang-format off @@ -43,6 +48,7 @@ namespace JGeometry { // clang-format on } dst[2] = vec1[2] * vec2[2]; +#endif } template diff --git a/include/Kaneshige/RaceLight.h b/include/Kaneshige/RaceLight.h index 2e58f17..7d2a105 100644 --- a/include/Kaneshige/RaceLight.h +++ b/include/Kaneshige/RaceLight.h @@ -12,7 +12,7 @@ public: void getTagName(u32); // UNUSED virtual ~RaceSceneLight(); // overide - virtual Mtx *getEffectMtx(); // overide + virtual MtxPtr getEffectMtx(); // overide virtual void calc(); // overide private: u8 _90[0xf4 - 0x90]; @@ -23,7 +23,7 @@ class RaceBalloonLight : public LtObjDiffuse public: RaceBalloonLight(u32); virtual ~RaceBalloonLight(); // overide - virtual Mtx *getEffectMtx(); // overide + virtual MtxPtr getEffectMtx(); // overide virtual void draw(); // overide private: @@ -36,7 +36,7 @@ public: RaceKartLight(RaceSceneLight *, int); virtual ~RaceKartLight(); // overide - virtual Mtx *getEffectMtx(); // overide + virtual MtxPtr getEffectMtx(); // overide virtual void draw(); // overide virtual void calc(); // overide @@ -57,4 +57,4 @@ private: u8 _90[0xd0 - 0x90]; }; -#endif \ No newline at end of file +#endif diff --git a/include/Kawano/driverMgr.h b/include/Kawano/driverMgr.h index edb0ba4..70a8794 100644 --- a/include/Kawano/driverMgr.h +++ b/include/Kawano/driverMgr.h @@ -87,7 +87,7 @@ public: // Virtual Table virtual void update(u16); // override - virtual void createModel(JKRSolidHeap *, u32, u32); // override + virtual bool createModel(JKRSolidHeap *, u32, u32); // override virtual void setCurrentViewNo(u32); // override virtual ~DriverModel(); diff --git a/include/macros.h b/include/macros.h index ad0e7ed..8755e74 100644 --- a/include/macros.h +++ b/include/macros.h @@ -20,9 +20,15 @@ #define CLEAR_PATH(x) __memclr((x), sizeof((x))) #define ALIGN(x) __attribute__((aligned(x))) +#ifdef __MWERKS__ #define DECL_SECTION(x) __declspec(section x) #define DECL_WEAK __declspec(weak) #define DONT_INLINE __attribute__((never_inline)) +#else +#define DECL_SECTION(x) +#define DECL_WEAK __attribute__((weak)) +#define DONT_INLINE __attribute__((noinline)) +#endif // Align X to the previous N bytes (N must be power of two) #define ALIGN_PREV(X, N) ((X) & ~((N)-1)) @@ -70,4 +76,4 @@ #define __declspec(x) #endif -#endif \ No newline at end of file +#endif diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/MSL_C/MSL_Common/math_api.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/MSL_C/MSL_Common/math_api.h index b29cf46..f2a7d3d 100644 --- a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/MSL_C/MSL_Common/math_api.h +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/MSL_C/MSL_Common/math_api.h @@ -12,7 +12,11 @@ int __fpclassifyf(float); int __signbitd(double); int __fpclassifyd(double); -inline int __fpclassifyf(f32 x) +#ifndef MATH_INLINE +#define MATH_INLINE inline +#endif + +MATH_INLINE int __fpclassifyf(f32 x) { switch ((*(s32*)&x) & 0x7f800000) { case 0x7f800000: { @@ -32,7 +36,8 @@ inline int __fpclassifyf(f32 x) } return 4; } -inline int __fpclassifyd(f64 x) + +MATH_INLINE int __fpclassifyd(f64 x) { switch (__HI(x) & 0x7ff00000) { case 0x7ff00000: { diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/Gecko_ExceptionPPC.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/Gecko_ExceptionPPC.h new file mode 100644 index 0000000..749e4ea --- /dev/null +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/Gecko_ExceptionPPC.h @@ -0,0 +1,231 @@ +#ifndef _RUNTIME_GECKO_EXCEPTIONPPC_H +#define _RUNTIME_GECKO_EXCEPTIONPPC_H + +#include "types.h" + +typedef u8 exaction_type; + +#define EXACTION_ENDBIT 0x80 +#define EXACTION_MASK 0x7F + +// EXAction structs + +#define EXACTION_ENDOFLIST 0 +#define EXACTION_BRANCH 1 + +typedef struct ex_branch { + exaction_type action; + u8 unused; + u16 target; +} ex_branch; + +#define EXACTION_DESTROYLOCAL 2 + +typedef struct ex_destroylocal { + exaction_type action; + u8 unused; + s16 local; + void* dtor; +} ex_destroylocal; + +#define EXACTION_DESTROYLOCALCOND 3 + +typedef struct ex_destroylocalcond { + exaction_type action; + u8 dlc_field; + s16 cond; + s16 local; + void* dtor; +} ex_destroylocalcond; + +#define ex_destroylocalcond_MakeField(regcond) (((regcond) << 7)) +#define ex_destroylocalcond_GetRegCond(field) ((field) >> 7) + +#define EXACTION_DESTROYLOCALPOINTER 4 + +typedef struct ex_destroylocalpointer { + exaction_type action; + u8 dlp_field; + s16 pointer; + void* dtor; +} ex_destroylocalpointer; + +#define ex_destroylocalpointer_MakeField(regpointer) (((regpointer) << 7)) +#define ex_destroylocalpointer_GetRegPointer(field) ((field) >> 7) + +#define EXACTION_DESTROYLOCALARRAY 5 + +typedef struct ex_destroylocalarray { + exaction_type action; + u8 unused; + s16 localarray; + u16 elements; + u16 element_size; + void* dtor; +} ex_destroylocalarray; + +#define EXACTION_DESTROYBASE 6 +#define EXACTION_DESTROYMEMBER 7 + +typedef struct ex_destroymember { + exaction_type action; + u8 dm_field; + s16 objectptr; + s32 offset; + void* dtor; +} ex_destroymember; + +#define ex_destroymember_MakeField(regpointer) (((regpointer) << 7)) +#define ex_destroymember_GetRegPointer(field) ((field) >> 7) + +#define EXACTION_DESTROYMEMBERCOND 8 + +typedef struct ex_destroymembercond { + exaction_type action; + u8 dmc_field; + s16 cond; + s16 objectptr; + s32 offset; + void* dtor; +} ex_destroymembercond; + +#define ex_destroymembercond_MakeField(regcond, regpointer) (((regcond) << 7) | (((regpointer)&0x1) << 6)) +#define ex_destroymembercond_GetRegCond(field) ((field) >> 7) +#define ex_destroymembercond_GetRegPointer(field) (((field) >> 6) & 0x1) + +#define EXACTION_DESTROYMEMBERARRAY 9 + +typedef struct ex_destroymemberarray { + exaction_type action; + u8 dma_field; + s16 objectptr; + s32 offset; + s32 elements; + s32 element_size; + void* dtor; +} ex_destroymemberarray; + +#define ex_destroymemberarray_MakeField(regpointer) (((regpointer) << 7)) +#define ex_destroymemberarray_GetRegPointer(field) ((field) >> 7) + +#define EXACTION_DELETEPOINTER 10 + +typedef struct ex_deletepointer { + exaction_type action; + u8 dp_field; + s16 objectptr; + void* deletefunc; +} ex_deletepointer; + +#define ex_deletepointer_MakeField(regpointer) (((regpointer) << 7)) +#define ex_deletepointer_GetRegPointer(field) ((field) >> 7) + +#define EXACTION_DELETEPOINTERCOND 11 + +typedef struct ex_deletepointercond { + exaction_type action; + u8 dpc_field; + s16 cond; + s16 objectptr; + void* deletefunc; +} ex_deletepointercond; + +#define ex_deletepointercond_MakeField(regcond, regpointer) (((regcond) << 7) | (((regpointer)&0x1) << 6)) +#define ex_deletepointercond_GetRegCond(field) ((field) >> 7) +#define ex_deletepointercond_GetRegPointer(field) (((field) >> 6) & 0x1) + +#define EXACTION_CATCHBLOCK 12 + +typedef struct ex_catchblock { + exaction_type action; + u8 unused; + char* catch_type; + u16 catch_pcoffset; + s16 cinfo_ref; +} ex_catchblock; + +#define EXACTION_ACTIVECATCHBLOCK 13 + +typedef struct ex_activecatchblock { + exaction_type action; + u8 unused; + s16 cinfo_ref; +} ex_activecatchblock; + +#define EXACTION_TERMINATE 14 + +typedef struct ex_terminate { + exaction_type action; + u8 unused; +} ex_terminate; + +#define EXACTION_SPECIFICATION 15 + +typedef struct ex_specification { + exaction_type action; + u8 unused; + u16 specs; + s32 pcoffset; + s32 cinfo_ref; + char* spec[]; +} ex_specification; + +#define EXACTION_CATCHBLOCK_32 16 + +typedef struct ex_catchblock_32 { + exaction_type action; + u8 unused; + char* catch_type; + s32 catch_pcoffset; + s32 cinfo_ref; +} ex_catchblock_32; + +// Other structs + +typedef struct ExceptionRangeSmall { + u16 start; + u16 end; + u16 action; +} ExceptionRangeSmall; + +typedef struct ExceptionTableSmall { + u16 et_field; + ExceptionRangeSmall ranges[0]; +} ExceptionTableSmall; + +typedef struct ExceptionRangeLarge { + u32 start; + u16 size; + u16 action; +} ExceptionRangeLarge; + +typedef struct ExceptionTableLarge { + u16 et_field; + u16 et_field2; + ExceptionRangeLarge ranges[]; +} ExceptionTableLarge; + +#define ET_MakeField(savedGPRs, savedFPRs, savedCR, hasframeptr, isLarge) \ + (((savedGPRs) << 11) | ((savedFPRs & 0x1f) << 6) | ((savedCR & 0x1) << 5) | ((hasframeptr & 0x1) << 4) | ((isLarge & 1) << 3)) + +#define ET_GetSavedGPRs(field) ((field) >> 11) +#define ET_GetSavedFPRs(field) (((field) >> 6) & 0x1f) +#define ET_GetSavedCR(field) (((field) >> 5) & 0x1) +#define ET_GetHasFramePtr(field) (((field) >> 4) & 0x1) +#define ET_IsLargeTable(field) (((field) >> 3) & 0x1) +#define ET_ClearLargeBit(field) ((field) & ~(1 << 3)) +#define ET_SetLargeBit(field) ((field) | (1 << 3)) + +#define ET_HasElfVector(field) (((field) >> 1) & 0x1) + +typedef struct ExceptionTableIndex { + u32 functionoffset; + u32 eti_field; + u32 exceptionoffset; +} ExceptionTableIndex; + +#define ETI_MakeField(direct, fsize) ((((s32)(direct)) << 31) | ((fsize)&0x7fffffff)) +#define ETI_GetDirectStore(field) ((field) >> 31) +#define ETI_GetFunctionSize(field) ((field)&0x7fffffff) + +#endif diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/NMWException.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/NMWException.h index 5d633cb..692055e 100644 --- a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/NMWException.h +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/NMWException.h @@ -8,21 +8,24 @@ extern "C" { #endif -#define CTORARG_TYPE int -#define CTORARG_PARTIAL (0) -#define CTORARG_COMPLETE (1) +typedef s16 vbase_ctor_arg_type; +typedef char local_cond_type; -#define CTORCALL_COMPLETE(ctor, objptr) (((void (*)(void*, CTORARG_TYPE))ctor)(objptr, CTORARG_COMPLETE)) - -#define DTORARG_TYPE int - -#define DTORCALL_COMPLETE(dtor, objptr) (((void (*)(void*, DTORARG_TYPE))dtor)(objptr, -1)) +typedef struct CatchInfo { + void* location; + void* typeinfo; + void* dtor; + void* sublocation; + s32 pointercopy; + void* stacktop; +} CatchInfo; void __unregister_fragment(int fragmentID); // struct __eti_init_info* info int __register_fragment(void* info, char* TOC); void* __register_global_object(void* object, void* destructor, void* regmem); void __destroy_global_chain(void); +extern char __throw_catch_compare(const char* throwtype, const char* catchtype, s32* offset_result); #ifdef __cplusplus } diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__mem.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__mem.h index 3f10fd8..198d2b4 100644 --- a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__mem.h +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__mem.h @@ -1,13 +1,14 @@ #ifndef RUNTIME_MEM_H #define RUNTIME_MEM_H +#include "macros.h" #include "types.h" #ifdef __cplusplus extern "C" { #endif -__declspec(section ".init") void* memcpy(void* dest, const void* src, size_t n); -__declspec(section ".init") void __fill_mem(void* dest, int val, size_t count); -__declspec(section ".init") void* memset(void* dest, int val, size_t count); +DECL_SECTION(".init") void* memcpy(void* dest, const void* src, size_t n); +DECL_SECTION(".init") void __fill_mem(void* dest, int val, size_t count); +DECL_SECTION(".init") void* memset(void* dest, int val, size_t count); #ifdef __cplusplus } diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__ppc_eabi_linker.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__ppc_eabi_linker.h index a2fa3d6..e8ad606 100644 --- a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__ppc_eabi_linker.h +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/__ppc_eabi_linker.h @@ -1,17 +1,19 @@ #ifndef __PPC_EABI_LINKER -#define __PPC_EABI_LINKER +#define __ppc_eabi_linker -__declspec(section ".init") extern char _stack_addr[]; -__declspec(section ".init") extern char _stack_end[]; -__declspec(section ".init") extern char _heap_addr[]; -__declspec(section ".init") extern char _heap_end[]; -__declspec(section ".init") extern const char _fextabindex_rom[]; -__declspec(section ".init") extern char _fextabindex[]; -__declspec(section ".init") extern char _eextabindex[]; +#include "macros.h" -__declspec(section ".init") extern char _SDA_BASE_[]; +DECL_SECTION(".init") extern char _stack_addr[]; +DECL_SECTION(".init") extern char _stack_end[]; +DECL_SECTION(".init") extern char _heap_addr[]; +DECL_SECTION(".init") extern char _heap_end[]; +DECL_SECTION(".init") extern const char _fextabindex_rom[]; +DECL_SECTION(".init") extern char _fextabindex[]; +DECL_SECTION(".init") extern char _eextabindex[]; -__declspec(section ".init") extern char _SDA2_BASE_[]; +DECL_SECTION(".init") extern char _SDA_BASE_[]; + +DECL_SECTION(".init") extern char _SDA2_BASE_[]; typedef struct __rom_copy_info { char* rom; @@ -19,14 +21,14 @@ typedef struct __rom_copy_info { unsigned int size; } __rom_copy_info; -__declspec(section ".init") extern __rom_copy_info _rom_copy_info[]; +DECL_SECTION(".init") extern __rom_copy_info _rom_copy_info[]; typedef struct __bss_init_info { char* addr; unsigned int size; } __bss_init_info; -__declspec(section ".init") extern __bss_init_info _bss_init_info[]; +DECL_SECTION(".init") extern __bss_init_info _bss_init_info[]; typedef struct __eti_init_info { void* eti_start; @@ -35,38 +37,38 @@ typedef struct __eti_init_info { unsigned long code_size; } __eti_init_info; -__declspec(section ".init") extern __eti_init_info _eti_init_info[]; -__declspec(section ".init") extern const char _f_init_rom[]; -__declspec(section ".init") extern char _f_init[]; -__declspec(section ".init") extern char _e_init[]; -__declspec(section ".init") extern const char _f_text_rom[]; -__declspec(section ".init") extern char _f_text[]; -__declspec(section ".init") extern char _e_text[]; -__declspec(section ".init") extern const char _f_rodata_rom[]; -__declspec(section ".init") extern char _f_rodata[]; -__declspec(section ".init") extern char _e_rodata[]; -__declspec(section ".init") extern const char _fextab_rom[]; -__declspec(section ".init") extern char _fextab[]; -__declspec(section ".init") extern char _eextab[]; -__declspec(section ".init") extern const char _f_data_rom[]; -__declspec(section ".init") extern char _f_data[]; -__declspec(section ".init") extern char _e_data[]; -__declspec(section ".init") extern char _f_bss[]; -__declspec(section ".init") extern char _e_bss[]; -__declspec(section ".init") extern const char _f_sdata_rom[]; -__declspec(section ".init") extern char _f_sdata[]; -__declspec(section ".init") extern char _e_sdata[]; -__declspec(section ".init") extern char _f_sbss[]; -__declspec(section ".init") extern char _e_sbss[]; -__declspec(section ".init") extern const char _f_sdata2_rom[]; -__declspec(section ".init") extern char _f_sdata2[]; -__declspec(section ".init") extern char _e_sdata2[]; -__declspec(section ".init") extern char _f_sbss2[]; -__declspec(section ".init") extern char _e_sbss2[]; -__declspec(section ".init") extern const char _f_PPC_EMB_sdata0_rom[]; -__declspec(section ".init") extern char _f_PPC_EMB_sdata0[]; -__declspec(section ".init") extern char _e_PPC_EMB_sdata0[]; -__declspec(section ".init") extern char _f_PPC_EMB_sbss0[]; -__declspec(section ".init") extern char _e_PPC_EMB_sbss0[]; +DECL_SECTION(".init") extern __eti_init_info _eti_init_info[]; +DECL_SECTION(".init") extern const char _f_init_rom[]; +DECL_SECTION(".init") extern char _f_init[]; +DECL_SECTION(".init") extern char _e_init[]; +DECL_SECTION(".init") extern const char _f_text_rom[]; +DECL_SECTION(".init") extern char _f_text[]; +DECL_SECTION(".init") extern char _e_text[]; +DECL_SECTION(".init") extern const char _f_rodata_rom[]; +DECL_SECTION(".init") extern char _f_rodata[]; +DECL_SECTION(".init") extern char _e_rodata[]; +DECL_SECTION(".init") extern const char _fextab_rom[]; +DECL_SECTION(".init") extern char _fextab[]; +DECL_SECTION(".init") extern char _eextab[]; +DECL_SECTION(".init") extern const char _f_data_rom[]; +DECL_SECTION(".init") extern char _f_data[]; +DECL_SECTION(".init") extern char _e_data[]; +DECL_SECTION(".init") extern char _f_bss[]; +DECL_SECTION(".init") extern char _e_bss[]; +DECL_SECTION(".init") extern const char _f_sdata_rom[]; +DECL_SECTION(".init") extern char _f_sdata[]; +DECL_SECTION(".init") extern char _e_sdata[]; +DECL_SECTION(".init") extern char _f_sbss[]; +DECL_SECTION(".init") extern char _e_sbss[]; +DECL_SECTION(".init") extern const char _f_sdata2_rom[]; +DECL_SECTION(".init") extern char _f_sdata2[]; +DECL_SECTION(".init") extern char _e_sdata2[]; +DECL_SECTION(".init") extern char _f_sbss2[]; +DECL_SECTION(".init") extern char _e_sbss2[]; +DECL_SECTION(".init") extern const char _f_PPC_EMB_sdata0_rom[]; +DECL_SECTION(".init") extern char _f_PPC_EMB_sdata0[]; +DECL_SECTION(".init") extern char _e_PPC_EMB_sdata0[]; +DECL_SECTION(".init") extern char _f_PPC_EMB_sbss0[]; +DECL_SECTION(".init") extern char _e_PPC_EMB_sbss0[]; #endif // __PPC_EABI_LINKER diff --git a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/global_destructor_chain.h b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/global_destructor_chain.h index 5458dda..52046c8 100644 --- a/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/global_destructor_chain.h +++ b/libs/PowerPC_EABI_Support/include/PowerPC_EABI_Support/Runtime/global_destructor_chain.h @@ -7,16 +7,6 @@ extern "C" { #endif -#define CTORARG_TYPE int -#define CTORARG_PARTIAL (0) -#define CTORARG_COMPLETE (1) - -#define CTORCALL_COMPLETE(ctor, objptr) (((void (*)(void*, CTORARG_TYPE))ctor)(objptr, CTORARG_COMPLETE)) - -#define DTORARG_TYPE int - -#define DTORCALL_COMPLETE(dtor, objptr) (((void (*)(void*, DTORARG_TYPE))dtor)(objptr, -1)) - typedef struct DestructorChain { struct DestructorChain* next; void* destructor; diff --git a/libs/PowerPC_EABI_Support/src/MSL_C/PPC_EABI/math_ppc.c b/libs/PowerPC_EABI_Support/src/MSL_C/PPC_EABI/math_ppc.c index 18e1afa..8054576 100644 --- a/libs/PowerPC_EABI_Support/src/MSL_C/PPC_EABI/math_ppc.c +++ b/libs/PowerPC_EABI_Support/src/MSL_C/PPC_EABI/math_ppc.c @@ -1,53 +1,12 @@ +#define MATH_INLINE #include "PowerPC_EABI_Support/MSL_C/PPC_EABI/math_ppc.h" #include "fdlibm.h" -int __fpclassifyf(float x) -{ - switch ((*(s32 *)&x) & 0x7f800000) - { - case 0x7f800000: - { - if ((*(s32 *)&x) & 0x007fffff) - return 1; - else - return 2; - break; - } - case 0: - { - if ((*(s32 *)&x) & 0x007fffff) - return 5; - else - return 3; - break; - } - } - return 4; -} +// Generated via math_api.h +// int __fpclassifyf(float x); -int __fpclassifyd(double x) -{ - switch (__HI(x) & 0x7ff00000) - { - case 0x7ff00000: - { - if ((__HI(x) & 0x000fffff) || (__LO(x) & 0xffffffff)) - return 1; - else - return 2; - break; - } - case 0: - { - if ((__HI(x) & 0x000fffff) || (__LO(x) & 0xffffffff)) - return 5; - else - return 3; - break; - } - } - return 4; -} +// Generated via math_api.h +// int __fpclassifyd(double x); /*double scalbn(double x, int y) { @@ -324,11 +283,11 @@ void atan2f(void) // UNUSED FUNCTION }*/ -__declspec(weak) float cosf(float __x) { return cos((double)__x); } +DECL_WEAK float cosf(float __x) { return cos((double)__x); } -__declspec(weak) float sinf(float __x) { return sin((double)__x); } +DECL_WEAK float sinf(float __x) { return sin((double)__x); } -__declspec(weak) float tanf(float __x) { return tan((double)__x); } +DECL_WEAK float tanf(float __x) { return tan((double)__x); } /*void coshf(void) { diff --git a/libs/PowerPC_EABI_Support/src/Runtime/Gecko_ExceptionPPC.cp b/libs/PowerPC_EABI_Support/src/Runtime/Gecko_ExceptionPPC.cp index 7fdd6f9..293c100 100644 --- a/libs/PowerPC_EABI_Support/src/Runtime/Gecko_ExceptionPPC.cp +++ b/libs/PowerPC_EABI_Support/src/Runtime/Gecko_ExceptionPPC.cp @@ -1,6 +1,5 @@ -#include "PowerPC_EABI_Support/Runtime/MWCPlusLib.h" -#include #include "PowerPC_EABI_Support/Runtime/Gecko_ExceptionPPC.h" +#include "PowerPC_EABI_Support/Runtime/MWCPlusPlusLib.h" #include "PowerPC_EABI_Support/Runtime/NMWException.h" #include "PowerPC_EABI_Support/Runtime/__ppc_eabi_linker.h" @@ -238,7 +237,7 @@ static exaction_type ExPPC_NextAction(ActionIterator* iter){ ExPPC_FindExceptionRecord(return_addr, &iter->info); if(iter->info.exception_record == 0){ - terminate(); + std::terminate(); } iter->current_SP = callers_SP; @@ -288,7 +287,7 @@ static exaction_type ExPPC_NextAction(ActionIterator* iter){ iter->info.action_pointer += sizeof(ex_specification) + ((ex_specification*)iter->info.action_pointer)->specs * sizeof(void*); break; default: - terminate(); + std::terminate(); } } @@ -436,7 +435,7 @@ static void ExPPC_UnwindStack(ThrowContext* context, MWExceptionInfo* info, void ExPPC_FindExceptionRecord(return_addr, info); if(info->exception_record == 0){ - terminate(); + std::terminate(); } context->FP = (ET_GetHasFramePtr(info->exception_record->et_field)) ? (char*)context->GPR[31] : context->SP; @@ -518,7 +517,7 @@ static void ExPPC_UnwindStack(ThrowContext* context, MWExceptionInfo* info, void info->action_pointer += sizeof(ex_specification) + ((ex_specification*)info->action_pointer)->specs * sizeof(void*); break; default: - terminate(); + std::terminate(); } if(action & EXACTION_ENDBIT) info->action_pointer = 0; @@ -656,7 +655,7 @@ static void ExPPC_ThrowHandler(ThrowContext* context){ ExPPC_FindExceptionRecord(context->returnaddr, &info); if(info.exception_record == 0){ - terminate(); + std::terminate(); } context->FP = (ET_GetHasFramePtr(info.exception_record->et_field)) ? (char*)context->GPR[31] : context->SP; @@ -688,7 +687,7 @@ static void ExPPC_ThrowHandler(ThrowContext* context){ continue; case EXACTION_TERMINATE: default: - terminate(); + std::terminate(); } break; } @@ -739,7 +738,7 @@ static void ExPPC_ThrowHandler(ThrowContext* context){ continue; case EXACTION_TERMINATE: default: - terminate(); + std::terminate(); } break; } @@ -889,4 +888,4 @@ asm void __throw(char* throwtype, void* location, void* dtor){ //unused void __end__catch(CatchInfo* catchinfo){ -} \ No newline at end of file +} diff --git a/libs/PowerPC_EABI_Support/src/Runtime/NMWException.cp b/libs/PowerPC_EABI_Support/src/Runtime/NMWException.cp index e575d02..8e8fd74 100644 --- a/libs/PowerPC_EABI_Support/src/Runtime/NMWException.cp +++ b/libs/PowerPC_EABI_Support/src/Runtime/NMWException.cp @@ -1,4 +1,5 @@ #include "PowerPC_EABI_Support/Runtime/NMWException.h" +#include "PowerPC_EABI_Support/Runtime/MWCPlusPlusLib.h" #pragma exceptions on @@ -14,14 +15,14 @@ namespace std abort(); } - static terminate_handler thandler = dthandler; + terminate_handler thandler = dthandler; static void duhandler() { terminate(); } - static unexpected_handler uhandler = duhandler; + unexpected_handler uhandler = duhandler; extern void terminate() { @@ -33,11 +34,11 @@ namespace std } } -extern "C" void __throw_catch_compare() { +extern "C" char __throw_catch_compare(const char* throwtype, const char* catchtype, s32* offset_result) { } -extern "C" void __construct_new_array() +extern "C" void *__construct_new_array(void *block, ConstructorDestructor ctor, ConstructorDestructor dtor_arg, size_t size, size_t n) { } @@ -83,7 +84,7 @@ extern "C" void __construct_array(void *ptr, void *ctor, void *dtor, size_t size } -extern "C" void __destroy_arr(void *block, void *dtor, size_t size, size_t n) +extern "C" void __destroy_arr(void *block, ConstructorDestructor *dtor, size_t size, size_t n) { -} \ No newline at end of file +} diff --git a/libs/PowerPC_EABI_Support/src/Runtime/global_destructor_chain.c b/libs/PowerPC_EABI_Support/src/Runtime/global_destructor_chain.c index 5876cd5..c017fde 100644 --- a/libs/PowerPC_EABI_Support/src/Runtime/global_destructor_chain.c +++ b/libs/PowerPC_EABI_Support/src/Runtime/global_destructor_chain.c @@ -1,4 +1,5 @@ #include "PowerPC_EABI_Support/Runtime/global_destructor_chain.h" +#include "PowerPC_EABI_Support/Runtime/MWCPlusPlusLib.h" DestructorChain *__global_destructor_chain; @@ -28,4 +29,4 @@ int __register_atexit(void (*func)(void)) { } -__declspec(section ".dtors") static void *const __destroy_global_chain_reference = __destroy_global_chain; \ No newline at end of file +__declspec(section ".dtors") static void *const __destroy_global_chain_reference = __destroy_global_chain; diff --git a/libs/dolphin/dsp/dsp.c b/libs/dolphin/dsp/dsp.c index 4487183..0e31951 100644 --- a/libs/dolphin/dsp/dsp.c +++ b/libs/dolphin/dsp/dsp.c @@ -122,4 +122,4 @@ DECL_WEAK DSPTaskInfo *DSPAddTask(DSPTaskInfo *task) #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/src/Kaneshige/Course/Course.cpp b/src/Kaneshige/Course/Course.cpp index 4821321..dba56ba 100644 --- a/src/Kaneshige/Course/Course.cpp +++ b/src/Kaneshige/Course/Course.cpp @@ -1867,13 +1867,14 @@ bool Course::Area::check(TVec3f *areaCoord, const TVec3f &worldPos) switch (mData->shape) { - case 1: + case 1: { f32 lenXZ = (areaCoord->x * areaCoord->x) + (areaCoord->z * areaCoord->z); if (lenXZ > mScale.x * mScale.x) return false; if (areaCoord->y < -100.0f || areaCoord->y > mScale.y) return false; break; + } default: if ((areaCoord->x < -mScale.x) || (areaCoord->x > mScale.x)) return false; @@ -2013,4 +2014,4 @@ bool Course::searchAppendGround(const TVec3f &pos) } return ret; -} \ No newline at end of file +} diff --git a/tools/decompctx.py b/tools/decompctx.py old mode 100644 new mode 100755 index e86d5ef..87cfb7e --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -23,8 +23,8 @@ include_dirs = [ # Add additional include directories here ] -include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r"^#ifndef\s+(.*)$") +include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') +guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") defines = set() diff --git a/tools/download_tool.py b/tools/download_tool.py old mode 100644 new mode 100755 index 7b386a4..f4512d0 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -56,6 +56,22 @@ def dtk_url(tag: str) -> str: return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" +def objdiff_cli_url(tag: str) -> str: + uname = platform.uname() + suffix = "" + system = uname.system.lower() + if system == "darwin": + system = "macos" + elif system == "windows": + suffix = ".exe" + arch = uname.machine.lower() + if arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/objdiff" + return f"{repo}/releases/download/{tag}/objdiff-cli-{system}-{arch}{suffix}" + + def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" @@ -70,6 +86,7 @@ TOOLS: Dict[str, Callable[[str], str]] = { "binutils": binutils_url, "compilers": compilers_url, "dtk": dtk_url, + "objdiff-cli": objdiff_cli_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, } diff --git a/tools/project.py b/tools/project.py index 43ca9d5..b4469fc 100644 --- a/tools/project.py +++ b/tools/project.py @@ -17,7 +17,7 @@ import os import platform import sys from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast from . import ninja_syntax from .ninja_syntax import serialize_path @@ -29,24 +29,93 @@ if sys.platform == "cygwin": f"\n(Current path: {sys.executable})" ) +Library = Dict[str, Any] + class Object: def __init__(self, completed: bool, name: str, **options: Any) -> None: self.name = name - self.base_name = Path(name).with_suffix("") self.completed = completed self.options: Dict[str, Any] = { - "add_to_all": True, + "add_to_all": None, "asflags": None, - "extra_asflags": None, + "asm_dir": None, "cflags": None, - "extra_cflags": None, + "extra_asflags": [], + "extra_cflags": [], + "extra_clang_flags": [], + "host": None, + "lib": None, "mw_version": None, + "progress_category": None, "shift_jis": None, "source": name, + "src_dir": None, } self.options.update(options) + # Internal + self.src_path: Optional[Path] = None + self.asm_path: Optional[Path] = None + self.src_obj_path: Optional[Path] = None + self.asm_obj_path: Optional[Path] = None + self.host_obj_path: Optional[Path] = None + self.ctx_path: Optional[Path] = None + + def resolve(self, config: "ProjectConfig", lib: Library) -> "Object": + # Use object options, then library options + obj = Object(self.completed, self.name, **lib) + for key, value in self.options.items(): + if value is not None or key not in obj.options: + obj.options[key] = value + + # Use default options from config + def set_default(key: str, value: Any) -> None: + if obj.options[key] is None: + obj.options[key] = value + + set_default("add_to_all", True) + set_default("asflags", config.asflags) + set_default("asm_dir", config.asm_dir) + set_default("host", False) + set_default("mw_version", config.linker_version) + set_default("shift_jis", config.shift_jis) + set_default("src_dir", config.src_dir) + + # Validate progress categories + def check_category(category: str): + if not any(category == c.id for c in config.progress_categories): + sys.exit( + f"Progress category '{category}' missing from config.progress_categories" + ) + + progress_category = obj.options["progress_category"] + if isinstance(progress_category, list): + for category in progress_category: + check_category(category) + elif progress_category is not None: + check_category(progress_category) + + # Resolve paths + build_dir = config.out_path() + obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"] + if obj.options["asm_dir"] is not None: + obj.asm_path = ( + Path(obj.options["asm_dir"]) / obj.options["source"] + ).with_suffix(".s") + base_name = Path(self.name).with_suffix("") + obj.src_obj_path = build_dir / "src" / f"{base_name}.o" + obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o" + obj.host_obj_path = build_dir / "host" / f"{base_name}.o" + obj.ctx_path = build_dir / "src" / f"{base_name}.ctx" + return obj + + +class ProgressCategory: + def __init__(self, id: str, name: str) -> None: + self.id = id + self.name = name + class ProjectConfig: def __init__(self) -> None: @@ -69,37 +138,53 @@ class ProjectConfig: self.wrapper: Optional[Path] = None # If None, download wibo on Linux self.sjiswrap_tag: Optional[str] = None # Git tag self.sjiswrap_path: Optional[Path] = None # If None, download + self.objdiff_tag: Optional[str] = None # Git tag + self.objdiff_path: Optional[Path] = None # If None, download # Project config self.non_matching: bool = False self.build_rels: bool = True # Build REL files self.check_sha_path: Optional[Path] = None # Path to version.sha1 self.config_path: Optional[Path] = None # Path to config.yml - self.debug: bool = False # Build with debug info self.generate_map: bool = False # Generate map file(s) self.asflags: Optional[List[str]] = None # Assembler flags self.ldflags: Optional[List[str]] = None # Linker flags - self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.libs: Optional[List[Library]] = None # List of libraries self.linker_version: Optional[str] = None # mwld version self.version: Optional[str] = None # Version name self.warn_missing_config: bool = False # Warn on missing unit configuration self.warn_missing_source: bool = False # Warn on missing source file self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial - self.rel_empty_file: Optional[ - str - ] = None # Object name for generating empty RELs + self.rel_empty_file: Optional[str] = ( + None # Object name for generating empty RELs + ) self.shift_jis = ( True # Convert source files from UTF-8 to Shift JIS automatically ) self.reconfig_deps: Optional[List[Path]] = ( None # Additional re-configuration dependency files ) + self.custom_build_rules: Optional[List[Dict[str, Any]]] = ( + None # Custom ninja build rules + ) + self.custom_build_steps: Optional[Dict[str, List[Dict[str, Any]]]] = ( + None # Custom build steps, types are ["pre-compile", "post-compile", "post-link", "post-build"] + ) + self.generate_compile_commands: bool = ( + True # Generate compile_commands.json for clangd + ) + self.extra_clang_flags: List[str] = [] # Extra flags for clangd - # Progress output and progress.json config + # Progress output, progress.json and report.json config + self.progress = True # Enable report.json generation and CLI progress output self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( - True # Include individual modules, disable for large numbers of modules + False # Include individual modules, disable for large numbers of modules + ) + self.progress_categories: List[ProgressCategory] = [] # Additional categories + self.print_progress_categories: Union[bool, List[str]] = ( + True # Print additional progress categories in the CLI progress output ) # Progress fancy printing @@ -125,16 +210,52 @@ class ProjectConfig: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + # Creates a map of object names to Object instances + # Options are fully resolved from the library and object + def objects(self) -> Dict[str, Object]: + out = {} for lib in self.libs or {}: - for obj in lib["objects"]: - if obj.name == name: - return lib, obj - return None + objects: List[Object] = lib["objects"] + for obj in objects: + if obj.name in out: + sys.exit(f"Duplicate object name {obj.name}") + out[obj.name] = obj.resolve(self, lib) + return out + # Gets the output path for build-related files. def out_path(self) -> Path: return self.build_dir / str(self.version) + # Gets the path to the compilers directory. + # Exits the program if neither `compilers_path` nor `compilers_tag` is provided. + def compilers(self) -> Path: + if self.compilers_path: + return self.compilers_path + elif self.compilers_tag: + return self.build_dir / "compilers" + else: + sys.exit("ProjectConfig.compilers_tag missing") + + # Gets the wrapper to use for compiler commands, if set. + def compiler_wrapper(self) -> Optional[Path]: + wrapper = self.wrapper + + if self.use_wibo(): + wrapper = self.build_dir / "tools" / "wibo" + if not is_windows() and wrapper is None: + wrapper = Path("wine") + + return wrapper + + # Determines whether or not to use wibo as the compiler wrapper. + def use_wibo(self) -> bool: + return ( + self.wibo_tag is not None + and sys.platform == "linux" + and platform.machine() in ("i386", "x86_64") + and self.wrapper is None + ) + def is_windows() -> bool: return os.name == "nt" @@ -146,11 +267,26 @@ CHAIN = "cmd /c " if is_windows() else "" EXE = ".exe" if is_windows() else "" -def make_flags_str(cflags: Union[str, List[str]]) -> str: - if isinstance(cflags, list): - return " ".join(cflags) - else: - return cflags +def file_is_asm(path: Path) -> bool: + return path.suffix.lower() == ".s" + + +def file_is_c(path: Path) -> bool: + return path.suffix.lower() == ".c" + + +def file_is_cpp(path: Path) -> bool: + return path.suffix.lower() in (".cc", ".cp", ".cpp", ".cxx") + + +def file_is_c_cpp(path: Path) -> bool: + return file_is_c(path) or file_is_cpp(path) + + +def make_flags_str(flags: Optional[List[str]]) -> str: + if flags is None: + return "" + return " ".join(flags) # Load decomp-toolkit generated config.json @@ -166,15 +302,15 @@ def load_build_config( f = open(build_config_path, "r", encoding="utf-8") build_config: Dict[str, Any] = json.load(f) config_version = build_config.get("version") - if not config_version: - # Invalid config.json + if config_version is None: + print("Invalid config.json, regenerating...") f.close() os.remove(build_config_path) return None dtk_version = str(config.dtk_tag)[1:] # Strip v if versiontuple(config_version) < versiontuple(dtk_version): - # Outdated config.json + print("Outdated config.json, regenerating...") f.close() os.remove(build_config_path) return None @@ -183,19 +319,22 @@ def load_build_config( return build_config -# Generate build.ninja and objdiff.json +# Generate build.ninja, objdiff.json and compile_commands.json def generate_build(config: ProjectConfig) -> None: + config.validate() + objects = config.objects() build_config = load_build_config(config, config.out_path() / "config.json") - generate_build_ninja(config, build_config) - generate_objdiff_config(config, build_config) + generate_build_ninja(config, objects, build_config) + generate_objdiff_config(config, objects, build_config) + generate_compile_commands(config, objects, build_config) # Generate build.ninja def generate_build_ninja( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[Dict[str, Any]], ) -> None: - config.validate() - out = io.StringIO() n = ninja_syntax.Writer(out) n.variable("ninja_required_version", "1.3") @@ -213,13 +352,8 @@ def generate_build_ninja( # Variables ### n.comment("Variables") - ldflags = " ".join(config.ldflags or []) - if config.generate_map: - ldflags += " -mapunused" - if config.debug: - ldflags += " -g" - n.variable("ldflags", ldflags) - if not config.linker_version: + n.variable("ldflags", make_flags_str(config.ldflags)) + if config.linker_version is None: sys.exit("ProjectConfig.linker_version missing") n.variable("mw_version", Path(config.linker_version)) n.newline() @@ -231,6 +365,7 @@ def generate_build_ninja( build_path = config.out_path() progress_path = build_path / "progress.json" + report_path = build_path / "report.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -248,17 +383,27 @@ def generate_build_ninja( deps="gcc", ) + cargo_rule_written = False + + def write_cargo_rule(): + nonlocal cargo_rule_written + if not cargo_rule_written: + n.pool("cargo", 1) + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + pool="cargo", + depfile=Path("$target") / "release" / "$bin.d", + deps="gcc", + ) + cargo_rule_written = True + if config.dtk_path is not None and config.dtk_path.is_file(): dtk = config.dtk_path elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" - n.rule( - name="cargo", - command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", - description="CARGO $bin", - depfile=Path("$target") / "release" / "$bin.d", - deps="gcc", - ) + write_cargo_rule() n.build( outputs=dtk, rule="cargo", @@ -283,6 +428,35 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.dtk_tag missing") + if config.objdiff_path is not None and config.objdiff_path.is_file(): + objdiff = config.objdiff_path + elif config.objdiff_path is not None: + objdiff = build_tools_path / "release" / f"objdiff-cli{EXE}" + write_cargo_rule() + n.build( + outputs=objdiff, + rule="cargo", + inputs=config.objdiff_path / "Cargo.toml", + implicit=config.objdiff_path / "Cargo.lock", + variables={ + "bin": "objdiff-cli", + "target": build_tools_path, + }, + ) + elif config.objdiff_tag: + objdiff = build_tools_path / f"objdiff-cli{EXE}" + n.build( + outputs=objdiff, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "objdiff-cli", + "tag": config.objdiff_tag, + }, + ) + else: + sys.exit("ProjectConfig.objdiff_tag missing") + if config.sjiswrap_path: sjiswrap = config.sjiswrap_path elif config.sjiswrap_tag: @@ -299,16 +473,10 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.sjiswrap_tag missing") + wrapper = config.compiler_wrapper() # Only add an implicit dependency on wibo if we download it - wrapper = config.wrapper wrapper_implicit: Optional[Path] = None - if ( - config.wibo_tag is not None - and sys.platform == "linux" - and platform.machine() in ("i386", "x86_64") - and config.wrapper is None - ): - wrapper = build_tools_path / "wibo" + if wrapper is not None and config.use_wibo(): wrapper_implicit = wrapper n.build( outputs=wrapper, @@ -319,15 +487,11 @@ def generate_build_ninja( "tag": config.wibo_tag, }, ) - if not is_windows() and wrapper is None: - wrapper = Path("wine") wrapper_cmd = f"{wrapper} " if wrapper else "" + compilers = config.compilers() compilers_implicit: Optional[Path] = None - if config.compilers_path: - compilers = config.compilers_path - elif config.compilers_tag: - compilers = config.build_dir / "compilers" + if config.compilers_path is None and config.compilers_tag is not None: compilers_implicit = compilers n.build( outputs=compilers, @@ -338,8 +502,6 @@ def generate_build_ninja( "tag": config.compilers_tag, }, ) - else: - sys.exit("ProjectConfig.compilers_tag missing") binutils_implicit = None if config.binutils_path: @@ -361,6 +523,17 @@ def generate_build_ninja( n.newline() + ### + # Helper rule for downloading all tools + ### + n.comment("Download all tools") + n.build( + outputs="tools", + rule="phony", + inputs=[dtk, sjiswrap, wrapper, compilers, binutils, objdiff], + ) + n.newline() + ### # Build rules ### @@ -443,6 +616,49 @@ def generate_build_ninja( ) n.newline() + if len(config.custom_build_rules or {}) > 0: + n.comment("Custom project build rules (pre/post-processing)") + for rule in config.custom_build_rules or {}: + n.rule( + name=cast(str, rule.get("name")), + command=cast(str, rule.get("command")), + description=rule.get("description", None), + depfile=rule.get("depfile", None), + generator=rule.get("generator", False), + pool=rule.get("pool", None), + restat=rule.get("restat", False), + rspfile=rule.get("rspfile", None), + rspfile_content=rule.get("rspfile_content", None), + deps=rule.get("deps", None), + ) + n.newline() + + def write_custom_step(step: str) -> List[str | Path]: + implicit: List[str | Path] = [] + if config.custom_build_steps and step in config.custom_build_steps: + n.comment(f"Custom build steps ({step})") + for custom_step in config.custom_build_steps[step]: + outputs = cast(List[str | Path], custom_step.get("outputs")) + + if isinstance(outputs, list): + implicit.extend(outputs) + else: + implicit.append(outputs) + + n.build( + outputs=outputs, + rule=cast(str, custom_step.get("rule")), + inputs=custom_step.get("inputs", None), + implicit=custom_step.get("implicit", None), + order_only=custom_step.get("order_only", None), + variables=custom_step.get("variables", None), + implicit_outputs=custom_step.get("implicit_outputs", None), + pool=custom_step.get("pool", None), + dyndep=custom_step.get("dyndep", None), + ) + n.newline() + return implicit + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -461,14 +677,13 @@ def generate_build_ninja( ) n.newline() + # Add all build steps needed before we compile (e.g. processing assets) + precompile_implicit = write_custom_step("pre-compile") + ### # Source files ### n.comment("Source files") - build_asm_path = build_path / "mod" - build_src_path = build_path / "src" - build_host_path = build_path / "host" - build_config_path = build_path / "config.json" def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") @@ -500,7 +715,6 @@ def generate_build_ninja( n.comment(f"Link {self.name}") if self.module_id == 0: elf_path = build_path / f"{self.name}.elf" - dol_path = build_path / f"{self.name}.dol" elf_ldflags = f"$ldflags -lcf {serialize_path(self.ldscript)}" if config.generate_map: elf_map = map_path(elf_path) @@ -511,16 +725,15 @@ def generate_build_ninja( outputs=elf_path, rule="link", inputs=self.inputs, - implicit=[self.ldscript, *mwld_implicit], + implicit=[ + *precompile_implicit, + self.ldscript, + *mwld_implicit, + *postcompile_implicit, + ], implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, ) - n.build( - outputs=dol_path, - rule="elf2dol", - inputs=elf_path, - implicit=dtk, - ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" @@ -565,95 +778,101 @@ def generate_build_ninja( host_source_inputs: List[Path] = [] source_added: Set[Path] = set() - def c_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path - ) -> Optional[Path]: - cflags_str = make_flags_str(options["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_flags_str(options["extra_cflags"]) - cflags_str += " " + extra_cflags_str - used_compiler_versions.add(options["mw_version"]) - - src_obj_path = build_src_path / f"{obj.base_name}.o" - src_base_path = build_src_path / obj.base_name - + def c_build(obj: Object, src_path: Path) -> Optional[Path]: # Avoid creating duplicate build rules - if src_obj_path in source_added: - return src_obj_path - source_added.add(src_obj_path) + if obj.src_obj_path is None or obj.src_obj_path in source_added: + return obj.src_obj_path + source_added.add(obj.src_obj_path) - shift_jis = options["shift_jis"] - if shift_jis is None: - shift_jis = config.shift_jis + cflags = obj.options["cflags"] + extra_cflags = obj.options["extra_cflags"] + + # Add appropriate language flag if it doesn't exist already + # Added directly to the source so it flows to other generation tasks + if not any(flag.startswith("-lang") for flag in cflags) and not any( + flag.startswith("-lang") for flag in extra_cflags + ): + # Ensure extra_cflags is a unique instance, + # and insert into there to avoid modifying shared sets of flags + extra_cflags = obj.options["extra_cflags"] = list(extra_cflags) + if file_is_cpp(src_path): + extra_cflags.insert(0, "-lang=c++") + else: + extra_cflags.insert(0, "-lang=c") + + cflags_str = make_flags_str(cflags) + if len(extra_cflags) > 0: + extra_cflags_str = make_flags_str(extra_cflags) + cflags_str += " " + extra_cflags_str + used_compiler_versions.add(obj.options["mw_version"]) # Add MWCC build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=src_obj_path, - rule="mwcc_sjis" if shift_jis else "mwcc", + outputs=obj.src_obj_path, + rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc", inputs=src_path, variables={ - "mw_version": Path(options["mw_version"]), + "mw_version": Path(obj.options["mw_version"]), "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": src_base_path, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), }, - implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + implicit=( + mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit + ), ) # Add ctx build rule - ctx_path = build_src_path / f"{obj.base_name}.ctx" - n.build( - outputs=ctx_path, - rule="decompctx", - inputs=src_path, - implicit=decompctx, - ) + if obj.ctx_path is not None: + n.build( + outputs=obj.ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) # Add host build rule - if options.get("host", False): - host_obj_path = build_host_path / f"{obj.base_name}.o" - host_base_path = build_host_path / obj.base_name + if obj.options["host"] and obj.host_obj_path is not None: n.build( - outputs=host_obj_path, - rule="host_cc" if src_path.suffix == ".c" else "host_cpp", + outputs=obj.host_obj_path, + rule="host_cc" if file_is_c(src_path) else "host_cpp", inputs=src_path, variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": host_base_path, + "basedir": os.path.dirname(obj.host_obj_path), + "basefile": obj.host_obj_path.with_suffix(""), }, ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) + if obj.options["add_to_all"]: + host_source_inputs.append(obj.host_obj_path) n.newline() - if options["add_to_all"]: - source_inputs.append(src_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj.src_obj_path) - return src_obj_path + return obj.src_obj_path def asm_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + obj: Object, src_path: Path, obj_path: Optional[Path] ) -> Optional[Path]: - asflags = options["asflags"] or config.asflags - if asflags is None: + if obj.options["asflags"] is None: sys.exit("ProjectConfig.asflags missing") - asflags_str = make_flags_str(asflags) - if options["extra_asflags"] is not None: - extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str = make_flags_str(obj.options["asflags"]) + if len(obj.options["extra_asflags"]) > 0: + extra_asflags_str = make_flags_str(obj.options["extra_asflags"]) asflags_str += " " + extra_asflags_str - asm_obj_path = build_asm_path / f"{obj.base_name}.o" - # Avoid creating duplicate build rules - if asm_obj_path in source_added: - return asm_obj_path - source_added.add(asm_obj_path) + if obj_path is None or obj_path in source_added: + return obj_path + source_added.add(obj_path) # Add assembler build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=asm_obj_path, + outputs=obj_path, rule="as", inputs=src_path, variables={"asflags": asflags_str}, @@ -661,57 +880,40 @@ def generate_build_ninja( ) n.newline() - if options["add_to_all"]: - source_inputs.append(asm_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj_path) - return asm_obj_path + return obj_path def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] - result = config.find_object(obj_name) - if not result: + obj = objects.get(obj_name) + if obj is None: if config.warn_missing_config and not build_obj["autogenerated"]: print(f"Missing configuration for {obj_name}") link_step.add(obj_path) return - lib, obj = result - lib_name = lib["lib"] - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - - unit_asm_path: Optional[Path] = None - if config.asm_dir is not None: - unit_asm_path = ( - Path(lib.get("asm_dir", config.asm_dir)) / options["source"] - ).with_suffix(".s") - link_built_obj = obj.completed built_obj_path: Optional[Path] = None - if unit_src_path.exists(): - if unit_src_path.suffix in (".c", ".C", ".cp", ".cpp"): + if obj.src_path is not None and obj.src_path.exists(): + if file_is_c_cpp(obj.src_path): # Add MWCC & host build rules - built_obj_path = c_build(obj, options, lib_name, unit_src_path) - elif unit_src_path.suffix == ".s": + built_obj_path = c_build(obj, obj.src_path) + elif file_is_asm(obj.src_path): # Add assembler build rule - built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + built_obj_path = asm_build(obj, obj.src_path, obj.src_obj_path) else: - sys.exit(f"Unknown source file type {unit_src_path}") + sys.exit(f"Unknown source file type {obj.src_path}") else: if config.warn_missing_source or obj.completed: - print(f"Missing source file {unit_src_path}") + print(f"Missing source file {obj.src_path}") link_built_obj = False # Assembly overrides - if unit_asm_path is not None and unit_asm_path.exists(): + if obj.asm_path is not None and obj.asm_path.exists(): link_built_obj = True - built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path) if link_built_obj and built_obj_path is not None: # Use the source-built object @@ -720,7 +922,10 @@ def generate_build_ninja( # Use the original (extracted) object link_step.add(obj_path) else: - sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") + lib_name = obj.options["lib"] + sys.exit( + f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}" + ) # Add DOL link step link_step = LinkStep(build_config) @@ -736,7 +941,7 @@ def generate_build_ninja( add_unit(unit, module_link_step) # Add empty object to empty RELs if len(module_link_step.inputs) == 0: - if not config.rel_empty_file: + if config.rel_empty_file is None: sys.exit("ProjectConfig.rel_empty_file missing") add_unit( { @@ -760,6 +965,9 @@ def generate_build_ninja( if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") + # Add all build steps needed before we link and after compiling objects + postcompile_implicit = write_custom_step("post-compile") + ### # Link ### @@ -768,6 +976,19 @@ def generate_build_ninja( link_outputs.append(step.output()) n.newline() + # Add all build steps needed after linking and before GC/Wii native format generation + postlink_implicit = write_custom_step("post-link") + + ### + # Generate DOL + ### + n.build( + outputs=link_steps[0].output(), + rule="elf2dol", + inputs=link_steps[0].partial_output(), + implicit=[*postlink_implicit, dtk], + ) + ### # Generate RELs ### @@ -782,7 +1003,7 @@ def generate_build_ninja( rspfile="$rspfile", rspfile_content="$in_newline", ) - generated_rels = [] + generated_rels: List[str] = [] for idx, link in enumerate(build_config["links"]): # Map module names to link steps link_steps_local = list( @@ -830,6 +1051,9 @@ def generate_build_ninja( ) n.newline() + # Add all build steps needed post-build (re-building archives and such) + postbuild_implicit = write_custom_step("post-build") + ### # Helper rule for building all source files ### @@ -867,7 +1091,7 @@ def generate_build_ninja( outputs=ok_path, rule="check", inputs=config.check_sha_path, - implicit=[dtk, *link_outputs], + implicit=[dtk, *link_outputs, *postbuild_implicit], ) n.newline() @@ -883,7 +1107,28 @@ def generate_build_ninja( n.build( outputs=progress_path, rule="progress", - implicit=[ok_path, configure_script, python_lib, config.config_path], + implicit=[ + ok_path, + configure_script, + python_lib, + report_path, + ], + ) + + ### + # Generate progress report + ### + n.comment("Generate progress report") + n.rule( + name="report", + command=f"{objdiff} report generate -o $out", + description="REPORT", + ) + report_implicit: List[str | Path] = [objdiff, "all_source"] + n.build( + outputs=report_path, + rule="report", + implicit=report_implicit, ) ### @@ -932,6 +1177,7 @@ def generate_build_ninja( ### # Split DOL ### + build_config_path = build_path / "config.json" n.comment("Split DOL into relocatable objects") n.rule( name="split", @@ -967,7 +1213,7 @@ def generate_build_ninja( configure_script, python_lib, python_lib_dir / "ninja_syntax.py", - *(config.reconfig_deps or []) + *(config.reconfig_deps or []), ], ) n.newline() @@ -979,8 +1225,10 @@ def generate_build_ninja( if build_config: if config.non_matching: n.default(link_outputs) - else: + elif config.progress: n.default(progress_path) + else: + n.default(ok_path) else: n.default(build_config_path) @@ -992,13 +1240,22 @@ def generate_build_ninja( # Generate objdiff.json def generate_objdiff_config( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[Dict[str, Any]], ) -> None: - if not build_config: + if build_config is None: return + # Load existing objdiff.json + existing_units = {} + if Path("objdiff.json").is_file(): + with open("objdiff.json", "r", encoding="utf-8") as r: + existing_config = json.load(r) + existing_units = {unit["name"]: unit for unit in existing_config["units"]} + objdiff_config: Dict[str, Any] = { - "min_version": "1.0.0", + "min_version": "2.0.0-beta.5", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -1014,16 +1271,17 @@ def generate_objdiff_config( "*.json", ], "units": [], + "progress_categories": [], } # decomp.me compiler name mapping - # Commented out versions have not been added to decomp.me yet COMPILER_MAP = { "GC/1.0": "mwcc_233_144", "GC/1.1": "mwcc_233_159", "GC/1.2.5": "mwcc_233_163", "GC/1.2.5e": "mwcc_233_163e", "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3": "mwcc_242_53", "GC/1.3.2": "mwcc_242_81", "GC/1.3.2r": "mwcc_242_81r", "GC/2.0": "mwcc_247_92", @@ -1048,91 +1306,138 @@ def generate_objdiff_config( "Wii/1.7": "mwcc_43_213", } - build_path = config.out_path() - - def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: - if build_obj["autogenerated"]: - # Skip autogenerated objects - return - + def add_unit( + build_obj: Dict[str, Any], module_name: str, progress_categories: List[str] + ) -> None: obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") + name = str(Path(module_name) / base_object).replace(os.sep, "/") unit_config: Dict[str, Any] = { - "name": Path(module_name) / base_object, + "name": name, "target_path": obj_path, + "base_path": None, + "scratch": None, + "metadata": { + "complete": None, + "reverse_fn_order": None, + "source_path": None, + "progress_categories": progress_categories, + "auto_generated": build_obj["autogenerated"], + }, + "symbol_mappings": None, } - result = config.find_object(obj_name) - if not result: + # Preserve existing symbol mappings + existing_unit = existing_units.get(name) + if existing_unit is not None: + unit_config["symbol_mappings"] = existing_unit.get("symbol_mappings") + + obj = objects.get(obj_name) + if obj is None: objdiff_config["units"].append(unit_config) return - lib, obj = result - src_dir = Path(lib.get("src_dir", config.src_dir)) - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = src_dir / str(options["source"]) - - if not unit_src_path.exists(): - objdiff_config["units"].append(unit_config) - return - - cflags = options["cflags"] - src_obj_path = build_path / "src" / f"{obj.base_name}.o" - src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" + src_exists = obj.src_path is not None and obj.src_path.exists() + if src_exists: + unit_config["base_path"] = obj.src_obj_path + unit_config["metadata"]["source_path"] = obj.src_path + cflags = obj.options["cflags"] reverse_fn_order = False - if type(cflags) is list: - for flag in cflags: - if not flag.startswith("-inline "): - continue - for value in flag.split(" ")[1].split(","): - if value == "deferred": - reverse_fn_order = True - elif value == "nodeferred": - reverse_fn_order = False + for flag in cflags: + if not flag.startswith("-inline "): + continue + for value in flag.split(" ")[1].split(","): + if value == "deferred": + reverse_fn_order = True + elif value == "nodeferred": + reverse_fn_order = False - # Filter out include directories - def keep_flag(flag): - return not flag.startswith("-i ") and not flag.startswith("-I ") + # Filter out include directories + def keep_flag(flag): + return not flag.startswith("-i ") and not flag.startswith("-I ") - cflags = list(filter(keep_flag, cflags)) + cflags = list(filter(keep_flag, cflags)) - # Add appropriate lang flag - if unit_src_path.suffix in (".cp", ".cpp"): - cflags.insert(0, "-lang=c++") - else: - cflags.insert(0, "-lang=c") - - unit_config["base_path"] = src_obj_path - unit_config["reverse_fn_order"] = reverse_fn_order - unit_config["complete"] = obj.completed - compiler_version = COMPILER_MAP.get(options["mw_version"]) + compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: - print(f"Missing scratch compiler mapping for {options['mw_version']}") + print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: + cflags_str = make_flags_str(cflags) + if len(obj.options["extra_cflags"]) > 0: + extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) + cflags_str += " " + extra_cflags_str unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, - "c_flags": make_flags_str(cflags), - "ctx_path": src_ctx_path, - "build_ctx": True, + "c_flags": cflags_str, } + if src_exists: + unit_config["scratch"].update( + { + "ctx_path": obj.ctx_path, + "build_ctx": True, + } + ) + category_opt: List[str] | str = obj.options["progress_category"] + if isinstance(category_opt, list): + progress_categories.extend(category_opt) + elif category_opt is not None: + progress_categories.append(category_opt) + unit_config["metadata"].update( + { + "complete": obj.completed, + "reverse_fn_order": reverse_fn_order, + "progress_categories": progress_categories, + } + ) objdiff_config["units"].append(unit_config) # Add DOL units for unit in build_config["units"]: - add_unit(unit, build_config["name"]) + progress_categories = [] + # Only include a "dol" category if there are any modules + # Otherwise it's redundant with the global report measures + if len(build_config["modules"]) > 0: + progress_categories.append("dol") + add_unit(unit, build_config["name"], progress_categories) # Add REL units for module in build_config["modules"]: for unit in module["units"]: - add_unit(unit, module["name"]) + progress_categories = [] + if config.progress_modules: + progress_categories.append("modules") + if config.progress_each_module: + progress_categories.append(module["name"]) + add_unit(unit, module["name"], progress_categories) + + # Add progress categories + def add_category(id: str, name: str): + objdiff_config["progress_categories"].append( + { + "id": id, + "name": name, + } + ) + + if len(build_config["modules"]) > 0: + add_category("dol", "DOL") + if config.progress_modules: + add_category("modules", "Modules") + if config.progress_each_module: + for module in build_config["modules"]: + add_category(module["name"], module["name"]) + for category in config.progress_categories: + add_category(category.id, category.name) + + def cleandict(d): + if isinstance(d, dict): + return {k: cleandict(v) for k, v in d.items() if v is not None} + elif isinstance(d, list): + return [cleandict(v) for v in d] + else: + return d # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: @@ -1140,140 +1445,343 @@ def generate_objdiff_config( def unix_path(input: Any) -> str: return str(input).replace(os.sep, "/") if input else "" - json.dump(objdiff_config, w, indent=4, default=unix_path) + json.dump(cleandict(objdiff_config), w, indent=2, default=unix_path) + + +def generate_compile_commands( + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[Dict[str, Any]], +) -> None: + if build_config is None or not config.generate_compile_commands: + return + + # The following code attempts to convert mwcc flags to clang flags + # for use with clangd. + + # Flags to ignore explicitly + CFLAG_IGNORE: Set[str] = { + # Search order modifier + # Has a different meaning to Clang, and would otherwise + # be picked up by the include passthrough prefix + "-I-", + "-i-", + } + CFLAG_IGNORE_PREFIX: Tuple[str, ...] = ( + # Recursive includes are not supported by modern compilers + "-ir ", + ) + + # Flags to replace + CFLAG_REPLACE: Dict[str, str] = {} + CFLAG_REPLACE_PREFIX: Tuple[Tuple[str, str], ...] = ( + # Includes + ("-i ", "-I"), + ("-I ", "-I"), + ("-I+", "-I"), + # Defines + ("-d ", "-D"), + ("-D ", "-D"), + ("-D+", "-D"), + ) + + # Flags with a finite set of options + CFLAG_REPLACE_OPTIONS: Tuple[Tuple[str, Dict[str, Tuple[str, ...]]], ...] = ( + # Exceptions + ( + "-Cpp_exceptions", + { + "off": ("-fno-cxx-exceptions",), + "on": ("-fcxx-exceptions",), + }, + ), + # RTTI + ( + "-RTTI", + { + "off": ("-fno-rtti",), + "on": ("-frtti",), + }, + ), + # Language configuration + ( + "-lang", + { + "c": ("--language=c", "--std=c99"), + "c99": ("--language=c", "--std=c99"), + "c++": ("--language=c++", "--std=c++98"), + "cplus": ("--language=c++", "--std=c++98"), + }, + ), + # Enum size + ( + "-enum", + { + "min": ("-fshort-enums",), + "int": ("-fno-short-enums",), + }, + ), + # Common BSS + ( + "-common", + { + "off": ("-fno-common",), + "on": ("-fcommon",), + }, + ), + ) + + # Flags to pass through + CFLAG_PASSTHROUGH: Set[str] = set() + CFLAG_PASSTHROUGH_PREFIX: Tuple[str, ...] = ( + "-I", # includes + "-D", # defines + ) + + clangd_config = [] + + def add_unit(build_obj: Dict[str, Any]) -> None: + obj = objects.get(build_obj["name"]) + if obj is None: + return + + # Skip unresolved objects + if ( + obj.src_path is None + or obj.src_obj_path is None + or not file_is_c_cpp(obj.src_path) + ): + return + + # Gather cflags for source file + cflags: list[str] = [] + + def append_cflags(flags: Iterable[str]) -> None: + # Match a flag against either a set of concrete flags, or a set of prefixes. + def flag_match( + flag: str, concrete: Set[str], prefixes: Tuple[str, ...] + ) -> bool: + if flag in concrete: + return True + + for prefix in prefixes: + if flag.startswith(prefix): + return True + + return False + + # Determine whether a flag should be ignored. + def should_ignore(flag: str) -> bool: + return flag_match(flag, CFLAG_IGNORE, CFLAG_IGNORE_PREFIX) + + # Determine whether a flag should be passed through. + def should_passthrough(flag: str) -> bool: + return flag_match(flag, CFLAG_PASSTHROUGH, CFLAG_PASSTHROUGH_PREFIX) + + # Attempts replacement for the given flag. + def try_replace(flag: str) -> bool: + replacement = CFLAG_REPLACE.get(flag) + if replacement is not None: + cflags.append(replacement) + return True + + for prefix, replacement in CFLAG_REPLACE_PREFIX: + if flag.startswith(prefix): + cflags.append(flag.replace(prefix, replacement, 1)) + return True + + for prefix, options in CFLAG_REPLACE_OPTIONS: + if not flag.startswith(prefix): + continue + + # "-lang c99" and "-lang=c99" are both generally valid option forms + option = flag.removeprefix(prefix).removeprefix("=").lstrip() + replacements = options.get(option) + if replacements is not None: + cflags.extend(replacements) + + return True + + return False + + for flag in flags: + # Ignore flags first + if should_ignore(flag): + continue + + # Then find replacements + if try_replace(flag): + continue + + # Pass flags through last + if should_passthrough(flag): + cflags.append(flag) + continue + + append_cflags(obj.options["cflags"]) + append_cflags(obj.options["extra_cflags"]) + cflags.extend(config.extra_clang_flags) + cflags.extend(obj.options["extra_clang_flags"]) + + unit_config = { + "directory": Path.cwd(), + "file": obj.src_path, + "output": obj.src_obj_path, + "arguments": [ + "clang", + "-nostdinc", + "-fno-builtin", + "--target=powerpc-eabi", + *cflags, + "-c", + obj.src_path, + "-o", + obj.src_obj_path, + ], + } + clangd_config.append(unit_config) + + # Add DOL units + for unit in build_config["units"]: + add_unit(unit) + + # Add REL units + for module in build_config["modules"]: + for unit in module["units"]: + add_unit(unit) + + # Write compile_commands.json + with open("compile_commands.json", "w", encoding="utf-8") as w: + + def default_format(o): + if isinstance(o, Path): + return o.resolve().as_posix() + return str(o) + + json.dump(clangd_config, w, indent=2, default=default_format) # Calculate, print and write progress to progress.json def calculate_progress(config: ProjectConfig) -> None: + config.validate() out_path = config.out_path() - build_config = load_build_config(config, out_path / "config.json") - if not build_config: - return + report_path = out_path / "report.json" + if not report_path.is_file(): + sys.exit(f"Report file {report_path} does not exist") - class ProgressUnit: - def __init__(self, name: str) -> None: - self.name: str = name - self.code_total: int = 0 - self.code_fancy_frac: int = config.progress_code_fancy_frac - self.code_fancy_item: str = config.progress_code_fancy_item - self.code_progress: int = 0 - self.data_total: int = 0 - self.data_fancy_frac: int = config.progress_data_fancy_frac - self.data_fancy_item: str = config.progress_data_fancy_item - self.data_progress: int = 0 - self.objects_progress: int = 0 - self.objects_total: int = 0 - self.objects: Set[Object] = set() + report_data: Dict[str, Any] = {} + with open(report_path, "r", encoding="utf-8") as f: + report_data = json.load(f) - def add(self, build_obj: Dict[str, Any]) -> None: - self.code_total += build_obj["code_size"] - self.data_total += build_obj["data_size"] + # Convert string numbers (u64) to int + def convert_numbers(data: Dict[str, Any]) -> None: + for key, value in data.items(): + if isinstance(value, str) and value.isdigit(): + data[key] = int(value) - # Avoid counting the same object in different modules twice - include_object = build_obj["name"] not in self.objects - if include_object: - self.objects.add(build_obj["name"]) - self.objects_total += 1 + convert_numbers(report_data["measures"]) + for category in report_data.get("categories", []): + convert_numbers(category["measures"]) - if build_obj["autogenerated"]: - # Skip autogenerated objects - return + # Output to GitHub Actions job summary, if available + summary_path = os.getenv("GITHUB_STEP_SUMMARY") + summary_file: Optional[IO[str]] = None + if summary_path: + summary_file = open(summary_path, "a", encoding="utf-8") + summary_file.write("```\n") - result = config.find_object(build_obj["name"]) - if not result: - return - - _, obj = result - if not obj.completed: - return - - self.code_progress += build_obj["code_size"] - self.data_progress += build_obj["data_size"] - if include_object: - self.objects_progress += 1 - - def code_frac(self) -> float: - return self.code_progress / self.code_total - - def data_frac(self) -> float: - return self.data_progress / self.data_total - - # Add DOL units - all_progress = ProgressUnit("All") if config.progress_all else None - dol_progress = ProgressUnit("DOL") - for unit in build_config["units"]: - if all_progress: - all_progress.add(unit) - dol_progress.add(unit) - - # Add REL units - rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress: List[ProgressUnit] = [] - for module in build_config["modules"]: - progress = ProgressUnit(module["name"]) - modules_progress.append(progress) - for unit in module["units"]: - if all_progress: - all_progress.add(unit) - if rels_progress: - rels_progress.add(unit) - progress.add(unit) + def progress_print(s: str) -> None: + print(s) + if summary_file: + summary_file.write(s + "\n") # Print human-readable progress - print("Progress:") + progress_print("Progress:") - def print_category(unit: Optional[ProgressUnit]) -> None: - if unit is None: - return + def print_category(name: str, measures: Dict[str, Any]) -> None: + total_code = measures.get("total_code", 0) + matched_code = measures.get("matched_code", 0) + matched_code_percent = measures.get("matched_code_percent", 0) + total_data = measures.get("total_data", 0) + matched_data = measures.get("matched_data", 0) + matched_data_percent = measures.get("matched_data_percent", 0) + total_functions = measures.get("total_functions", 0) + matched_functions = measures.get("matched_functions", 0) + complete_code_percent = measures.get("complete_code_percent", 0) + total_units = measures.get("total_units", 0) + complete_units = measures.get("complete_units", 0) - code_frac = unit.code_frac() - data_frac = unit.data_frac() - print( - f" {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)" + progress_print( + f" {name}: {matched_code_percent:.2f}% matched, {complete_code_percent:.2f}% linked ({complete_units} / {total_units} files)" + ) + progress_print( + f" Code: {matched_code} / {total_code} bytes ({matched_functions} / {total_functions} functions)" + ) + progress_print( + f" Data: {matched_data} / {total_data} bytes ({matched_data_percent:.2f}%)" ) - print(f" Code: {unit.code_progress} / {unit.code_total} bytes") - print(f" Data: {unit.data_progress} / {unit.data_total} bytes") - if config.progress_use_fancy: - print( - "\nYou have {} out of {} {} and {} out of {} {}.".format( - math.floor(code_frac * unit.code_fancy_frac), - unit.code_fancy_frac, - unit.code_fancy_item, - math.floor(data_frac * unit.data_fancy_frac), - unit.data_fancy_frac, - unit.data_fancy_item, - ) - ) - if all_progress: - print_category(all_progress) - print_category(dol_progress) - module_count = len(build_config["modules"]) - if module_count > 0: - print_category(rels_progress) - if config.progress_each_module: - for progress in modules_progress: - print_category(progress) + print_category("All", report_data["measures"]) + for category in report_data.get("categories", []): + if config.print_progress_categories is True or ( + isinstance(config.print_progress_categories, list) + and category["id"] in config.print_progress_categories + ): + print_category(category["name"], category["measures"]) + + if config.progress_use_fancy: + measures = report_data["measures"] + total_code = measures.get("total_code", 0) + total_data = measures.get("total_data", 0) + if total_code == 0 or total_data == 0: + return + code_frac = measures.get("complete_code", 0) / total_code + data_frac = measures.get("complete_data", 0) / total_data + + progress_print( + "\nYou have {} out of {} {} and {} out of {} {}.".format( + math.floor(code_frac * config.progress_code_fancy_frac), + config.progress_code_fancy_frac, + config.progress_code_fancy_item, + math.floor(data_frac * config.progress_data_fancy_frac), + config.progress_data_fancy_frac, + config.progress_data_fancy_item, + ) + ) + + # Finalize GitHub Actions job summary + if summary_file: + summary_file.write("```\n") + summary_file.close() # Generate and write progress.json progress_json: Dict[str, Any] = {} - def add_category(category: str, unit: ProgressUnit) -> None: - progress_json[category] = { - "code": unit.code_progress, - "code/total": unit.code_total, - "data": unit.data_progress, - "data/total": unit.data_total, + def add_category(id: str, measures: Dict[str, Any]) -> None: + progress_json[id] = { + "code": measures.get("complete_code", 0), + "code/total": measures.get("total_code", 0), + "data": measures.get("complete_data", 0), + "data/total": measures.get("total_data", 0), + "matched_code": measures.get("matched_code", 0), + "matched_code/total": measures.get("total_code", 0), + "matched_data": measures.get("matched_data", 0), + "matched_data/total": measures.get("total_data", 0), + "matched_functions": measures.get("matched_functions", 0), + "matched_functions/total": measures.get("total_functions", 0), + "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100), + "fuzzy_match/total": 10000, + "units": measures.get("complete_units", 0), + "units/total": measures.get("total_units", 0), } - if all_progress: - add_category("all", all_progress) - add_category("dol", dol_progress) - if len(build_config["modules"]) > 0: - if rels_progress: - add_category("modules", rels_progress) - if config.progress_each_module: - for progress in modules_progress: - add_category(progress.name, progress) + if config.progress_all: + add_category("all", report_data["measures"]) + else: + # Support for old behavior where "dol" was the main category + add_category("dol", report_data["measures"]) + for category in report_data.get("categories", []): + add_category(category["id"], category["measures"]) + with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=4) + json.dump(progress_json, w, indent=2) diff --git a/tools/transform_dep.py b/tools/transform_dep.py old mode 100644 new mode 100755 diff --git a/tools/upload_progress.py b/tools/upload_progress.py old mode 100644 new mode 100755