Automate even more downloads (#191)

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - name: Build
       run: |
-        python configure.py --map --version ${{matrix.version}} --compilers /compilers/GC
+        python configure.py -m -v ${{matrix.version}} --compilers /compilers/GC --powerpc /opt/devkitpro/devkitPPC/bin
         ninja
     - name: Upload progress
       if: github.ref == 'refs/heads/main'
@@ -50,4 +50,4 @@ jobs:
     - name: Git config
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - name: Build
-      run: make -j$(nproc) MAPGENFLAG=1 VERSION=${{matrix.version}} COMPILERS=/compilers/GC
+      run: make -j$(nproc) MAPGENFLAG=1 VERSION=${{matrix.version}} COMPILERS=/compilers/GC POWERPC=/opt/devkitpro/devkitPPC/bin

.gitignore

@@ -25,8 +25,8 @@
 include/*.s
 
 build/
-tools/mwcc_compiler/*    
-!tools/mwcc_compiler/.gitkeep
+tools/mwcc_compiler/
+tools/powerpc/
 !aoi.exe
 decomp/
 errors.txt

Makefile

@@ -81,11 +81,11 @@ MWCC_VERSION := 2.6
 MWLD_VERSION := 2.6
 
 # Programs
-DEVKITPPC ?= tools/devkitPPC
+POWERPC ?= tools/powerpc
 ifeq ($(WINDOWS),1)
   WINE :=
-  AS      := $(DEVKITPPC)/bin/powerpc-eabi-as.exe
-  CPP     := $(DEVKITPPC)/bin/powerpc-eabi-cpp.exe -P
+  AS      := $(POWERPC)/powerpc-eabi-as.exe
+  CPP     := $(POWERPC)/powerpc-eabi-cpp.exe -P
   PYTHON  := python
 else
   WIBO   := $(shell command -v wibo 2> /dev/null)
@@ -96,8 +96,8 @@ else
   endif
   # Disable wine debug output for cleanliness
   export WINEDEBUG ?= -all
-  AS      := $(DEVKITPPC)/bin/powerpc-eabi-as
-  CPP     := $(DEVKITPPC)/bin/powerpc-eabi-cpp -P
+  AS      := $(POWERPC)/powerpc-eabi-as
+  CPP     := $(POWERPC)/powerpc-eabi-cpp -P
   PYTHON  := python3
 endif
 COMPILERS ?= tools/mwcc_compiler

README.MD

@@ -28,13 +28,9 @@ This repository builds the following DOLs:
 
 1. Clone the repo using `git clone https://github.com/projectPiki/pikmin2/`
 
-2. Install required python modules with `pip install -r requirements.txt`.
+2. Run `configure.py` to automatically download/build any other necessary files for the project.
 
-3. Download [GC_WII_COMPILERS.zip](https://files.decomp.dev/compilers_20230715.zip) and extract the contents of the GC folder to `tools/mwcc_compiler/`. For example, your directory structure should look like `pikmin2/tools/mwcc_compiler/2.6/` (along with the other versions).
-
-4. Run `python configure.py` to automatically download/build any other necessary files for the project.
-
-5. Run `ninja` (recommended) or `make -j` in a command prompt or terminal.
+3. Enter `ninja` (recommended) or `make -j` in a command prompt or terminal.
 	- `-j` Allows `make` to use multiple threads, speeding up the process.
 	- `ninja` already implicitly uses the max amount of cores available, only use `-j` to *decrease* the thread count.
 	- If just `-j` gives errors on your setup, try specifying a set number of threads, e.g. `make -j 4`.
@@ -59,20 +55,22 @@ This repository builds the following DOLs:
 - To set up objdiff for this project:
 	1. Run `objdiff.exe`.
 	2. Set:
-		- Build config: `None`.
-		- Select project dir: `/path/to/cloned/pikmin2`
-		- Select target build dir: `/path/to/cloned/pikmin2/build/pikmin2.usa/asm`
-		- Select base build dir: `/path/to/cloned/pikmin2/build/pikmin2.usa/src`
-			- NB: you may have to specify a custom make path if the desired `make` version is not the default. On Windows, this is likely:
-			`/path/to/devkitPro/msys2/usr/bin/make.exe`
-	3. Select desired object file to diff (likely from `pikmin2/build/pikmin2.usa/asm/LIBRARY/OBJECTFILE.o`).
+		- Project dir: `/path/to/cloned/pikmin2`.
+		- Custom make program: `ninja` or `make`, depending on what was used to build previously.
+			- If specifying the name isn't enough, it's likely the program isn't on your `PATH`; try an absolute path instead.
+		- (`make` only) Build config: `None`.
+		- (`make` only) Select target build dir: `/path/to/cloned/pikmin2/build/pikmin2.usa/asm`.
+		- (`make` only) Select base build dir: `/path/to/cloned/pikmin2/build/pikmin2.usa/src`.
+	3. Select desired object file to diff:
+		- With `ninja`, an explorer viewer is provided after applying the previous settings.
+		- With `make`, you must explicitly select the object each time (likely from `pikmin2/build/pikmin2.usa/asm/LIBRARY/OBJECTFILE.o`).
 	4. Decomp it!
 
 ### Generating Context for decomp.me Scratches
 
 - [decomp.me](https://decomp.me/) is an online decompilation sharing hub, allowing 'scratches' of functions to be generated and collaborated on.
 - Stand-alone decompilation packages and tools such as decomp.me require information on the functions and structures of the project in order to parse extracted blocks correctly. The easiest way to do this is to pass the tool just the necessary 'context' for the file, i.e. a set of all the headers used by the file that's being worked on.
-- A recursive context processing script is included in the repo (<a href="https://github.com/projectPiki/pikmin2/tree/main/tools/decompctx.py">tools/decompctx.py</a>), which generates a `ctx.c` file in the root directory.
+- A recursive context processing script is included in the repo ([tools/decompctx.py](https://github.com/projectPiki/pikmin2/tree/main/tools/decompctx.py)), which generates a `ctx.c` file in the root directory.
 	- The contents of this can then be copied and pasted into the 'Context' section of a decomp.me scratch or similar.
 - To use, call the generator via the terminal/command line from the root directory (replacing DIRECTORY and FILE as required):
 	```python tools/decompctx.py src/DIRECTORY/FILE.cpp```

asmdiff.sh

@@ -1,8 +1,8 @@
 #!/bin/bash -e
 
 VERSION="${VERSION:=usa}"
-DEVKITPPC="${DEVKITPPC:=tools/devkitPPC}"
-OBJDUMP="$DEVKITPPC/bin/powerpc-eabi-objdump -Dz -bbinary -EB -mpowerpc -M gekko"
+POWERPC="${POWERPC:=tools/powerpc}"
+OBJDUMP="$POWERPC/powerpc-eabi-objdump -Dz -bbinary -EB -mpowerpc -M gekko"
 if [ ! -z "$1" ]; then
   OPTIONS="--start-address=$(($1)) --stop-address=$(($2))"
 fi

configure.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 LIBS = [
     {
         "lib": "JStudio_JParticle",
@@ -1660,7 +1658,7 @@ LIBS = [
     },
 ]
 
-if __name__ == "__main__":
+def main():
     import os
     import io
     import sys
@@ -1702,10 +1700,11 @@ if __name__ == "__main__":
         help="don't build and use static libs",
     )
     parser.add_argument(
-        "--devkitppc",
-        dest="devkitppc",
+        "--powerpc",
+        dest="powerpc",
         type=Path,
-        help="path to devkitPPC",
+        default=Path("tools/powerpc"),
+        help="path to powerpc-eabi tools",
     )
     if os.name != "nt" and not "_NT-" in os.uname().sysname:
         parser.add_argument(
@@ -1754,9 +1753,6 @@ if __name__ == "__main__":
 
     n.comment("The arguments passed to configure.py, for rerunning it.")
     configure_args = sys.argv[1:]
-    # Ignore DEVKITPPC env var on Windows
-    if os.name != "nt" and "DEVKITPPC" in os.environ and not args.devkitppc:
-        configure_args.extend(["--devkitppc", os.environ["DEVKITPPC"]])
     n.variable("configure_args", configure_args)
     n.variable("python", f'"{sys.executable}"')
     n.newline()
@@ -1773,15 +1769,6 @@ if __name__ == "__main__":
     else:
         sys.exit(f'Invalid version "{args.version}"')
     build_path = args.build_dir / f"pikmin2.{version}"
-    if args.devkitppc:
-        dkp_path = args.devkitppc
-    elif "DEVKITPPC" in os.environ:
-        dkp_path = Path(os.environ["DEVKITPPC"])
-    else:
-        dkp_path = Path("tools/devkitPPC")
-        if not dkp_path.exists():
-            import tools.download_ppc
-            tools.download_ppc.main()
 
     cflags_base = f"-proc gekko -nodefaults -Cpp_exceptions off -RTTI off -fp hard -fp_contract on -O4,p -maxerrors 1 -enum int -inline auto -str reuse,readonly -nosyspath -use_lmw_stmw on -sdata 8 -sdata2 8 -DVERNUM={version_num} -i include -i include/stl"
     if args.debug:
@@ -1825,8 +1812,6 @@ if __name__ == "__main__":
     ###
     # Tooling
     ###
-    n.comment("decomp-toolkit")
-
     tools_path = Path("tools")
 
     def path(input):
@@ -1837,6 +1822,7 @@ if __name__ == "__main__":
         else:
             return [str(input)]
 
+    n.comment("decomp-toolkit")
     if args.build_dtk:
         dtk = tools_path / "release" / f"dtk{exe}"
         n.rule(
@@ -1871,13 +1857,51 @@ if __name__ == "__main__":
         )
     n.newline()
 
+    if args.powerpc == Path("tools/powerpc"):
+        n.comment("powerpc")
+        download_ppc = tools_path / "download_ppc.py"
+        n.rule(
+            name="download_ppc",
+            command=f"$python {download_ppc}",
+            description="DOWNLOAD $out",
+        )
+        n.build(
+            outputs=path("tools/powerpc/finish"),
+            rule="download_ppc",
+            implicit=path(["tools/powerpc", download_ppc]),
+        )
+        n.newline()
+        # FIXME: Manual download because the above doesn't work for some reason
+        if not Path("tools/powerpc").exists():
+            import tools.download_ppc
+            tools.download_ppc.main()
+
+    if args.compilers == Path("tools/mwcc_compiler"):
+        n.comment("mwcc-compilers")
+        download_mwcc = tools_path / "download_mwcc.py"
+        n.rule(
+            name="download_mwcc",
+            command=f"$python {download_mwcc}",
+            description="DOWNLOAD $out",
+        )
+        n.build(
+            outputs=path("tools/mwcc_compiler"),
+            rule="download_mwcc",
+            implicit=path(["tools/mwcc_compiler", download_mwcc]),
+        )
+        n.newline()
+        # FIXME: Manual download because the above doesn't work for some reason
+        if not Path("tools/mwcc_compiler").exists():
+            import tools.download_mwcc
+            tools.download_mwcc.main()
+
     ###
     # Rules
     ###
     compiler_path = args.compilers / "$mw_version"
     mwcc = compiler_path / "mwcceppc.exe"
     mwld = compiler_path / "mwldeppc.exe"
-    gnu_as = dkp_path / "bin" / f"powerpc-eabi-as{exe}"
+    gnu_as = args.powerpc / f"powerpc-eabi-as{exe}"
 
     mwcc_cmd = f"{chain}{wine}{mwcc} $cflags -MMD -c $in -o $basedir"
     mwld_cmd = f"{wine}{mwld} $ldflags -o $out @$out.rsp"
@@ -2230,3 +2254,6 @@ if __name__ == "__main__":
     ###
     with open("objdiff.json", "w") as w:
         json.dump(objdiff_config, w, indent=4)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,2 +0,0 @@
-# Extract devkitPPC archive
-zstandard>=0.22.0

tools/.gitignore

@@ -5,4 +5,3 @@ elf2dol
 !UpdateReadme.exe
 !UpdateReadme.runtimeconfig.json
 dtk
-devkitPPC

tools/download_mwcc.py

@@ -0,0 +1,45 @@
+import urllib.request
+import sys
+import os
+import stat
+import tempfile
+import shutil
+import zipfile
+
+if sys.platform == "cygwin":
+    sys.exit(
+        f"Cygwin/MSYS2 is not supported."
+        f"\nPlease use native Windows Python instead."
+        f"\nPlease run pacman -R python in msys2."
+        f"\n(Current path: {sys.executable})"
+    )
+
+HARDLINK = "https://files.decomp.dev/compilers_20230715.zip"
+
+
+def main() -> None:
+    output = f"{os.path.dirname(__file__)}/mwcc_compiler"
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_zip = f"{tmp_dir}/mwcc_compiler.zip"
+        tmp_gc = f"{tmp_dir}/GC"
+
+        request = urllib.request.Request(
+            url=HARDLINK,
+            headers={"User-Agent": "Mozilla/5.0"},
+        )
+
+        with urllib.request.urlopen(request) as src, open(tmp_zip, "wb") as dst:
+            shutil.copyfileobj(src, dst)
+
+        with zipfile.ZipFile(tmp_zip) as zip_file:
+            zip_file.extractall(tmp_dir)
+
+        shutil.move(tmp_gc, output)
+
+    st = os.stat(output)
+    os.chmod(output, st.st_mode | stat.S_IEXEC)
+
+
+if __name__ == "__main__":
+    main()

tools/download_ppc.py

@@ -3,10 +3,8 @@ import sys
 import os
 import stat
 import platform
-import shutil
 import tempfile
-import tarfile
-import zstandard
+import zipfile
 
 if sys.platform == "cygwin":
     sys.exit(
@@ -16,47 +14,30 @@ if sys.platform == "cygwin":
         f"\n(Current path: {sys.executable})"
     )
 
-# TODO: Less hardcoded elements
-REPO = "https://wii.leseratte10.de/devkitPro/devkitPPC/r44%20%282023-08-07%29/"
+REPO = "https://github.com/encounter/gc-wii-binutils"
 
 
 def main() -> None:
-    output = f"{os.path.dirname(__file__)}/devkitPPC"
+    output = f"{os.path.dirname(__file__)}/powerpc"
 
     uname = platform.uname()
     system = uname.system.lower()
     arch = uname.machine.lower()
     if system == "darwin":
-        system = "osx"
+        system = "macos"
+        arch = "universal"
     if arch == "amd64":
         arch = "x86_64"
-    if arch == "x86_32" and system == "windows":
-        system = "win32"
-        arch = "1686"
-    if arch in ["armv8", "arm64v8", "aarch64"]:
-        arch = "aarch64"
+    if arch == "x86_32":
+        arch = "i686"
 
-    with tempfile.TemporaryDirectory() as tmp:
-        tmp_zst = f"{tmp}/tmp.tar.zst"
-        tmp_tar = f"{tmp}/tmp.tar"
-        tmp_dir = f"{tmp}/tmp"
-        tmp_ppc = f"{tmp_dir}/opt/devkitpro/devkitPPC"
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_zip = f"{tmp_dir}/powerpc.zip"
+        url = f"{REPO}/releases/latest/download/{system}-{arch}.zip"
 
-        request = urllib.request.Request(
-            url=f"{REPO}/devkitPPC-r44.2-2-{system}_{arch}.pkg.tar.zst",
-            headers={"User-Agent": "Mozilla/5.0"},
-        )
-
-        with urllib.request.urlopen(request) as src, open(tmp_zst, "wb") as dst:
-            shutil.copyfileobj(src, dst)
-
-        with open(tmp_zst, "rb") as src, open(tmp_tar, "wb") as dst:
-            zstandard.ZstdDecompressor().copy_stream(src, dst)
-
-        with tarfile.open(tmp_tar) as src:
-            src.extractall(tmp_dir)
-
-        shutil.move(tmp_ppc, output)
+        urllib.request.urlretrieve(url, tmp_zip)
+        with zipfile.ZipFile(tmp_zip) as zip_file:
+            zip_file.extractall(output)
 
     st = os.stat(output)
     os.chmod(output, st.st_mode | stat.S_IEXEC)