Merge branch 'feature/ci-build' into 'master'

Implement CI builds & automated PR testing

See merge request banjo.decomp/banjo-kazooie!61

.dockerignore

@@ -0,0 +1,18 @@
+.dockerignore
+.gitlab-ci.yml
+Dockerfile
+README.md
+LICENSE
+
+asm/
+assets/
+include/
+ido/
+build/
+bin/
+src/
+
+baserom*
+*.z64
+
+Makefile

.gitlab-ci.yml

@@ -0,0 +1,72 @@
+# This CI will run:
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'    # ... on merge requests
+    - if: $CI_COMMIT_TAG                                  # ... on tags
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH         # ... on default (master/main) branch
+
+stages:
+  - build-image
+  - test
+
+
+variables:
+  # Fetch submodules
+  GIT_SUBMODULE_STRATEGY: recursive
+  # Only fetch the latest commit (shallow clone, faster)
+  GIT_SUBMODULE_DEPTH: 1
+  GIT_DEPTH: 1
+
+# Builds the image and pushes it to the registry
+# This image contains all the tooling necessary to run the compilation tools
+build-image:
+  stage: build-image
+  # Run image build only if packages changed
+  only:
+    changes:
+      - Dockerfile
+      - packages.txt
+      - requirements.txt
+      - .gitlab-ci.yml
+      - tools/*
+  # Set up the docker daemon for building the image
+  image: docker:latest
+  services:
+    - docker:dind
+  variables:
+    DOCKER_DRIVER: overlay2
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY     # Login to the registry
+    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME || true        # Pull the image if it exists
+                                                                        # Build the image and tag it with the branch name and latest
+    - |
+      docker build \
+      --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME \
+      -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME \
+      -t $CI_REGISTRY_IMAGE:latest \
+      .
+    - docker push --all-tags $CI_REGISTRY_IMAGE                         # Push the image to the registry
+
+# Try to compile the code inside the image to make sure it works (run docker container)
+build-test-us10:
+  stage: test
+  image:
+    name: $CI_REGISTRY_IMAGE:latest
+    entrypoint: [""]
+  before_script:
+    # Download the baserom from $BASEROM_<VER>_URL, decrypt with $BASEROM_<VER>_KEY and save as baserom.us.v10.z64, and check the sha1sum against $BASEROM_<VER>_SHA1
+    - curl -L "$BASEROM_US10_URL" -o baserom.us.v10.enc.z64
+    - openssl enc -d -aes-256-cbc -in baserom.us.v10.enc.z64 -out baserom.us.v10.z64 -k "$BASEROM_US10_KEY"
+    - FILE_SHA1=$(sha1sum baserom.us.v10.z64 | awk '{ print $1 }')
+    - echo "Calculated SHA1 - $FILE_SHA1"
+    - echo "Expected SHA1   - $BASEROM_US10_SHA1"
+    - if [ "${FILE_SHA1}" != "${BASEROM_US10_SHA1}" ]; then echo "Checksum verification failed"; exit 1; else echo "Checksum verification passed"; fi
+  script:
+    # Compile the code
+    - make
+    # Check if the resulting ROM is the same as the expected one
+    - FILE_SHA1=$(sha1sum build/us.v10/banjo.us.v10.z64 | awk '{ print $1 }')
+    - echo "Calculated SHA1 - $FILE_SHA1"
+    - echo "Expected SHA1   - $BASEROM_US10_SHA1"
+    - if [ "${FILE_SHA1}" != "${BASEROM_US10_SHA1}" ]; then echo "Checksum verification failed"; exit 1; else echo "Checksum verification passed"; fi

Dockerfile

@@ -1,14 +1,22 @@
-FROM ubuntu:20.04 as build
-
+FROM ubuntu:20.04 AS build
 ENV DEBIAN_FRONTEND=noninteractive
 
-COPY packages.txt /
-RUN apt-get update && apt-get install -y $(cat packages.txt)
+# (for debug purposes)
+RUN echo "System arch: $(uname -a)\nDPKG arch: $(dpkg --print-architecture)"
 
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+# Install package dependencies
+COPY packages.txt ./
+RUN apt-get update && apt-get install -y $(cat packages.txt) && rm packages.txt
 
-COPY requirements.txt /
+# Install Rust/Cargo
+# NOTE: We replace /proc/self/exe with /bin/sh in the script to avoid issues with Docker
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sed 's#/proc/self/exe#\/bin\/sh#g' | CARGO_HOME=/opt/cargo sh -s -- -y
+ENV PATH=/opt/cargo/bin:$PATH
+
+# Install Python dependencies
+COPY requirements.txt ./
+COPY tools/n64splat/requirements.txt ./tools/n64splat/requirements.txt
 RUN python3 -m pip install -r requirements.txt
 
-RUN mkdir /banjo
 WORKDIR /banjo
+ENTRYPOINT ["/bin/bash", "-c"]

README.md

@@ -2,23 +2,68 @@
 
 <img src="./progress/progress_total.svg">
 
+### Baserom checksums
+
+- `baserom.us.v10.z64`: `1fe1632098865f639e22c11b9a81ee8f29c75d7a`
+- `baserom.us.v11.z64`: `ded6ee166e740ad1bc810fd678a84b48e245ab80`
+- `baserom.jp.z64`:     `90726d7e7cd5bf6cdfd38f45c9acbf4d45bd9fd8`
+- `baserom.pal.z64`:    `bb359a75941df74bf7290212c89fbc6e2c5601fe`
+
 ## Building
 
-Grab tools
+The following instructions should work on the following platforms:
+- Ubuntu 18.04 or higher (x86_64)
+- Docker only
+    - Linux (x86_64, ARM)
+    - macOS (x86_64, ARM)
+
+### Local (Linux)
+
+Works with Ubuntu 18.04 or higher.
+
+#### 1. Install dependencies
+
+```sh
+sudo apt-get update && sudo apt-get install -y $(cat packages.txt)
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+python3 -m pip install -r requirements.txt
+```
+
+#### 2. Grab tools
 
 ```sh
 git submodule update --init --recursive
+cd tools/n64splat
+python3 -m pip install -r requirements.txt
 ```
 
-Drop in `US v1.0` as `baserom.us.v10.z64` (sha1sum: `1fe1632098865f639e22c11b9a81ee8f29c75d7a`)
+#### 3. Add baserom
 
-To extract and build everything
+Add the file for `US v1.0` as `baserom.us.v10.z64` in the project folder.
+
+(optional): Check the baserom checksum
+
+```sh
+sha1sum baserom.us.v10.z64
+```
+
+The output should match the checksum specified above.
+
+#### 4. Build
+
+To extract and build everything simply run:
 
 ```sh
 make
 ```
 
-where the following are supported values of `<module_id>`
+If you want to build a specific module, instead do:
+
+```sh
+make <module_id>
+```
+
+...where the following are supported values of `<module_id>`
 - `core1`
 - `core2`
 - `MM`
@@ -35,19 +80,87 @@ where the following are supported values of `<module_id>`
 - `fight`
 - `cutscenes`
 
-### Prerequisites
+### Local (Docker - Linux/macOS)
 
-Ubuntu 18.04 or higher.
+#### 1. Get the Docker image
+
+(if available) you can pull it from GitLab (but you need to be logged in):
 
 ```sh
-sudo apt-get update && sudo apt-get install -y $(cat packages.txt)
-
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-
-python3 -m pip install -r requirements.txt
+docker login registry.gitlab.com
+docker pull registry.gitlab.com/banjo.decomp/banjo-kazooie:latest
 ```
 
-### Other versions
+(otherwise) you can build it yourself:
+
+```sh
+docker build -t banjo-kazooie .
+```
+
+**NOTE for ARM users** (Windows ARM, Raspberry Pi and similar, or Apple Silicon): Use this command instead:
+
+```sh
+docker build --platform linux/amd64 -t banjo-kazooie .
+```
+
+#### 2. Add baserom
+
+Follow the same instructions as Step 3 above in "Local (Linux)".
+
+#### 3. Run the Docker container
+
+```sh
+docker run -it --rm -v $(pwd):/banjo banjo-kazooie 
+```
+
+**NOTE for ARM users**: Use this command instead:
+
+```sh
+docker run --platform linux/amd64 -it --rm -v $(pwd):/banjo banjo-kazooie 
+```
+
+#### 4. Build
+
+Follow the same instructions as Step 4 above in "Local (Linux)".
+
+To exit Docker, simply type `exit`.
+
+### Cloud (GitLab CI)
+
+These are the instructions for building on GitLab CI.
+This applies to the main repo - **if you have a fork**, you will need to follow these steps too!
+
+#### 1. Upload the baserom
+
+Upload the file for `US v1.0` as `baserom.us.v10.enc.z64` to a remote server where it can be downloaded from with `wget` or `curl`. The file has to be encrypted with `AES-256-CBC`, as follows:
+
+```sh
+openssl enc -aes-256-cbc -salt -in baserom.us.v10.z64 -out baserom.us.v10.enc.z64
+```
+
+Then, upload the encrypted file to a server and get a direct download link.
+
+Sharing services like Google Drive, Dropbox, or OneDrive might not work, as they require manual interaction to download the file.
+
+#### 2. Set up environment variables
+
+In your GitLab project, go to `Settings > CI/CD > Variables` and add the following variables (for each version):
+
+- `BASEROM_<VER>_URL`: a direct download URL for the baserom.us.v10.z64 file (see above); this file has to be encrypted with `AES-256-CBC`
+- `BASEROM_<VER>_KEY`: the AES key used to encrypt the baserom file above
+- `BASEROM_<VER>_SHA1`: the SHA1 checksum of the baserom file; simply use the one mentioned above
+
+Replace `<VER>` with the version you are using:
+- `USA10`
+- `USA11`
+- `JP`
+- `PAL`
+
+#### 3. Trigger the pipeline
+
+Push a commit to your repository and you should see a new pipeline starting in the `CI/CD > Pipelines` section! 
+
+## Other versions
 
 Drop in `us.v11`, `jp`, or `pal` as `baserom.<version>.z64` e.g. `baserom.us.v11.z64`
 

packages.txt

@@ -10,3 +10,4 @@ python3
 python3-pip
 unzip
 wget
+openssl