decomp: ensure out_folder exists before being written to

This allows the extractor's decompilation routine to do all the things the normal decompiler can (ie. extract audio)

.clang-format

@@ -10,7 +10,7 @@ IncludeCategories:
     Priority: 2
   - Regex: '^(<|")common.*'
     Priority: 3
-  - Regex: '^"third-party.*'
+  - Regex: '^"(third-party|fmt|curl).*'
     Priority: 4
     SortPriority: 5
   - Regex: '.*'

.github/scripts/releases/extract_build_unix.sh

@@ -32,4 +32,4 @@ cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
 cp -r $SOURCE/goal_src $DEST/data
 cp -r $SOURCE/game/assets $DEST/data/game/
 cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
-cp -r $SOURCE/custom_levels $DEST/data
+cp -r $SOURCE/custom_assets $DEST/data

.github/scripts/releases/extract_build_windows.sh

@@ -24,4 +24,4 @@ cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
 cp -r $SOURCE/goal_src $DEST/data
 cp -r $SOURCE/game/assets $DEST/data/game/
 cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
-cp -r $SOURCE/custom_levels $DEST/data
+cp -r $SOURCE/custom_assets $DEST/data

.github/workflows/draft-new-release.yaml

@@ -23,7 +23,7 @@ jobs:
       - name: Bump Version and Push Tag
         if: github.repository == 'open-goal/jak-project'
         id: tag_version
-        uses: mathieudutour/github-tag-action@v6.1
+        uses: mathieudutour/github-tag-action@v6.2
         with:
           github_token: ${{ secrets.BOT_PAT }}
           tag_prefix: v

.github/workflows/linux-build-clang.yaml

@@ -20,13 +20,6 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -39,11 +32,13 @@ jobs:
             libxinerama-dev libxcursor-dev libpulse-dev \
             libxi-dev zip ninja-build libgl1-mesa-dev libssl-dev
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.28.1
+          variant: sccache
+          key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - name: CMake Generation
         env:
@@ -51,8 +46,8 @@ jobs:
           CXX: clang++
         run: |
           cmake -B build --preset=${{ inputs.cmakePreset }} \
-            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 
       - name: Build Project
         run: cmake --build build --parallel $((`nproc`))

.github/workflows/linux-build-gcc.yaml

@@ -16,13 +16,6 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -40,11 +33,13 @@ jobs:
             sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
             sudo update-alternatives --set g++ /usr/bin/g++-10
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.28.1
+          variant: sccache
+          key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - name: CMake Generation
         env:
@@ -53,18 +48,11 @@ jobs:
         run: |
           cmake -B build --preset=${{ inputs.cmakePreset }} \
             -DCODE_COVERAGE=OFF \
-            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 
       - name: Build Project
         run: cmake --build build --parallel $((`nproc`)) -- -w dupbuild=warn
 
       - name: Run Tests
         run: ./test.sh
-
-      # - name: Submit Coverage Report to Codacy
-      #   uses: codacy/codacy-coverage-reporter-action@v1
-      #   continue-on-error: true
-      #   with:
-      #     project-token: ${{ secrets.CODACY_PROJECT_KEY }}
-      #     coverage-reports: ./build/goalc-test_coverage.info

.github/workflows/macos-build-arm.yaml

@@ -13,16 +13,9 @@ on:
 jobs:
   build:
     name: ARM
-    runs-on: macos-latest
+    runs-on: macos-12
     timeout-minutes: 120
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -33,11 +26,13 @@ jobs:
       - name: Install Package Dependencies
         run: arch -arm64 brew install cmake ninja
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.28.1
+          variant: sccache
+          key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - name: CMake Generation
         env:
@@ -45,8 +40,8 @@ jobs:
           CXX: clang++
         run: |
           cmake -B build --preset=${{ inputs.cmakePreset }} \
-            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 
       - name: Build Project
         run: cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))

.github/workflows/macos-build.yaml

@@ -17,16 +17,9 @@ on:
 jobs:
   build:
     name: Intel
-    runs-on: macos-11
+    runs-on: macos-12
     timeout-minutes: 120
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -34,11 +27,13 @@ jobs:
       - name: Install Package Dependencies
         run: brew install cmake nasm ninja
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: macos-11-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.28.1
+          variant: sccache
+          key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - name: CMake Generation
         env:
@@ -46,8 +41,8 @@ jobs:
           CXX: clang++
         run: |
           cmake -B build --preset=${{ inputs.cmakePreset }} \
-            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 
       - name: Build Project
         run: cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))

.github/workflows/windows-build-clang.yaml

@@ -21,13 +21,6 @@ jobs:
     # Set some sort of timeout in the event of run-away builds.  We are limited on concurrent jobs so, get rid of them.
     timeout-minutes: 60
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -42,10 +35,13 @@ jobs:
             choco install ${{ github.workspace }}/third-party/nasm/old/nasm.2.15.05.nupkg
           }
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          variant: sccache
+          key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - uses: ilammy/msvc-dev-cmd@v1
 
@@ -58,14 +54,13 @@ jobs:
 
       - name: CMake Generation
         shell: cmd
-        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe .
+        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache .
 
       - name: Build Project
         shell: cmd
         run: cmake --build build --parallel %NUMBER_OF_PROCESSORS%
 
       - name: Run Tests
-        timeout-minutes: 10
         env:
           GTEST_OUTPUT: "xml:opengoal-test-report.xml"
         run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"

.github/workflows/windows-build-msvc.yaml

@@ -16,13 +16,6 @@ jobs:
     runs-on: windows-2022
     timeout-minutes: 60
 
-    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
-      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
-      BUILDCACHE_COMPRESS_FORMAT: ZSTD
-      BUILDCACHE_COMPRESS_LEVEL: 19
-      BUILDCACHE_DIRECT_MODE: true
-      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -37,10 +30,13 @@ jobs:
             choco install ${{ github.workspace }}/third-party/nasm/old/nasm.2.15.05.nupkg
           }
 
-      - name: Setup Buildcache
-        uses: mikehardy/build[email protected].0
+      - name: Setup sccache
+        uses: hendrikmuhs/ccache-action@v1.2.13
         with:
-          cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          variant: sccache
+          key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}
+          restore-keys: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          max-size: 1000M
 
       - uses: ilammy/msvc-dev-cmd@v1
 
@@ -52,7 +48,7 @@ jobs:
 
       - name: CMake Generation - MSVC
         shell: cmd
-        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe .
+        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache .
 
       - name: Build Project
         shell: cmd

.gitignore

@@ -54,7 +54,9 @@ imgui.ini
 node_modules/
 
 # texture replacements
-texture_replacements/*
+custom_assets/jak1/texture_replacements/*
+custom_assets/jak2/texture_replacements/*
+custom_assets/jak3/texture_replacements/*
 
 # generated cmake files
 svnrev.h
@@ -76,3 +78,4 @@ __pycache__/
 /TODO.md
 unifont-15.0.03.ttf
 *.diff
+goalc-report.html

.vs/launch.vs.json

@@ -7,42 +7,58 @@
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Unit-Tests - Summary",
-      "args": ["--gtest_brief=1"]
+      "args": [
+        "--gtest_brief=1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Unit-Tests - Verbose",
-      "args": ["--gtest_brief=0"]
+      "args": [
+        "--gtest_brief=0"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Draft Tests - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*Draft*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*Draft*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - TypeConsistency - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*TypeConsistency*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*TypeConsistency*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - TypeConsistency - Jak 2 - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*Jak2TypeConsistency*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*Jak2TypeConsistency*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - WithGameTests - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*WithGameTests*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*WithGameTests*\""
+      ]
     },
     {
       "type": "default",
@@ -113,56 +129,110 @@
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 1 - Runtime",
-      "args": ["-v", "--game", "jak1", "--", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak1",
+        "--",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 1 - Runtime (boot)",
-      "args": ["-v", "--game", "jak1", "--", "-boot", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--portable",
+        "--game",
+        "jak1",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (no boot)",
-      "args": ["-v", "--game", "jak2", "--", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (boot)",
-      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (release)",
-      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-boot",
+        "-fakeiso"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
-      "projectTarget": "goalc.exe (bin\\goalc.exe)",
-      "name": "REPL",
-      "args": ["--user-auto"]
+      "projectTarget": "gk.exe (bin\\gk.exe)",
+      "name": "Game - Jak 3 - Runtime (boot)",
+      "args": [
+        "-v",
+        "--game",
+        "jak3",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc.exe (bin\\goalc.exe)",
       "name": "REPL - Jak 1",
-      "args": ["--user-auto", "--game", "jak1"]
+      "args": [
+        "--user-auto",
+        "--game",
+        "jak1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc.exe (bin\\goalc.exe)",
       "name": "REPL - Jak 2",
-      "args": ["--user-auto", "--game", "jak2"]
+      "args": [
+        "--user-auto",
+        "--game",
+        "jak2"
+      ]
     },
     {
       "type": "default",
@@ -243,19 +313,45 @@
         "ntsc_v1"
       ]
     },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
+      "name": "Decompiler - Jak 3 - Extract",
+      "args": [
+        "${workspaceRoot}/decompiler/config/jak3/jak3_config.jsonc",
+        "${workspaceRoot}/iso_data",
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "ntsc_v1",
+        "--config-override \"{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, \\\"allowed_objects\\\": []}\""
+      ]
+    },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
       "name": "Tools - EE Memory Analyze - Jak 1",
-      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak1"]
+      "args": [
+        "\"${workspaceRoot}/eeMemory.bin\"",
+        "--output-path",
+        "\"${workspaceRoot}\"",
+        "--game",
+        "jak1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
       "name": "Tools - EE Memory Analyze - Jak 2",
-      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak2"]
+      "args": [
+        "\"${workspaceRoot}/eeMemory.bin\"",
+        "--output-path",
+        "\"${workspaceRoot}\"",
+        "--game",
+        "jak2"
+      ]
     },
     {
       "type": "default",
@@ -272,35 +368,53 @@
       "project": "CMakeLists.txt",
       "projectTarget": "extractor.exe (bin\\extractor.exe)",
       "name": "Tools - Extractor - Full",
-      "args": ["\"E:\\ISOs\\Jak\\Jak 1.iso\""]
+      "args": [
+        "\"E:\\ISOs\\Jak\\Jak 1.iso\""
+      ]
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "lsp.exe (bin\\lsp.exe)",
-      "name" : "Tools - LSP",
-      "args" : []
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "lsp.exe (bin\\lsp.exe)",
+      "name": "Tools - LSP",
+      "args": []
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
-      "name" : "Tools - Type Searcher",
-      "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--size", 255, "--fields", "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"]
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "type_searcher.exe (bin\\type_searcher.exe)",
+      "name": "Tools - Type Searcher",
+      "args": [
+        "--game",
+        "jak2",
+        "--output-path",
+        "./search-results.json",
+        "--size",
+        255,
+        "--fields",
+        "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"
+      ]
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "formatter.exe (bin\\formatter.exe)",
-      "name" : "Tools - Formatter - Inplace",
-      "args" : ["--write", "--file", "C:\\Users\\xtvas\\Repos\\opengoal\\jak-project\\decompiler_out\\jak3\\mood-h_disasm.gc"]
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "formatter.exe (bin\\formatter.exe)",
+      "name": "Tools - Formatter - Inplace",
+      "args": [
+        "--write",
+        "--file",
+        "C:\\Users\\xtvas\\Repositories\\opengoal\\jak-project\\goal_src\\jak1\\engine\\camera\\cam-states.gc"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Formatter",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*FormatterTests*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*FormatterTests*\""
+      ]
     }
   ]
-}
+}

.vscode/launch.json

@@ -8,10 +8,10 @@
       "name": "run python script",
       "type": "python",
       "request": "launch",
-      "program": "${workspaceFolder}/scripts/gsrc/copy-common-naming.py",
+      "program": "${workspaceFolder}/scripts/gsrc/compare-compilation-outputs.py",
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}",
-      "args": ["--update-names-from-refs", "--decompiler", "./out/build/Release/bin/decompiler"]
+      "args": []
     },
   ]
 }

.vscode/settings.json

@@ -7,5 +7,6 @@
     },
     "editor.wordBasedSuggestions": "matchingDocuments",
     "editor.snippetSuggestions": "top"
-  }
+  },
+  "cmake.configureOnOpen": false
 }

CMakeLists.txt

@@ -101,7 +101,6 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
     message(STATUS "AppleClang detected - Setting Defaults")
     set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} \
-        -march=native \
         -Wall \
         -Winit-self \
         -ggdb \
@@ -117,6 +116,12 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
         -Wsign-promo \
         -fdiagnostics-color=always"
         )
+    
+    # pin to AVX for macOS, hopefully all macOS runners have atleast this architecture
+    # technically speaking, SSE4 is the cutoff for Apple Silicon so...only a matter of time!
+    if(NOT CMAKE_CXX_COMPILER_TARGET STREQUAL "arm64-apple-darwin")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
+    endif()
 
     # additional c++ flags for release mode for our projects
     if(CMAKE_BUILD_TYPE MATCHES "Release")
@@ -221,6 +226,10 @@ build_third_party_lib(tree-sitter tree-sitter)
 # native OS dialogs for error messages
 build_third_party_lib(libtinyfiledialogs libtinyfiledialogs)
 
+# build format library
+include_directories(third-party/fmt/include)
+build_third_party_lib(fmt fmt)
+
 # build common library
 add_subdirectory(common)
 
@@ -273,9 +282,6 @@ include(test/CMakeLists.txt)
 # build lzokay library
 build_third_party_lib(lzokay lzokay)
 
-# build format library
-build_third_party_lib(fmt fmt)
-
 build_third_party_lib(stb_image stb_image)
 build_third_party_lib(tiny_gltf tiny_gltf)
 build_third_party_lib(xdelta3 xdelta3)

common/CMakeLists.txt

@@ -51,7 +51,7 @@ add_library(common
         repl/config.cpp
         repl/nrepl/ReplClient.cpp
         repl/nrepl/ReplServer.cpp
-        repl/util.cpp
+        repl/repl_wrapper.cpp
         serialization/subtitles/subtitles_v1.cpp
         serialization/subtitles/subtitles_v2.cpp
         serialization/subtitles/subtitles.cpp
@@ -66,6 +66,7 @@ add_library(common
         type_system/TypeSpec.cpp
         type_system/TypeSystem.cpp
         util/Assert.cpp
+        util/ast_util.cpp
         util/BitUtils.cpp
         util/compress.cpp
         util/crc32.cpp
@@ -86,10 +87,11 @@ add_library(common
         util/term_util.cpp
         util/Timer.cpp
         util/unicode_util.cpp
+        util/gltf_util.cpp
         versions/versions.cpp
         )
 
-target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs)
+target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs tiny_gltf)
 
 if(WIN32)
     target_link_libraries(common wsock32 ws2_32 windowsapp)

common/audio/audio_formats.cpp

@@ -3,7 +3,7 @@
 #include "common/log/log.h"
 #include "common/util/BinaryWriter.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 /*!
  * Write a wave file from a vector of samples.

common/cross_os_debug/xdbg.cpp

@@ -11,7 +11,7 @@
 #include "common/goal_constants.h"
 #include "common/util/Timer.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 #ifdef __linux
 #include <fcntl.h>

common/cross_sockets/XSocket.cpp

@@ -19,7 +19,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 #include "common/log/log.h"
 // clang-format on
 

common/cross_sockets/XSocketClient.cpp

@@ -5,6 +5,8 @@
 #include "common/cross_sockets/XSocket.h"
 #include "common/log/log.h"
 
+#include "fmt/core.h"
+
 // clang-format off
 #ifdef _WIN32
 #define NOMINMAX
@@ -13,9 +15,7 @@
 #include <WinSock2.h>
 #include <WS2tcpip.h>
 #endif
-#include "common/repl/nrepl/ReplServer.h"
 
-#include "third-party/fmt/core.h"
 // clang-format on
 
 XSocketClient::XSocketClient(int _tcp_port) {

common/cross_sockets/XSocketServer.cpp

@@ -4,7 +4,7 @@
 #include "common/cross_sockets/XSocket.h"
 #include "common/common_types.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 #ifdef _WIN32
 #define NOMINMAX
@@ -35,7 +35,7 @@ void XSocketServer::shutdown_server() {
   close_server_socket();
 }
 
-bool XSocketServer::init_server() {
+bool XSocketServer::init_server(bool failure_may_occur) {
   listening_socket = open_socket(AF_INET, SOCK_STREAM, 0);
   if (listening_socket < 0) {
     listening_socket = -1;
@@ -76,19 +76,27 @@ bool XSocketServer::init_server() {
   addr.sin_port = htons(tcp_port);
 
   if (bind(listening_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
-    lg::error("[XSocketServer:{}] failed to bind", tcp_port);
+    if (failure_may_occur) {
+      lg::debug("[XSocketServer:{}] failed to bind", tcp_port);
+    } else {
+      lg::error("[XSocketServer:{}] failed to bind", tcp_port);
+    }
     close_server_socket();
     return false;
   }
 
   if (listen(listening_socket, 0) < 0) {
-    lg::error("[XSocketServer:{}] failed to listen", tcp_port);
+    if (failure_may_occur) {
+      lg::debug("[XSocketServer:{}] failed to listen", tcp_port);
+    } else {
+      lg::error("[XSocketServer:{}] failed to listen", tcp_port);
+    }
     close_server_socket();
     return false;
   }
 
   server_initialized = true;
-  lg::info("[XSocketServer:{}] initialized", tcp_port);
+  lg::debug("[XSocketServer:{}] initialized", tcp_port);
   post_init();
   return true;
 }

common/cross_sockets/XSocketServer.h

@@ -20,7 +20,7 @@ class XSocketServer {
   XSocketServer(const XSocketServer&) = delete;
   XSocketServer& operator=(const XSocketServer&) = delete;
 
-  bool init_server();
+  bool init_server(bool failure_may_occur = false);
   void shutdown_server();
   void close_server_socket();
 

common/custom_data/TFrag3Data.cpp

@@ -9,10 +9,16 @@
 #include "third-party/sse2neon/sse2neon.h"
 #endif
 
+#include "common/log/log.h"
 #include "common/util/Assert.h"
 
 namespace tfrag3 {
 
+void PackedTimeOfDay::serialize(Serializer& ser) {
+  ser.from_pod_vector(&data);
+  ser.from_ptr(&color_count);
+}
+
 void PackedTieVertices::serialize(Serializer& ser) {
   ser.from_pod_vector(&color_indices);
   ser.from_pod_vector(&matrices);
@@ -71,10 +77,9 @@ void TfragTree::serialize(Serializer& ser) {
     draw.serialize(ser);
   }
 
-  // ser.from_pod_vector(&vertices);
   ser.from_pod_vector(&packed_vertices.vertices);
   ser.from_pod_vector(&packed_vertices.cluster_origins);
-  ser.from_pod_vector(&colors);
+  colors.serialize(ser);
   bvh.serialize(ser);
   ser.from_ptr(&use_strips);
 }
@@ -398,7 +403,7 @@ void TieTree::serialize(Serializer& ser) {
   }
 
   packed_vertices.serialize(ser);
-  ser.from_pod_vector(&colors);
+  colors.serialize(ser);
   bvh.serialize(ser);
 
   ser.from_ptr(&has_per_proto_visibility_toggle);
@@ -406,7 +411,7 @@ void TieTree::serialize(Serializer& ser) {
 }
 
 void ShrubTree::serialize(Serializer& ser) {
-  ser.from_pod_vector(&time_of_day_colors);
+  time_of_day_colors.serialize(ser);
   ser.from_pod_vector(&indices);
   packed_vertices.serialize(ser);
   if (ser.is_saving()) {
@@ -422,6 +427,30 @@ void ShrubTree::serialize(Serializer& ser) {
   ser.from_string_vector(&proto_names);
 }
 
+void HfragmentBucket::serialize(Serializer& ser) {
+  ser.from_pod_vector(&corners);
+  ser.from_ptr(&montage_table);
+}
+
+void Hfragment::serialize(Serializer& ser) {
+  ser.from_pod_vector(&vertices);
+  ser.from_pod_vector(&indices);
+  ser.from_pod_vector(&corners);
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(buckets.size());
+  } else {
+    buckets.resize(ser.load<size_t>());
+  }
+  for (auto& x : buckets) {
+    x.serialize(ser);
+  }
+  time_of_day_colors.serialize(ser);
+  ser.from_ptr(&wang_tree_tex_id);
+  ser.from_ptr(&draw_mode);
+  ser.from_ptr(&occlusion_offset);
+}
+
 void BVH::serialize(Serializer& ser) {
   ser.from_ptr(&first_leaf_node);
   ser.from_ptr(&last_leaf_node);
@@ -463,6 +492,7 @@ void MercDraw::serialize(Serializer& ser) {
   ser.from_ptr(&first_index);
   ser.from_ptr(&index_count);
   ser.from_ptr(&num_triangles);
+  ser.from_ptr(&no_strip);
 }
 
 void Blerc::serialize(Serializer& ser) {
@@ -604,6 +634,8 @@ void Level::serialize(Serializer& ser) {
     tree.serialize(ser);
   }
 
+  hfrag.serialize(ser);
+
   collision.serialize(ser);
   merc_data.serialize(ser);
 
@@ -654,8 +686,7 @@ void PackedShrubVertices::memory_usage(MemoryUsageTracker* tracker) const {
 }
 
 void ShrubTree::memory_usage(MemoryUsageTracker* tracker) const {
-  tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY,
-               sizeof(TimeOfDayColor) * time_of_day_colors.size());
+  tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY, sizeof(u8) * time_of_day_colors.data.size());
   packed_vertices.memory_usage(tracker);
   tracker->add(MemoryUsageCategory::SHRUB_DRAW, sizeof(ShrubDraw) * static_draws.size());
   tracker->add(MemoryUsageCategory::SHRUB_IND, sizeof(u32) * indices.size());
@@ -683,7 +714,7 @@ void TieTree::memory_usage(MemoryUsageTracker* tracker) const {
                  draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
   }
   packed_vertices.memory_usage(tracker);
-  tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+  tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(u8) * colors.data.size());
 
   for (auto& draw : instanced_wind_draws) {
     draw.memory_usage(tracker);
@@ -707,7 +738,7 @@ void TfragTree::memory_usage(MemoryUsageTracker* tracker) const {
                  draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
   }
   packed_vertices.memory_usage(tracker);
-  tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+  tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(u8) * colors.data.size());
   tracker->add(MemoryUsageCategory::TFRAG_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
 }
 
@@ -720,6 +751,13 @@ void IndexTexture::memory_usage(MemoryUsageTracker* tracker) const {
   tracker->add(MemoryUsageCategory::SPECIAL_TEXTURE, 256 * 4);  // clut
 }
 
+void Hfragment::memory_usage(tfrag3::MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::HFRAG_VERTS, vertices.size() * sizeof(HfragmentVertex));
+  tracker->add(MemoryUsageCategory::HFRAG_INDEX, indices.size() * sizeof(u32));
+  tracker->add(MemoryUsageCategory::HFRAG_TIME_OF_DAY, time_of_day_colors.data.size() * sizeof(u8));
+  tracker->add(MemoryUsageCategory::HFRAG_CORNERS, corners.size() * sizeof(HfragmentCorner));
+}
+
 void Level::memory_usage(MemoryUsageTracker* tracker) const {
   for (const auto& texture : textures) {
     texture.memory_usage(tracker);
@@ -740,6 +778,7 @@ void Level::memory_usage(MemoryUsageTracker* tracker) const {
   for (const auto& tree : shrub_trees) {
     tree.memory_usage(tracker);
   }
+  hfrag.memory_usage(tracker);
   collision.memory_usage(tracker);
   merc_data.memory_usage(tracker);
 }
@@ -783,6 +822,11 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
       {"merc-mod-draw-1", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_1]},
       {"merc-mod-draw-2", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_2]},
       {"blerc", mem_use.data[tfrag3::MemoryUsageCategory::BLERC]},
+      {"hfrag-verts", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_VERTS]},
+      {"hfrag-index", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_INDEX]},
+      {"hfrag-time-of-day", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_TIME_OF_DAY]},
+      {"hfrag-corners", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_CORNERS]}
+
   };
   for (auto& known : known_categories) {
     total_accounted += known.second;
@@ -795,8 +839,8 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
 
   for (const auto& x : known_categories) {
     if (x.second) {
-      fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
-                 100.f * (float)x.second / uncompressed_data_size);
+      lg::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
+                100.f * (float)x.second / uncompressed_data_size);
     }
   }
 }

common/custom_data/Tfrag3Data.h

@@ -18,7 +18,7 @@ namespace tfrag3 {
 // - if changing any large things (vertices, vis, bvh, colors, textures) update get_memory_usage
 // - if adding a new category to the memory usage, update extract_level to print it.
 
-constexpr int TFRAG3_VERSION = 39;
+constexpr int TFRAG3_VERSION = 41;
 
 enum MemoryUsageCategory {
   TEXTURE,
@@ -61,6 +61,11 @@ enum MemoryUsageCategory {
   MERC_MOD_TABLE,
   BLERC,
 
+  HFRAG_VERTS,
+  HFRAG_INDEX,
+  HFRAG_TIME_OF_DAY,
+  HFRAG_CORNERS,
+
   COLLISION,
 
   NUM_CATEGORIES
@@ -269,18 +274,23 @@ struct BVH {
   void serialize(Serializer& ser);
 };
 
-// A time-of-day color. Each stores 8 colors. At a given "time of day", they are interpolated
-// to find a single color which goes into a color palette.
-struct TimeOfDayColor {
-  math::Vector<u8, 4> rgba[8];
+// This is split into groups of 4 colors.
+// The data in these groups is stored first by palette, then color, then channel.
+struct PackedTimeOfDay {
+  std::vector<u8> data;
+  u32 color_count = 0;
+  void serialize(Serializer& ser);
 
-  bool operator==(const TimeOfDayColor& other) const {
-    for (size_t i = 0; i < 8; i++) {
-      if (rgba[i] != other.rgba[i]) {
-        return false;
-      }
-    }
-    return true;
+  u8 read(int color, int palette, int channel) const {
+    const int color_quad = color / 4;
+    const int color_in_quad = color % 4;
+    return data[color_quad * 4 * 4 * 8 + palette * 4 * 4 + color_in_quad * 4 + channel];
+  }
+
+  u8& read(int color, int palette, int channel) {
+    const int color_quad = color / 4;
+    const int color_in_quad = color % 4;
+    return data[color_quad * 4 * 4 * 8 + palette * 4 * 4 + color_in_quad * 4 + channel];
   }
 };
 
@@ -319,8 +329,8 @@ struct TfragTree {
   TFragmentTreeKind kind;        // our tfrag kind
   std::vector<StripDraw> draws;  // the actual topology and settings
   PackedTfragVertices packed_vertices;
-  std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
-  BVH bvh;                             // the bvh for frustum culling
+  PackedTimeOfDay colors;  // vertex colors (pre-interpolation)
+  BVH bvh;                 // the bvh for frustum culling
   bool use_strips = true;
 
   struct {
@@ -401,7 +411,7 @@ struct TieTree {
   std::array<u32, kNumTieCategories + 1> category_draw_indices;
 
   PackedTieVertices packed_vertices;
-  std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
+  PackedTimeOfDay colors;  // vertex colors (pre-interpolation)
 
   std::vector<InstancedStripDraw> instanced_wind_draws;
   std::vector<TieWindInstance> wind_instance_info;
@@ -422,7 +432,7 @@ struct TieTree {
 
 struct ShrubTree {
   // todo some visibility structure
-  std::vector<TimeOfDayColor> time_of_day_colors;  // multiplier colors
+  PackedTimeOfDay time_of_day_colors;  // multiplier colors
 
   PackedShrubVertices packed_vertices;
   std::vector<ShrubDraw> static_draws;  // the actual topology and settings
@@ -441,6 +451,43 @@ struct ShrubTree {
   void unpack();
 };
 
+struct HfragmentVertex {
+  float height = 0;
+  u32 vi = 0;
+  u16 color_index = 0;
+  u8 u = 0, v = 0;
+  u32 pad = 0;
+};
+
+struct HfragmentCorner {
+  math::Vector<float, 4> bsphere;
+  u32 vis_id = 0;
+  u32 index_start = 0;
+  u32 index_length = 0;
+  u32 num_tris = 0;
+};
+
+struct HfragmentBucket {
+  std::vector<u32> corners;
+  std::array<u16, 16> montage_table;
+  void serialize(Serializer& ser);
+};
+
+struct Hfragment {
+  std::vector<HfragmentVertex> vertices;
+  std::vector<u32> indices;
+  std::vector<HfragmentCorner> corners;
+  std::vector<HfragmentBucket> buckets;
+  PackedTimeOfDay time_of_day_colors;
+
+  std::array<s32, 4> wang_tree_tex_id;
+  DrawMode draw_mode;
+  u32 occlusion_offset;
+
+  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
+};
+
 struct CollisionMesh {
   struct Vertex {
     float x, y, z;
@@ -483,6 +530,8 @@ struct MercDraw {
   u32 first_index;
   u32 index_count;
   u32 num_triangles;
+  // no strip hack for custom models
+  bool no_strip = false;
   void serialize(Serializer& ser);
 };
 
@@ -566,6 +615,7 @@ struct Level {
   std::array<std::vector<TfragTree>, TFRAG_GEOS> tfrag_trees;
   std::array<std::vector<TieTree>, TIE_GEOS> tie_trees;
   std::vector<ShrubTree> shrub_trees;
+  Hfragment hfrag;
   CollisionMesh collision;
   MercModelGroup merc_data;
   u16 version2 = TFRAG3_VERSION;

common/dma/dma.cpp

@@ -2,7 +2,7 @@
 
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 std::string DmaTag::print() {
   std::string result;

common/dma/dma.h

@@ -11,7 +11,7 @@
 #include "common/common_types.h"
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 struct DmaStats {
   double sync_time_ms = 0;

common/dma/dma_copy.cpp

@@ -6,7 +6,7 @@
 #include "common/log/log.h"
 #include "common/util/Timer.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 /*!
  * Convert a DMA chain to an array of bytes that can be directly fed to VIF.

common/dma/gs.cpp

@@ -2,8 +2,8 @@
 
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/format.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
 
 std::string reg_descriptor_name(GifTag::RegisterDescriptor reg) {
   switch (reg) {

common/formatter/README.md

@@ -12,7 +12,6 @@ Also in general I found it _really_ hard to find modern and easy to understand a
 Atleast for me, it helps to understand the path code takes as it flows through the formatting process.  Originally I tried to do as much at once for sake of efficiency but this just makes things incredibly hard to reason about and fix bugs.  So like any problem that is complex, break it down, the formatter goes through many distinct phases chipping away at the problem:
 
 ```mermaid
-%%{init: {'theme': 'dark', "flowchart" : { "curve" : "basis" } } }%%
 flowchart TB
   subgraph top1 [Build a Formatting Tree]
     direction TB

common/formatter/formatter.cpp

@@ -2,13 +2,16 @@
 
 #include "formatter_tree.h"
 
+#include "common/formatter/rules/formatting_rules.h"
+#include "common/formatter/rules/rule_config.h"
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
+#include "common/util/ast_util.h"
 #include "common/util/string_util.h"
 
 #include "tree_sitter/api.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 // Declare the `tree_sitter_opengoal` function, which is
 // implemented by the `tree-sitter-opengoal` library.
@@ -30,14 +33,16 @@ int hang_indentation_width(const FormatterTreeNode& curr_node) {
   return 1 + hang_indentation_width(first_elt);
 }
 
-// TODO - this doesn't account for paren's width contribution!
 int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
   if (curr_node.token) {
     return curr_node.token->length();
   }
   int width = 1;
-  for (const auto& ref : curr_node.refs) {
-    width += get_total_form_inlined_width(ref);
+  for (int i = 0; i < curr_node.refs.size(); i++) {
+    width += get_total_form_inlined_width(curr_node.refs.at(i));
+    if (i != curr_node.refs.size() - 1) {
+      width += 1;  // add the space between elements
+    }
   }
   return width + 1;
 }
@@ -46,7 +51,9 @@ int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
 void apply_formatting_config(
     FormatterTreeNode& curr_node,
     std::optional<std::shared_ptr<formatter_rules::config::FormFormattingConfig>>
-        config_from_parent = {}) {
+        config_from_parent = {},
+    std::optional<std::shared_ptr<formatter_rules::config::FormFormattingConfig>>
+        config_merge_from_parent = {}) {
   using namespace formatter_rules;
   // node is empty, base-case
   if (curr_node.token || curr_node.refs.empty()) {
@@ -62,6 +69,13 @@ void apply_formatting_config(
       predefined_config = config::opengoal_form_config.at(form_head.value());
       curr_node.formatting_config = predefined_config.value();
     }
+
+    if (config_merge_from_parent) {
+      const auto& merge_config = *config_merge_from_parent.value();
+      curr_node.formatting_config.parent_mutable_extra_indent +=
+          merge_config.parent_mutable_extra_indent;
+      curr_node.formatting_config.prevent_inlining = merge_config.prevent_inlining;
+    }
   } else if (config_from_parent) {
     // TODO - doesn't merge just replaces, a bit inflexible
     predefined_config = *config_from_parent.value();
@@ -101,7 +115,7 @@ void apply_formatting_config(
       if (curr_node.formatting_config.has_constant_pairs) {
         for (int i = 0; i < (int)curr_node.refs.size(); i++) {
           auto& child_ref = curr_node.refs.at(i);
-          const auto type = child_ref.metadata.node_type;
+          const auto& type = child_ref.metadata.node_type;
           if (constant_types.find(type) == constant_types.end() &&
               constant_pairs::is_element_second_in_constant_pair(curr_node, child_ref, i)) {
             child_ref.formatting_config.parent_mutable_extra_indent = 2;
@@ -126,13 +140,16 @@ void apply_formatting_config(
     auto& ref = curr_node.refs.at(i);
     if (!ref.token) {
       // If the child has a pre-defined configuration at that index, we pass it along
-      if (predefined_config &&
-          predefined_config->index_configs.find(i) != predefined_config->index_configs.end()) {
-        apply_formatting_config(ref, predefined_config->index_configs.at(i));
+      if (predefined_config && predefined_config->index_config_override.find(i) !=
+                                   predefined_config->index_config_override.end()) {
+        apply_formatting_config(ref, {}, predefined_config->index_config_override.at(i));
+      } else if (predefined_config && predefined_config->index_configs.find(i) !=
+                                          predefined_config->index_configs.end()) {
+        apply_formatting_config(ref, predefined_config->index_configs.at(i), {});
       } else if (predefined_config && predefined_config->default_index_config) {
-        apply_formatting_config(ref, predefined_config->default_index_config);
+        apply_formatting_config(ref, predefined_config->default_index_config, {});
       } else {
-        apply_formatting_config(ref);
+        apply_formatting_config(ref, {}, {});
       }
     }
   }
@@ -144,16 +161,33 @@ void apply_formatting_config(
     // Find the maximum number of columns
     int max_columns = 0;
     for (const auto& field : curr_node.refs) {
-      if (field.refs.size() > max_columns) {
+      if ((int)field.refs.size() > max_columns) {
         max_columns = field.refs.size();
       }
     }
+    // if only one field has a value in the max col position, it looks weird for it to be indented
+    bool ignore_final_column_width = true;
+    int fields_with_atleast_max_col = 0;
+    for (const auto& field : curr_node.refs) {
+      if ((int)field.refs.size() == max_columns) {
+        fields_with_atleast_max_col++;
+        if (fields_with_atleast_max_col > 1) {
+          ignore_final_column_width = false;
+          break;
+        }
+      }
+    }
     // Now find the column max widths
     std::vector<int> column_max_widths = {};
     for (int col = 0; col < max_columns; col++) {
       column_max_widths.push_back(0);
+      // -2 because its the indentation before the final column that we want to skip
+      if (col >= curr_node.formatting_config.num_columns_to_compute_widths ||
+          (ignore_final_column_width && col == max_columns - 2)) {
+        continue;
+      }
       for (const auto& field : curr_node.refs) {
-        if (field.refs.size() > col) {
+        if ((int)field.refs.size() > col) {
           const auto width = get_total_form_inlined_width(field.refs.at(col));
           if (width > column_max_widths.at(col)) {
             column_max_widths[col] = width;
@@ -199,6 +233,10 @@ bool form_contains_node_that_prevents_inlining(const FormatterTreeNode& curr_nod
 bool can_node_be_inlined(const FormatterTreeNode& curr_node, int cursor_pos) {
   using namespace formatter_rules;
   if (curr_node.formatting_config.force_inline) {
+    // Ensure there are no comments, this still trumps this
+    if (form_contains_comment(curr_node)) {
+      return false;
+    }
     return true;
   }
   // First off, we cannot inline the top level
@@ -261,14 +299,16 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     // Add new line entry
     if (ref.token) {
       // Cleanup block-comments
-      std::string val = ref.token_str();
       if (ref.metadata.node_type == "block_comment") {
-        // TODO - change this sanitization to return a list of lines instead of a single new-lined
-        // line
-        val = comments::format_block_comment(ref.token_str());
+        const auto comment_lines = comments::format_block_comment(ref.token_str());
+        for (const auto& line : comment_lines) {
+          form_lines.push_back(line);
+        }
+      } else {
+        form_lines.push_back(ref.token_str());
       }
-      form_lines.push_back(val);
-      if (!curr_node.metadata.is_top_level && i == curr_node.refs.size() - 1 &&
+
+      if (!curr_node.metadata.is_top_level && i == (int)curr_node.refs.size() - 1 &&
           (ref.metadata.is_comment)) {
         // if there's an inline comment at the end of a form, we have to force the paren to the next
         // line and do a new-line paren this is ugly, but we have no choice!
@@ -289,26 +329,41 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
         (curr_node.formatting_config.hang_forms ||
          curr_node.formatting_config.combine_first_two_lines)) {
-      form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+      form_lines.at(0) += fmt::format(" {}", str_util::ltrim(form_lines.at(1)));
       form_lines.erase(form_lines.begin() + 1);
     } else if ((i + 1) < (int)curr_node.refs.size()) {
       const auto& next_ref = curr_node.refs.at(i + 1);
       // combine the next inline comment or constant pair
       if ((next_ref.metadata.node_type == "comment" && next_ref.metadata.is_inline) ||
           (curr_node.formatting_config.has_constant_pairs &&
-           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1))) {
+           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1)) ||
+          constant_pairs::is_element_second_in_constant_pair_new(curr_node.refs.at(i), next_ref)) {
         // TODO
         // has issues with not consolidating first lines, this should probably just be moved to
         // outside this loop for simplicity, do it later
         if (next_ref.token) {
-          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token.value());
+          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token_str());
           i++;
+          // We have to handle hang-consolidation here or else it will never be reached above!
+          if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
+              (curr_node.formatting_config.hang_forms ||
+               curr_node.formatting_config.combine_first_two_lines)) {
+            form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+            form_lines.erase(form_lines.begin() + 1);
+          }
         } else if (can_node_be_inlined(next_ref, cursor_pos)) {
           const auto& lines = apply_formatting(next_ref, {}, cursor_pos);  // TODO - cursor pos
           for (const auto& line : lines) {
             form_lines.at(form_lines.size() - 1) += fmt::format(" {}", line);
           }
           i++;
+          // We have to handle hang-consolidation here or else it will never be reached above!
+          if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
+              (curr_node.formatting_config.hang_forms ||
+               curr_node.formatting_config.combine_first_two_lines)) {
+            form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+            form_lines.erase(form_lines.begin() + 1);
+          }
         }
         if (!curr_node.metadata.is_top_level && next_ref.metadata.node_type == "comment" &&
             (i + 1) == (int)curr_node.refs.size()) {
@@ -325,7 +380,9 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
   // Consolidate any lines if the configuration requires it
   // TODO there is a hack here so that multi-line forms that are consolidated still line up properly
   // i have to make consolidate a more first-class feature of the config
-  if (curr_node.formatting_config.inline_until_index(form_lines)) {
+  // TODO - hacky, but prevents a bad situation, clean up
+  if (curr_node.formatting_config.inline_until_index(form_lines) &&
+      !str_util::contains(form_lines.at(0), ";")) {
     std::vector<std::string> new_form_lines = {};
     const auto original_form_head_width = str_util::split(form_lines.at(0), '\n').at(0).length();
     bool consolidating_lines = true;
@@ -351,9 +408,9 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
 
   // Add any column padding
   if (!curr_node.formatting_config.list_element_column_widths.empty()) {
-    for (int i = 0; i < form_lines.size(); i++) {
+    for (int i = 0; i < (int)form_lines.size(); i++) {
       const auto& token = form_lines.at(i);
-      if (i < form_lines.size() - 1) {
+      if (i < (int)form_lines.size() - 1) {
         form_lines[i] = str_util::pad_right(
             token, curr_node.formatting_config.list_element_column_widths.at(i), ' ');
       }
@@ -371,22 +428,32 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     form_lines[form_lines.size() - 1] =
         fmt::format("{}{}", form_lines[form_lines.size() - 1], form_surround_end);
   }
-  std::string curr_form = "";
-  if (curr_node.formatting_config.parent_mutable_extra_indent > 0) {
-    curr_form += str_util::repeat(curr_node.formatting_config.parent_mutable_extra_indent, " ");
-  }
   if (inline_form) {
-    form_lines = {fmt::format("{}", fmt::join(form_lines, " "))};
+    // NOTE - not sure about this, if we are inlining a form, it always makes sense to eliminate
+    // trailing whitespace the only issue i can foresee is related to strings that span multiple
+    // lines.
+    std::vector<std::string> new_form_lines = {};
+    for (const auto& form_line : form_lines) {
+      new_form_lines.push_back(str_util::ltrim(form_line));
+    }
+    form_lines = {fmt::format("{}", fmt::join(new_form_lines, " "))};
   } else {
+    bool currently_in_block_comment = false;
     for (int i = 0; i < (int)form_lines.size(); i++) {
-      if (i > 0) {
-        auto& line = form_lines.at(i);
+      auto& line = form_lines.at(i);
+      if (str_util::contains(line, "|#")) {
+        currently_in_block_comment = false;
+      }
+      if (i > 0 && !currently_in_block_comment) {
         line = fmt::format("{}{}",
                            str_util::repeat(curr_node.formatting_config.indentation_width_for_index(
                                                 curr_node.formatting_config, i),
                                             " "),
                            line);
       }
+      if (str_util::contains(line, "#|") && !str_util::contains(line, "|#")) {
+        currently_in_block_comment = true;
+      }
     }
   }
   return form_lines;
@@ -400,8 +467,6 @@ std::string join_formatted_lines(const std::vector<std::string>& lines,
 std::optional<std::string> formatter::format_code(const std::string& source) {
   // Create a parser.
   std::shared_ptr<TSParser> parser(ts_parser_new(), TreeSitterParserDeleter());
-
-  // Set the parser's language (JSON in this case).
   ts_parser_set_language(parser.get(), tree_sitter_opengoal());
 
   // Build a syntax tree based on source code stored in a string.

common/formatter/formatter.h

@@ -3,13 +3,8 @@
 #include <optional>
 #include <string>
 
-#include "common/formatter/rules/formatting_rules.h"
-#include "common/formatter/rules/rule_config.h"
-
 #include "tree_sitter/api.h"
 
-// TODO:
-// - Considering _eventually_ adding line-length heuristics
 namespace formatter {
 
 struct TreeSitterParserDeleter {

common/formatter/formatter_tree.cpp

@@ -2,7 +2,7 @@
 
 #include "common/util/string_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 std::string get_source_code(const std::string& source, const TSNode& node) {
   uint32_t start = ts_node_start_byte(node);
@@ -79,18 +79,18 @@ FormatterTree::FormatterTree(const std::string& source, const TSNode& root_node)
 }
 
 const std::unordered_map<std::string, std::vector<std::string>> node_type_ignorable_contents = {
-    {"list_lit", {"(", ")"}},
-    {"quoting_lit", {"'"}},
-    {"unquoting_lit", {","}},
-    {"quasi_quoting_lit", {"`"}}};
+    {"list_lit", {"(", ")"}}};
 
 // TODO make an imperative version eventually
+// TODO - cleanup duplication
 void FormatterTree::construct_formatter_tree_recursive(const std::string& source,
                                                        TSNode curr_node,
                                                        FormatterTreeNode& tree_node,
                                                        std::optional<std::string> node_prefix) {
   if (ts_node_child_count(curr_node) == 0) {
-    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
+    auto new_node = FormatterTreeNode(source, curr_node);
+    new_node.node_prefix = node_prefix;
+    tree_node.refs.push_back(new_node);
     return;
   }
   const std::string curr_node_type = ts_node_type(curr_node);
@@ -105,11 +105,37 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
     tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
     return;
   } else if (curr_node_type == "quoting_lit") {
-    next_node_prefix = "'";
+    if (node_prefix) {
+      node_prefix.value() += "'";
+    } else {
+      node_prefix = "'";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   } else if (curr_node_type == "unquoting_lit") {
-    next_node_prefix = ",";
+    if (node_prefix) {
+      node_prefix.value() += ",";
+    } else {
+      node_prefix = ",";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   } else if (curr_node_type == "quasi_quoting_lit") {
-    next_node_prefix = "`";
+    if (node_prefix) {
+      node_prefix.value() += "`";
+    } else {
+      node_prefix = "`";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
+  } else if (curr_node_type == "unquote_splicing_lit") {
+    if (node_prefix) {
+      node_prefix.value() += ",@";
+    } else {
+      node_prefix = ",@";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   }
   std::vector<std::string> skippable_nodes = {};
   if (node_type_ignorable_contents.find(curr_node_type) != node_type_ignorable_contents.end()) {
@@ -117,6 +143,7 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
   }
   for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
     const auto child_node = ts_node_child(curr_node, i);
+    auto debug_child = ts_node_string(child_node);
     const auto contents = get_source_code(source, child_node);
     bool skip_node = false;
     for (const auto& skippable_content : skippable_nodes) {
@@ -129,19 +156,22 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
       continue;
     }
     if (curr_node_type == "list_lit") {
-      construct_formatter_tree_recursive(source, child_node, list_node, next_node_prefix);
+      construct_formatter_tree_recursive(source, child_node, list_node, {});
       if (node_prefix) {
         list_node.node_prefix = node_prefix;
       }
     } else {
-      construct_formatter_tree_recursive(source, child_node, tree_node, next_node_prefix);
-      // TODO - im not sure if this is correct
+      construct_formatter_tree_recursive(source, child_node, tree_node, node_prefix);
       if (node_prefix && !tree_node.refs.empty()) {
         tree_node.refs.at(tree_node.refs.size() - 1).node_prefix = node_prefix;
       }
     }
   }
   if (curr_node_type == "list_lit") {
+    // special case for empty lists
+    if (node_prefix && !list_node.node_prefix) {
+      list_node.node_prefix = node_prefix;
+    }
     tree_node.refs.push_back(list_node);
   }
 }

common/formatter/rules/formatting_rules.cpp

@@ -4,7 +4,7 @@
 
 #include "common/util/string_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace formatter_rules {
 
@@ -12,12 +12,16 @@ namespace formatter_rules {
 // differentiate between a quoted symbol and a quoted form
 const std::set<std::string> constant_types = {"kwd_lit",  "num_lit",  "str_lit",
                                               "char_lit", "null_lit", "bool_lit"};
+const std::set<std::string> constant_type_forms = {"meters", "seconds", "degrees"};
 
 namespace constant_list {
 bool is_constant_list(const FormatterTreeNode& node) {
   if (!node.is_list() || node.refs.empty()) {
     return false;
   }
+  if (!node.refs.at(0).token) {
+    return true;
+  }
   const auto& type = node.refs.at(0).metadata.node_type;
   return constant_types.find(type) != constant_types.end();
 }
@@ -36,21 +40,28 @@ bool should_insert_blank_line(const FormatterTreeNode& containing_node,
   if (node.metadata.is_comment && node.metadata.num_blank_lines_following == 0) {
     return false;
   }
-  // If the next form is a comment and is inline, don't insert a comment
+  // If the next form is a comment and is inline, don't insert a new line
   if ((index + 1) < (int)containing_node.refs.size() &&
       containing_node.refs.at(index + 1).metadata.is_comment &&
       containing_node.refs.at(index + 1).metadata.is_inline) {
     return false;
   }
 
-  // TODO - only if the form doesn't fit on a single line
+  if (node.formatting_config.elide_top_level_newline) {
+    if ((index + 1) < (int)containing_node.refs.size() &&
+        containing_node.refs.at(index + 1).metadata.is_comment) {
+      return true;
+    }
+    return false;
+  }
+
   return true;
 }
 
 }  // namespace blank_lines
 
 namespace comments {
-std::string format_block_comment(const std::string& comment) {
+std::vector<std::string> format_block_comment(const std::string& comment) {
   // Normalize block comments, remove any trailing or leading whitespace
   // Only allow annotations on the first line, like #|@file
   // Don't mess with internal indentation as the user might intend it to be a certain way.
@@ -71,12 +82,22 @@ std::string format_block_comment(const std::string& comment) {
   // Remove trailing whitespace
   comment_contents = str_util::rtrim(comment_contents);
   // remove |#
-  // TODO - check suffix
-  comment_contents.pop_back();
-  comment_contents.pop_back();
+  if (str_util::ends_with(comment_contents, "|#")) {
+    comment_contents.pop_back();
+    comment_contents.pop_back();
+  }
   comment_contents = str_util::rtrim(comment_contents);
-  new_comment += fmt::format("\n{}\n|#", comment_contents);
-  return new_comment;
+  std::vector<std::string> lines = {new_comment};
+  const auto contents_as_lines = str_util::split_string(comment_contents, "\n");
+  if (contents_as_lines.size() > 1) {
+    for (const auto& line : contents_as_lines) {
+      lines.push_back(line);
+    }
+    lines.push_back("|#");
+  } else {
+    lines.at(0) = fmt::format("{} {} |#", new_comment, str_util::trim(contents_as_lines.at(0)));
+  }
+  return lines;
 }
 }  // namespace comments
 
@@ -101,6 +122,46 @@ bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node
   return true;
 }
 
+// TODO - potentially remove the above
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node) {
+  if (prev_node.metadata.node_type == "kwd_lit") {
+    // Handle standard constant types
+    // TODO - pair up sym_names as well
+    if (constant_types.find(curr_node.metadata.node_type) != constant_types.end()) {
+      if (curr_node.metadata.node_type != "kwd_lit") {
+        // NOTE - there is ambiugity here which cannot be totally solved (i think?)
+        // if the element itself is also a keyword, assume this is two adjacent keywords and they
+        // should not be paired
+        return true;
+      }
+    }
+    // Quoted symbols
+    if (curr_node.metadata.node_type == "sym_name" && curr_node.node_prefix &&
+        (curr_node.node_prefix.value() == "'" || curr_node.node_prefix.value() == ",")) {
+      return true;
+    }
+    if (!curr_node.refs.empty()) {
+      // Constant forms special cases (ie. meters)
+      if (constant_type_forms.find(curr_node.refs.at(0).token_str()) != constant_type_forms.end()) {
+        return true;
+      }
+      // If they are just a list of symbol names (enum or simple method call)
+      bool all_symbols = true;
+      for (const auto& ref : curr_node.refs) {
+        if (ref.metadata.node_type != "sym_name") {
+          all_symbols = false;
+          break;
+        }
+      }
+      if (all_symbols) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 bool form_should_be_constant_paired(const FormatterTreeNode& node) {
   // Criteria for a list to be constant paired:
   // - needs to start with a non-symbol

common/formatter/rules/formatting_rules.h

@@ -36,7 +36,7 @@ bool should_insert_blank_line(const FormatterTreeNode& containing_node,
 //
 // Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/comments.md
 namespace comments {
-std::string format_block_comment(const std::string& comment);
+std::vector<std::string> format_block_comment(const std::string& comment);
 }
 
 // Paired elements in a list will be kept in-line rather than the default new-line indentation
@@ -72,6 +72,8 @@ const static int min_pair_amount = 4;
 bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node,
                                         const FormatterTreeNode& node,
                                         const int index);
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node);
 bool form_should_be_constant_paired(const FormatterTreeNode& node);
 }  // namespace constant_pairs
 

common/formatter/rules/rule_config.cpp

@@ -1,24 +1,146 @@
 #include "rule_config.h"
 
+#include "common/util/string_util.h"
+
 namespace formatter_rules {
 namespace config {
 
+static FormFormattingConfig new_inlinable_simple_flow_rule() {
+  return {.config_set = true, .hang_forms = false};
+}
+
 static FormFormattingConfig new_permissive_flow_rule() {
   return {.config_set = true, .hang_forms = false, .combine_first_two_lines = true};
 }
 
-static FormFormattingConfig new_flow_rule(int start_index) {
+static FormFormattingConfig new_flow_rule(int start_index, bool has_constant_pairs = false) {
   return {.config_set = true,
           .hang_forms = false,
-          .inline_until_index = [start_index](const std::vector<std::string>& /*curr_lines*/) {
-            return start_index;
-          }};
+          .inline_until_index =
+              [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+          .has_constant_pairs = has_constant_pairs};
+}
+
+static FormFormattingConfig new_function_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index =
+          [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+      .has_constant_pairs = has_constant_pairs};
+  auto arg_list_config = std::make_shared<FormFormattingConfig>();
+  arg_list_config->force_inline = true;
+  arg_list_config->hang_forms = false;
+  cfg.index_configs.emplace(2, arg_list_config);
+  return cfg;
+}
+
+static FormFormattingConfig new_inlineable_flow_rule(int start_index,
+                                                     bool has_constant_pairs = false) {
+  return {.config_set = true,
+          .hang_forms = false,
+          .inline_until_index =
+              [start_index](const std::vector<std::string>& curr_lines) {
+                int total_width = 0;
+                for (const auto& line : curr_lines) {
+                  // Check for comments
+                  // TODO - this shows how this isn't really the best strategy but it holds up
+                  if (str_util::contains(line, ";")) {
+                    // Can't inline, there's a comment!
+                    return start_index;
+                  }
+                  total_width += line.length();
+                  // an empty line implies a new-line was forced, this is bleeding implementation
+                  // details, but fine for now
+                  if (line.empty()) {
+                    return start_index;
+                  }
+                }
+                if (total_width <= 120) {
+                  return (int)curr_lines.size();
+                }
+                return start_index;
+              },
+          .has_constant_pairs = has_constant_pairs};
+}
+
+static FormFormattingConfig new_defstate_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index =
+          [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+      .has_constant_pairs = has_constant_pairs};
+  std::vector<int> state_handler_indexes = {4,  6, 8, 10,
+                                            12, 14};  // NOTE - not all of these have to be defined
+  for (const auto& index : state_handler_indexes) {
+    auto temp_config = std::make_shared<FormFormattingConfig>();
+    temp_config->prevent_inlining = true;
+    temp_config->parent_mutable_extra_indent = 2;
+    cfg.index_config_override.emplace(index, temp_config);
+  }
+  return cfg;
+}
+
+static FormFormattingConfig new_defmethod_rule(int start_index, bool has_constant_pairs = false) {
+  // TODO - might be nice to have a function that returns a config based on a given index, instead
+  // of hardcoding them!
+  // Right now this only works for non-`new` methods (else we may bleed into the body of a normal
+  // method)
+  auto arg_list_config = std::make_shared<FormFormattingConfig>();
+  arg_list_config->force_inline = true;
+  arg_list_config->hang_forms = false;
+  FormFormattingConfig cfg = {.config_set = true,
+                              .hang_forms = false,
+                              .inline_until_index =
+                                  [start_index](const std::vector<std::string>& curr_lines) {
+                                    if (curr_lines.size() >= 2 && curr_lines.at(1) == "new") {
+                                      // defmethod was changed to omit the type name for everything
+                                      // except the `new` method, so special case.
+                                      return start_index + 1;
+                                    }
+                                    return start_index;
+                                  },
+                              .has_constant_pairs = has_constant_pairs};
+  cfg.index_configs.emplace(2, arg_list_config);
+  return cfg;
+}
+
+static FormFormattingConfig new_lambda_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {.config_set = true,
+                              .hang_forms = false,
+                              .inline_until_index =
+                                  [start_index](const std::vector<std::string>& curr_lines) {
+                                    if (curr_lines.size() >= 2 && curr_lines.at(1) == ":behavior") {
+                                      // defmethod was changed to omit the type name for everything
+                                      // except the `new` method, so special case.
+                                      return start_index + 2;
+                                    }
+                                    return start_index;
+                                  },
+                              .has_constant_pairs = has_constant_pairs};
+  return cfg;
+}
+
+static FormFormattingConfig new_defenum_rule() {
+  auto temp_list_config = std::make_shared<FormFormattingConfig>();
+  temp_list_config->force_inline = true;
+  temp_list_config->hang_forms = false;
+  return {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index = [](const std::vector<std::string>& /*curr_lines*/) { return 2; },
+      .has_constant_pairs = true,
+      .default_index_config = temp_list_config,
+  };
 }
 
 static FormFormattingConfig new_deftype_rule(
     int start_index,
+    int num_columns_to_compute_widths,
     const std::vector<int>& inlining_preventation_indices) {
   FormFormattingConfig cfg;
+  cfg.has_constant_pairs = true;
   cfg.config_set = true;
   cfg.hang_forms = false;
   cfg.inline_until_index = [start_index](std::vector<std::string> curr_lines) {
@@ -39,6 +161,7 @@ static FormFormattingConfig new_deftype_rule(
     temp_config->default_index_config = temp_list_config;
     if (index == 3) {
       temp_config->determine_column_widths_for_list_elements = true;
+      temp_config->num_columns_to_compute_widths = num_columns_to_compute_widths;
     }
     cfg.index_configs.emplace(index, temp_config);
   }
@@ -74,6 +197,29 @@ static FormFormattingConfig new_binding_rule(int form_head_width) {
   return cfg;
 }
 
+static FormFormattingConfig new_inline_binding_rule(int form_head_width) {
+  FormFormattingConfig cfg;
+  cfg.config_set = true;
+  cfg.hang_forms = false;
+  cfg.combine_first_two_lines = true;
+  auto binding_list_config = std::make_shared<FormFormattingConfig>();
+  binding_list_config->config_set = true;
+  binding_list_config->hang_forms = false;
+  binding_list_config->indentation_width = 1;
+  binding_list_config->indentation_width_for_index = [form_head_width](FormFormattingConfig /*cfg*/,
+                                                                       int index) {
+    if (index == 0) {
+      return 0;
+    }
+    return form_head_width;
+  };
+  binding_list_config->should_prevent_inlining = [](FormFormattingConfig /*config*/, int num_refs) {
+    return false;
+  };
+  cfg.index_configs.emplace(1, binding_list_config);
+  return cfg;
+}
+
 static FormFormattingConfig new_pair_rule(bool combine_first_two_expr) {
   FormFormattingConfig cfg;
   cfg.config_set = true;
@@ -88,23 +234,70 @@ static FormFormattingConfig new_pair_rule(bool combine_first_two_expr) {
   return cfg;
 }
 
+static FormFormattingConfig new_top_level_inline_form(bool elide_new_line) {
+  return {.force_inline = true, .elide_top_level_newline = elide_new_line};
+}
+
 const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config = {
     {"case", new_pair_rule(true)},
     {"cond", new_pair_rule(false)},
-    {"defmethod", new_flow_rule(3)},
-    {"deftype", new_deftype_rule(3, {3, 4, 5, 6})},
+    {"#cond", new_pair_rule(false)},
+    {"in-package", new_top_level_inline_form(true)},
+    {"bundles", new_top_level_inline_form(true)},
+    {"require", new_top_level_inline_form(true)},
+    {"def-art-elt", new_top_level_inline_form(true)},
+    {"def-joint-node", new_top_level_inline_form(true)},
+    {"declare-file", new_top_level_inline_form(false)},
+    {"defenum", new_defenum_rule()},
+    {"defmethod", new_defmethod_rule(3)},
+    {"lambda", new_lambda_rule(2)},
+    {"deftype", new_deftype_rule(3, 1, {3, 4, 5, 6})},
     {"defun", new_flow_rule(3)},
+    {"defun-recursive", new_flow_rule(4)},
+    {"defun-debug-recursive", new_flow_rule(4)},
     {"defun-debug", new_flow_rule(3)},
     {"defbehavior", new_flow_rule(4)},
-    {"if", new_permissive_flow_rule()},
+    {"if", new_inlineable_flow_rule(2)},
+    {"#if", new_inlineable_flow_rule(2)},
     {"define", new_permissive_flow_rule()},
+    {"def-mips2c", new_permissive_flow_rule()},
+    {"defconstant", new_permissive_flow_rule()},
+    {"defglobalconstant", new_permissive_flow_rule()},
+    {"defmethod-mips2c", new_permissive_flow_rule()},
     {"define-extern", new_permissive_flow_rule()},
-    {"defmacro", new_flow_rule(3)},
+    {"declare-type", new_permissive_flow_rule()},
+    {"defmacro", new_function_rule(3)},
+    {"desfun", new_function_rule(3)},
+    {"defskelgroup", new_flow_rule(2, true)},
+    {"defpartgroup", new_flow_rule(2, true)},
+    {"defpart", new_flow_rule(2, true)},
+    {"defstate", new_defstate_rule(3, true)},
+    {"behavior", new_flow_rule(2)},
     {"dotimes", new_flow_rule(2)},
+    {"dolist", new_flow_rule(2)},
+    {"process-spawn-function", new_flow_rule(2)},
     {"let", new_binding_rule(4)},
+    {"protect", new_binding_rule(4)},
+    {"let*", new_binding_rule(5)},
     {"rlet", new_binding_rule(5)},
     {"when", new_flow_rule(2)},
+    {"unless", new_flow_rule(2)},
+    {"with-profiler", new_flow_rule(2)},
+    {"with-pc", new_flow_rule(0)},
+    {"#unless", new_flow_rule(2)},
+    {"#when", new_flow_rule(2)},
+    {"countdown", new_flow_rule(2)},
+    {"until", new_flow_rule(2)},
+    {"loop", new_flow_rule(0)},
+    {"while", new_flow_rule(2)},
     {"begin", new_flow_rule(0)},
+    {"seval", new_flow_rule(0)},
+    {"with-pp", new_flow_rule(0)},
+    {"with-gensyms", new_flow_rule(2)},
+    {"with-vf0", new_flow_rule(0)},
+    {"with-vf", new_inline_binding_rule(8)},
+    {"with-cnt-vif-block", new_inline_binding_rule(19)},
+    {"local-vars", new_inlinable_simple_flow_rule()},
     {"with-dma-buffer-add-bucket", new_flow_rule(2)}};
 }  // namespace config
 }  // namespace formatter_rules

common/formatter/rules/rule_config.h

@@ -7,7 +7,6 @@
 #include <unordered_map>
 #include <vector>
 
-// TODO - some way to apply a config to all list elements (index configs with -1?)
 namespace formatter_rules {
 namespace config {
 struct FormFormattingConfig {
@@ -33,9 +32,14 @@ struct FormFormattingConfig {
   int parent_mutable_extra_indent = 0;
   std::optional<std::shared_ptr<FormFormattingConfig>> default_index_config;
   std::unordered_map<int, std::shared_ptr<FormFormattingConfig>> index_configs = {};
+  // TODO / NOTe - not fully implemented, only doing `parent_mutable_extra_indent` right now
+  std::unordered_map<int, std::shared_ptr<FormFormattingConfig>> index_config_override = {};
 
   bool determine_column_widths_for_list_elements = false;
+  int num_columns_to_compute_widths = 0;
   std::vector<int> list_element_column_widths = {};
+
+  bool elide_top_level_newline = false;
 };
 
 extern const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config;

common/global_profiler/GlobalProfiler.cpp

@@ -9,7 +9,7 @@
 #include "common/util/FileUtil.h"
 #include "common/common_types.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 #include "third-party/json.hpp"
 
 #ifdef OS_POSIX
@@ -175,7 +175,7 @@ void GlobalProfiler::dump_to_json(const std::string& path) {
       lg::debug("out of order: {} {} {} ms", event.ts / 1000.f, info.debug / 1000.f,
                 (info.debug - event.ts) / 1000000.f);
       lg::debug("  idx: {}, range {} {}", event_idx, info.lowest_at_target, info.highest_at_target);
-      lg::debug("  now: {}", m_next_idx);
+      lg::debug("  now: {}", m_next_idx.load());
     }
     info.debug = event.ts;
   }

common/goos/Interpreter.cpp

@@ -16,7 +16,7 @@
 #include "common/util/string_util.h"
 #include "common/util/unicode_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace goos {
 Interpreter::Interpreter(const std::string& username) {
@@ -571,104 +571,6 @@ Object Interpreter::eval(Object obj, const std::shared_ptr<EnvironmentObject>& e
   }
 }
 
-EnvironmentMap::EnvironmentMap() {
-  clear();
-}
-
-void EnvironmentMap::clear() {
-  m_entries.clear();
-  m_power_of_two_size = 3;  // 2 ^ 3 = 8
-  m_entries.resize(8);
-  m_used_entries = 0;
-  m_next_resize = (m_entries.size() * kMaxUsed);
-  m_mask = 0b111;
-}
-
-Object* EnvironmentMap::lookup(InternedSymbolPtr str) {
-  if (m_entries.size() < 10) {
-    for (auto& e : m_entries) {
-      if (e.key == str.name_ptr) {
-        return &e.value;
-      }
-    }
-    return nullptr;
-  }
-  u32 hash = crc32((const u8*)&str.name_ptr, sizeof(const char*));
-
-  // probe
-  for (u32 i = 0; i < m_entries.size(); i++) {
-    u32 slot_addr = (hash + i) & m_mask;
-    auto& slot = m_entries[slot_addr];
-    if (!slot.key) {
-      return nullptr;
-    } else {
-      if (slot.key != str.name_ptr) {
-        continue;  // bad hash
-      }
-      return &slot.value;
-    }
-  }
-
-  // should be impossible to reach.
-  ASSERT_NOT_REACHED();
-}
-
-void EnvironmentMap::set(InternedSymbolPtr ptr, const Object& obj) {
-  u32 hash = crc32((const u8*)&ptr.name_ptr, sizeof(const char*));
-
-  // probe
-  for (u32 i = 0; i < m_entries.size(); i++) {
-    u32 slot_addr = (hash + i) & m_mask;
-    auto& slot = m_entries[slot_addr];
-    if (!slot.key) {
-      // not found, insert!
-      slot.key = ptr.name_ptr;
-      slot.value = obj;
-      m_used_entries++;
-
-      if (m_used_entries >= m_next_resize) {
-        resize();
-      }
-      return;
-    } else {
-      if (slot.key == ptr.name_ptr) {
-        slot.value = obj;
-        return;
-      }
-    }
-  }
-
-  // should be impossible to reach.
-  ASSERT_NOT_REACHED();
-}
-
-void EnvironmentMap::resize() {
-  m_power_of_two_size++;
-  m_mask = (1U << m_power_of_two_size) - 1;
-
-  std::vector<Entry> new_entries(m_entries.size() * 2);
-  for (const auto& old_entry : m_entries) {
-    if (old_entry.key) {
-      bool done = false;
-      u32 hash = crc32((const u8*)&old_entry.key, sizeof(const char*));
-      for (u32 i = 0; i < new_entries.size(); i++) {
-        u32 slot_addr = (hash + i) & m_mask;
-        auto& slot = new_entries[slot_addr];
-        if (!slot.key) {
-          slot.key = old_entry.key;
-          slot.value = std::move(old_entry.value);
-          done = true;
-          break;
-        }
-      }
-      ASSERT(done);
-    }
-  }
-
-  m_entries = std::move(new_entries);
-  m_next_resize = kMaxUsed * m_entries.size();
-}
-
 namespace {
 
 /*!

common/goos/Object.cpp

@@ -47,7 +47,7 @@
 #include "common/util/crc32.h"
 #include "common/util/print_float.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace goos {
 

common/goos/Object.h

@@ -55,6 +55,7 @@
 
 #include "common/common_types.h"
 #include "common/util/Assert.h"
+#include "common/util/crc32.h"
 
 namespace goos {
 
@@ -456,23 +457,111 @@ class PairObject : public HeapObject {
   ~PairObject() = default;
 };
 
-class EnvironmentMap {
+template <typename T>
+class InternedPtrMap {
  public:
-  EnvironmentMap(const EnvironmentMap&) = delete;
-  EnvironmentMap& operator=(const EnvironmentMap&) = delete;
-  EnvironmentMap();
-  Object* lookup(InternedSymbolPtr ptr);
-  void set(InternedSymbolPtr ptr, const Object& obj);
-  void clear();
+  InternedPtrMap(const InternedPtrMap&) = delete;
+  InternedPtrMap& operator=(const InternedPtrMap&) = delete;
+  InternedPtrMap() { clear(); }
+
+  T* lookup(InternedSymbolPtr str) {
+    if (m_entries.size() < 10) {
+      for (auto& e : m_entries) {
+        if (e.key == str.name_ptr) {
+          return &e.value;
+        }
+      }
+      return nullptr;
+    }
+    u32 hash = crc32((const u8*)&str.name_ptr, sizeof(const char*));
+
+    // probe
+    for (u32 i = 0; i < m_entries.size(); i++) {
+      u32 slot_addr = (hash + i) & m_mask;
+      auto& slot = m_entries[slot_addr];
+      if (!slot.key) {
+        return nullptr;
+      } else {
+        if (slot.key != str.name_ptr) {
+          continue;  // bad hash
+        }
+        return &slot.value;
+      }
+    }
+
+    // should be impossible to reach.
+    ASSERT_NOT_REACHED();
+  }
+  void set(InternedSymbolPtr ptr, const T& obj) {
+    u32 hash = crc32((const u8*)&ptr.name_ptr, sizeof(const char*));
+
+    // probe
+    for (u32 i = 0; i < m_entries.size(); i++) {
+      u32 slot_addr = (hash + i) & m_mask;
+      auto& slot = m_entries[slot_addr];
+      if (!slot.key) {
+        // not found, insert!
+        slot.key = ptr.name_ptr;
+        slot.value = obj;
+        m_used_entries++;
+
+        if (m_used_entries >= m_next_resize) {
+          resize();
+        }
+        return;
+      } else {
+        if (slot.key == ptr.name_ptr) {
+          slot.value = obj;
+          return;
+        }
+      }
+    }
+
+    // should be impossible to reach.
+    ASSERT_NOT_REACHED();
+  }
+  void clear() {
+    m_entries.clear();
+    m_power_of_two_size = 3;  // 2 ^ 3 = 8
+    m_entries.resize(8);
+    m_used_entries = 0;
+    m_next_resize = (m_entries.size() * kMaxUsed);
+    m_mask = 0b111;
+  }
 
  private:
   struct Entry {
     const char* key = nullptr;
-    Object value;
+    T value;
   };
   std::vector<Entry> m_entries;
 
-  void resize();
+  void resize() {
+    m_power_of_two_size++;
+    m_mask = (1U << m_power_of_two_size) - 1;
+
+    std::vector<Entry> new_entries(m_entries.size() * 2);
+    for (const auto& old_entry : m_entries) {
+      if (old_entry.key) {
+        bool done = false;
+        u32 hash = crc32((const u8*)&old_entry.key, sizeof(const char*));
+        for (u32 i = 0; i < new_entries.size(); i++) {
+          u32 slot_addr = (hash + i) & m_mask;
+          auto& slot = new_entries[slot_addr];
+          if (!slot.key) {
+            slot.key = old_entry.key;
+            slot.value = std::move(old_entry.value);
+            done = true;
+            break;
+          }
+        }
+        ASSERT(done);
+      }
+    }
+
+    m_entries = std::move(new_entries);
+    m_next_resize = kMaxUsed * m_entries.size();
+  }
   int m_power_of_two_size = 0;
   int m_used_entries = 0;
   int m_next_resize = 0;
@@ -480,6 +569,8 @@ class EnvironmentMap {
   static constexpr float kMaxUsed = 0.7;
 };
 
+using EnvironmentMap = InternedPtrMap<Object>;
+
 class EnvironmentObject : public HeapObject {
  public:
   std::string name;

common/goos/ParseHelpers.cpp

@@ -1,6 +1,6 @@
 #include "ParseHelpers.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace goos {
 

common/goos/PrettyPrinter.cpp

@@ -15,7 +15,7 @@
 #include "common/goos/PrettyPrinter2.h"
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace pretty_print {
 

common/goos/PrettyPrinter2.cpp

@@ -3,8 +3,8 @@
 #include "common/common_types.h"
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/format.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
 
 namespace pretty_print {
 

common/goos/Printer.cpp

@@ -6,7 +6,7 @@
 #include "common/goos/Object.h"
 #include "common/util/print_float.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace pretty_print {
 namespace {

common/goos/Reader.cpp

@@ -12,11 +12,10 @@
 #include "Reader.h"
 
 #include "common/log/log.h"
-#include "common/repl/util.h"
 #include "common/util/FileUtil.h"
 #include "common/util/FontUtils.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace goos {
 

common/goos/Reader.h

@@ -18,7 +18,7 @@
 
 #include "common/goos/Object.h"
 #include "common/goos/TextDB.h"
-#include "common/repl/util.h"
+#include "common/repl/repl_wrapper.h"
 #include "common/util/Assert.h"
 
 namespace goos {

common/goos/TextDB.cpp

@@ -15,7 +15,7 @@
 
 #include "common/util/FileUtil.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace goos {
 

common/log/log.cpp

@@ -5,7 +5,7 @@
 #include <cstdlib>
 #include <mutex>
 
-#include "third-party/fmt/color.h"
+#include "fmt/color.h"
 #ifdef _WIN32  // see lg::initialize
 #define NOMINMAX
 #define WIN32_LEAN_AND_MEAN
@@ -163,7 +163,7 @@ void set_file(const std::string& filename,
         file_util::find_files_in_dir(fs::path(complete_filename).parent_path(),
                                      std::regex(fmt::format("{}\\.(\\d\\.)?log", filename)));
     for (const auto& file : old_log_files) {
-      lg::info("removing {}", file.string());
+      lg::debug("removing {}", file.string());
       fs::remove(file);
     }
     // remove the oldest log file if there are more than LOG_ROTATE_MAX
@@ -172,9 +172,9 @@ void set_file(const std::string& filename,
     // sort the names and remove them
     existing_log_files = file_util::sort_filepaths(existing_log_files, true);
     if (existing_log_files.size() > (LOG_ROTATE_MAX - 1)) {
-      lg::info("removing {} log files", existing_log_files.size() - (LOG_ROTATE_MAX - 1));
+      lg::debug("removing {} log files", existing_log_files.size() - (LOG_ROTATE_MAX - 1));
       for (int i = 0; i < (int)existing_log_files.size() - (LOG_ROTATE_MAX - 1); i++) {
-        lg::info("removing {}", existing_log_files.at(i).string());
+        lg::debug("removing {}", existing_log_files.at(i).string());
         fs::remove(existing_log_files.at(i));
       }
     }

common/log/log.h

@@ -7,8 +7,8 @@
 #endif
 #include <string>
 
-#include "third-party/fmt/color.h"
-#include "third-party/fmt/core.h"
+#include "fmt/color.h"
+#include "fmt/core.h"
 
 namespace lg {
 

common/math/Vector.h

@@ -2,7 +2,7 @@
 
 #include <cmath>
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace math {
 

common/repl/config.cpp

@@ -2,20 +2,26 @@
 
 #include "common/versions/versions.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace REPL {
 void to_json(json& j, const Config& obj) {
   j = json{
+      {"nreplPort", obj.nrepl_port},
       {"gameVersionFolder", obj.game_version_folder},
       {"numConnectToTargetAttempts", obj.target_connect_attempts},
       {"asmFileSearchDirs", obj.asm_file_search_dirs},
       {"keybinds", obj.keybinds},
       {"perGameHistory", obj.per_game_history},
+      {"permissiveRedefinitions", obj.permissive_redefinitions},
   };
 }
 
 void from_json(const json& j, Config& obj) {
+  // TODO - make a camelCase variant of json_serialize/deserialize macros
+  if (j.contains("nreplPort")) {
+    j.at("nreplPort").get_to(obj.nrepl_port);
+  }
   if (j.contains("gameVersionFolder")) {
     j.at("gameVersionFolder").get_to(obj.game_version_folder);
   }
@@ -55,6 +61,9 @@ void from_json(const json& j, Config& obj) {
   if (j.contains("perGameHistory")) {
     j.at("perGameHistory").get_to(obj.per_game_history);
   }
+  if (j.contains("permissiveRedefinitions")) {
+    j.at("permissiveRedefinitions").get_to(obj.permissive_redefinitions);
+  }
   // if there is game specific configuration, override any values we just set
   if (j.contains(version_to_game_name(obj.game_version))) {
     from_json(j.at(version_to_game_name(obj.game_version)), obj);

common/repl/config.h

@@ -26,11 +26,14 @@ struct KeyBind {
 void to_json(json& j, const KeyBind& obj);
 void from_json(const json& j, KeyBind& obj);
 
+// TODO - per-game config
 struct Config {
   GameVersion game_version;
   Config(GameVersion _game_version) : game_version(_game_version){};
 
   // this is the default REPL configuration
+  int nrepl_port = 8181;
+  int temp_nrepl_port = -1;
   std::string game_version_folder;
   int target_connect_attempts = 30;
   std::vector<std::string> asm_file_search_dirs = {};
@@ -45,6 +48,14 @@ struct Config {
       {KeyBind::Modifier::CTRL, "B", "Displays the most recently caught backtrace", "(:di)"},
       {KeyBind::Modifier::CTRL, "N", "Full build of the game", "(mi)"}};
   bool per_game_history = true;
+  bool permissive_redefinitions = false;
+
+  int get_nrepl_port() {
+    if (temp_nrepl_port != -1) {
+      return temp_nrepl_port;
+    }
+    return nrepl_port;
+  }
 };
 void to_json(json& j, const Config& obj);
 void from_json(const json& j, Config& obj);

common/repl/nrepl/ReplClient.cpp

@@ -4,7 +4,7 @@
 #include "common/cross_sockets/XSocket.h"
 #include "common/versions/versions.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 #ifdef _WIN32
 #define NOMINMAX

common/repl/nrepl/ReplServer.cpp

@@ -4,7 +4,7 @@
 #include "common/cross_sockets/XSocket.h"
 #include "common/versions/versions.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 #ifdef _WIN32
 #define NOMINMAX
@@ -16,10 +16,6 @@
 #include "common/log/log.h"
 // clang-format on
 
-// TODO - basically REPL to listen and inject commands into a running REPL
-// - we will need a C++ side client as well which will let us communicate with the repl via for
-// example, ImgUI
-//
 // TODO - The server also needs to eventually return the result of the evaluation
 
 ReplServer::~ReplServer() {
@@ -31,7 +27,18 @@ ReplServer::~ReplServer() {
 
 void ReplServer::post_init() {
   // Add the listening socket to our set of sockets
-  lg::info("[nREPL:{}:{}] awaiting connections", tcp_port, listening_socket);
+  lg::debug("[nREPL:{}:{}] awaiting connections", tcp_port, listening_socket);
+}
+
+void ReplServer::error_response(int socket, const std::string& error) {
+  std::string msg = fmt::format("[ERROR]: {}", error);
+  auto resp = write_to_socket(socket, msg.c_str(), msg.size());
+  if (resp == -1) {
+    lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
+             ntohs(addr.sin_port), socket);
+    close_socket(socket);
+    client_sockets.erase(socket);
+  }
 }
 
 void ReplServer::ping_response(int socket) {
@@ -52,7 +59,6 @@ std::optional<std::string> ReplServer::get_msg() {
 
   // Add the server's main listening socket (where we accept clients from)
   FD_SET(listening_socket, &read_sockets);
-
   int max_sd = listening_socket;
   for (const int& sock : client_sockets) {
     if (sock > max_sd) {
@@ -64,12 +70,11 @@ std::optional<std::string> ReplServer::get_msg() {
   }
 
   // Wait for activity on _something_, with a timeout so we don't get stuck here on exit.
-  struct timeval timeout;
-  timeout.tv_sec = 0;
-  timeout.tv_usec = 100000;
+  struct timeval timeout = {0, 100000};
   auto activity = select(max_sd + 1, &read_sockets, NULL, NULL, &timeout);
-
-  if (activity < 0) {  // TODO - || error!
+  if (activity < 0 && errno != EINTR) {
+    lg::error("[nREPL:{}] select error, returned: {}, errno: {}", tcp_port, activity,
+              strerror(errno));
     return std::nullopt;
   }
 
@@ -78,44 +83,46 @@ std::optional<std::string> ReplServer::get_msg() {
     socklen_t addr_len = sizeof(addr);
     auto new_socket = accept_socket(listening_socket, (sockaddr*)&addr, &addr_len);
     if (new_socket < 0) {
-      // TODO - handle error
+      if (new_socket != -1) {
+        lg::error("[nREPL:{}] accept error, returned: {}, errono: {}", tcp_port, new_socket,
+                  strerror(errno));
+      }
     } else {
       lg::info("[nREPL:{}]: New socket connection: {}:{}:{}", tcp_port, address_to_string(addr),
                ntohs(addr.sin_port), new_socket);
-
       // Say hello
       ping_response(new_socket);
       // Track the new socket
       if ((int)client_sockets.size() < max_clients) {
         client_sockets.insert(new_socket);
       } else {
-        // TODO - Respond with NO
+        // Respond with NO and close the socket
+        lg::warn("[nREPL:{}]: Maximum clients reached. Rejecting connection.", tcp_port);
+        error_response(new_socket, "Maximum clients reached. Rejecting connection.");
+        close_socket(new_socket);
       }
     }
   }
 
-  // otherwise (and no matter what) check all the clients to see if they have sent us anything
-  // else its some IO operation on some other socket
-  //
-  // RACE - the first client wins
-
-  // TODO - there are ways to do this with iterators but, couldn't figure it out!
-  std::vector<int> sockets_to_scan(client_sockets.begin(), client_sockets.end());
-  for (const int& sock : sockets_to_scan) {
+  // Check all clients for activity
+  for (auto it = client_sockets.begin(); it != client_sockets.end();) {
+    int sock = *it;
     if (FD_ISSET(sock, &read_sockets)) {
       // Attempt to read a header
-      // TODO - should this be in a loop?
       auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
-      if (req_bytes == 0) {
-        // Socket disconnected
+      if (req_bytes <= 0) {
         // TODO - add a queue of messages in the REPL::Wrapper so we can print _BEFORE_ the prompt
         // is output
-        lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
-                 ntohs(addr.sin_port), sock);
-
+        if (req_bytes == 0) {
+          lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr));
+        } else {
+          lg::warn("[nREPL:{}] Error reading from socket on {}: {}", tcp_port,
+                   address_to_string(addr), strerror(errno));
+        }
         // Cleanup the socket and remove it from our set
         close_socket(sock);
-        client_sockets.erase(sock);
+        it = client_sockets.erase(it);  // Erase and move to the next element
+        continue;
       } else {
         // Otherwise, process the message
         auto* header = (ReplServerHeader*)(header_buffer.data());
@@ -123,7 +130,12 @@ std::optional<std::string> ReplServer::get_msg() {
         int expected_size = header->length;
         int got = 0;
         int tries = 0;
+        bool skip_to_next_socket = false;
         while (got < expected_size) {
+          if (want_exit_callback()) {
+            lg::warn("[nREPL:{}] Terminating nREPL early", tcp_port);
+            return std::nullopt;
+          }
           tries++;
           if (tries > 100) {
             break;
@@ -135,11 +147,25 @@ std::optional<std::string> ReplServer::get_msg() {
                 tcp_port, got, expected_size, buffer.size());
             return std::nullopt;
           }
-          auto x = read_from_socket(sock, buffer.data() + got, expected_size - got);
-          if (want_exit_callback()) {
-            return std::nullopt;
+          auto bytes_read = read_from_socket(sock, buffer.data() + got, expected_size - got);
+          if (bytes_read <= 0) {
+            if (bytes_read == 0) {
+              lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr));
+            } else {
+              lg::warn("[nREPL:{}] Error reading from socket on {}: {}", tcp_port,
+                       address_to_string(addr), strerror(errno));
+            }
+            close_socket(sock);
+            it = client_sockets.erase(it);  // Erase and move to the next element
+            skip_to_next_socket = true;
+            break;
           }
-          got += x > 0 ? x : 0;
+          got += bytes_read;
+          std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+
+        if (skip_to_next_socket) {
+          continue;
         }
 
         switch (header->type) {
@@ -153,6 +179,7 @@ std::optional<std::string> ReplServer::get_msg() {
         }
       }
     }
+    ++it;
   }
   return std::nullopt;
 }

common/repl/nrepl/ReplServer.h

@@ -27,5 +27,6 @@ class ReplServer : public XSocketServer {
   fd_set read_sockets;
   std::set<int> client_sockets = {};
 
+  void error_response(int socket, const std::string& error);
   void ping_response(int socket);
 };

common/repl/repl_wrapper.cpp

@@ -1,43 +1,73 @@
-#include "util.h"
+#include "repl_wrapper.h"
 
 #include "common/util/FileUtil.h"
 #include "common/util/json_util.h"
 #include "common/util/string_util.h"
 #include "common/versions/versions.h"
 
-#include "third-party/fmt/color.h"
-#include "third-party/fmt/core.h"
+#include "fmt/color.h"
+#include "fmt/core.h"
 #include "third-party/replxx/include/replxx.hxx"
-// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
-// or perhaps, show the docstring for the current function being used?)
 
 namespace REPL {
 void Wrapper::clear_screen() {
   repl.clear_screen();
 }
 
-void Wrapper::print_welcome_message() {
-  // TODO - dont print on std-out
-  // Welcome message / brief intro for documentation
-  std::string ascii;
-  ascii += " _____             _____ _____ _____ __    \n";
-  ascii += "|     |___ ___ ___|   __|     |  _  |  |   \n";
-  ascii += "|  |  | . | -_|   |  |  |  |  |     |  |__ \n";
-  ascii += "|_____|  _|___|_|_|_____|_____|__|__|_____|\n";
-  ascii += "      |_|                                  \n";
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
-
-  fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
-             versions::GOAL_VERSION_MINOR);
-  fmt::print("Run {} or {} for help with common commands and REPL usage.\n",
-             fmt::styled("(repl-help)", fmt::emphasis::bold | fg(fmt::color::cyan)),
-             fmt::styled("(repl-keybinds)", fmt::emphasis::bold | fg(fmt::color::cyan)));
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
-  fmt::print(" to connect to the local target.\n");
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
-  fmt::print(" to rebuild the entire game.\n\n");
+void Wrapper::print_welcome_message(const std::vector<std::string>& loaded_projects) {
+  std::string message;
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "        ..:::::..\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "     .:-----------:.\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "         .-----.");
+  message += fmt::format(fmt::emphasis::bold, "         Welcome to OpenGOAL {}.{} [{}]",
+                         versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR,
+                         fmt::format(fg(fmt::color::gray), "{}", build_revision()));
+  if (!username.empty() && username != "#f" && username != "unknown") {
+    message += fmt::format(fg(fmt::color::light_green), " {}", username);
+  }
+  message += "!\n";
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "          .---.");
+  if (repl_config.game_version == GameVersion::Jak1) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::orange), "jak1"));
+  } else if (repl_config.game_version == GameVersion::Jak2) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::purple), "jak2"));
+  } else if (repl_config.game_version == GameVersion::Jak3) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::gold), "jak3"));
+  } else {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::magenta), "jakx"));
+  }
+  const auto loaded_projects_str = fmt::format("{}", fmt::join(loaded_projects, ","));
+  message += fmt::format(fg(fmt::color::gray), "{}\n", loaded_projects_str);
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  .        ---        .");
+  message +=
+      fmt::format("  Project Path: {}\n",
+                  fmt::format(fg(fmt::color::gray), file_util::get_jak_project_dir().string()));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  -       :===:       -");
+  message += "  nREPL:";
+  if (!nrepl_alive) {
+    message += fmt::format(fg(fmt::color::red), "DISABLED\n");
+  } else {
+    message += fmt::format(fg(fmt::color::light_green), " Listening on {}\n",
+                           repl_config.get_nrepl_port());
+  }
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  --.   .--: :--.   .--");
+  message += "  Source File Search Dirs: ";
+  const auto search_dir_string =
+      fmt::format("{}", fmt::join(repl_config.asm_file_search_dirs, ","));
+  message += fmt::format("[{}]\n", fmt::format(fg(fmt::color::gray), search_dir_string));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  .=======.    =======.");
+  message += fmt::format("  {} or {} for basic help and usage\n",
+                         fmt::format(fg(fmt::color::cyan), "(repl-help)"),
+                         fmt::format(fg(fmt::color::cyan), "(repl-keybinds)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "   .-=====-. .-=====-");
+  message +=
+      fmt::format("    {} to connect to the game\n", fmt::format(fg(fmt::color::cyan), "(lt)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "     .-===========-.");
+  message += fmt::format("     {} to recompile the active project.\n",
+                         fmt::format(fg(fmt::color::cyan), "(mi)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "         .-===-.\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "            .\n");
+  fmt::print("{}", message);
 }
 
 void Wrapper::print_to_repl(const std::string& str) {
@@ -242,20 +272,24 @@ StartupFile load_user_startup_file(const std::string& username, const GameVersio
   return startup_file;
 }
 
-REPL::Config load_repl_config(const std::string& username, const GameVersion game_version) {
+REPL::Config load_repl_config(const std::string& username,
+                              const GameVersion game_version,
+                              const int nrepl_port) {
   auto repl_config_path =
       file_util::get_jak_project_dir() / "goal_src" / "user" / username / "repl-config.json";
+  REPL::Config loaded_config(game_version);
   if (file_util::file_exists(repl_config_path.string())) {
     try {
-      REPL::Config config(game_version);
       auto repl_config_data =
           parse_commented_json(file_util::read_text_file(repl_config_path), "repl-config.json");
-      from_json(repl_config_data, config);
-      return config;
+      from_json(repl_config_data, loaded_config);
+      loaded_config.temp_nrepl_port = nrepl_port;
+      return loaded_config;
     } catch (std::exception& e) {
-      REPL::Config config(game_version);
+      // do nothing
     }
   }
-  return REPL::Config(game_version);
+  loaded_config.temp_nrepl_port = nrepl_port;
+  return loaded_config;
 }
 }  // namespace REPL

common/repl/repl_wrapper.h

@@ -17,18 +17,20 @@ struct StartupFile {
 };
 
 class Wrapper {
-  replxx::Replxx repl;
-
  public:
   std::string username;
   Config repl_config;
   StartupFile startup_file;
+  bool nrepl_alive = false;
   std::vector<std::string> examples{};
   std::vector<std::pair<std::string, replxx::Replxx::Color>> regex_colors{};
 
   Wrapper(GameVersion version) : repl_config(version) {}
-  Wrapper(const std::string& _username, const Config& config, const StartupFile& startup)
-      : username(_username), repl_config(config), startup_file(startup) {}
+  Wrapper(const std::string& _username,
+          const Config& config,
+          const StartupFile& startup,
+          bool nrepl_alive)
+      : username(_username), repl_config(config), startup_file(startup), nrepl_alive(nrepl_alive) {}
   replxx::Replxx& get_repl() { return repl; }
   void init_settings();
   void reload_startup_file();
@@ -36,7 +38,7 @@ class Wrapper {
   // Functionality / Commands
   void clear_screen();
   void print_to_repl(const std::string& str);
-  void print_welcome_message();
+  void print_welcome_message(const std::vector<std::string>& loaded_projects);
   void set_history_max_size(size_t len);
   const char* readline(const std::string& prompt);
   void add_to_history(const std::string& line);
@@ -47,11 +49,14 @@ class Wrapper {
   std::pair<std::string, bool> get_current_repl_token(std::string const& context);
 
  private:
+  replxx::Replxx repl;
   replxx::Replxx::key_press_handler_t commit_text_action(std::string text_to_commit);
   std::vector<REPL::KeyBind> keybindings = {};
 };
 
 std::string find_repl_username();
 StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version);
-REPL::Config load_repl_config(const std::string& username, const GameVersion game_version);
+REPL::Config load_repl_config(const std::string& username,
+                              const GameVersion game_version,
+                              const int nrepl_port);
 }  // namespace REPL

common/serialization/subtitles/subtitles.cpp

@@ -5,7 +5,7 @@
 #include "common/util/FileUtil.h"
 #include "common/util/json_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 void open_subtitle_project(const std::string& project_kind,
                            const std::string& file_path,

common/serialization/subtitles/subtitles_v1.cpp

@@ -5,7 +5,7 @@
 
 #include "subtitles_v2.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 void to_json(json& j, const SubtitleCutsceneLineMetadataV1& obj) {
   json_serialize(frame_start);

common/serialization/subtitles/subtitles_v2.cpp

@@ -5,7 +5,7 @@
 
 #include "subtitles_v1.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 void to_json(json& j, const SubtitleLineMetadata& obj) {
   json_serialize(frame_start);
@@ -89,7 +89,9 @@ const std::unordered_map<std::string, u16> jak2_speaker_name_to_enum_val = {
     {"citizen-male", 31},
     {"citizen-female", 32},
     {"oracle", 33},
-    {"precursor", 34}};
+    {"precursor", 34},
+    {"metalkor-before-consite", 35},
+    {"metalkor-intro", 36}};
 
 GameSubtitlePackage read_json_files_v2(const GameSubtitleDefinitionFile& file_info) {
   GameSubtitlePackage package;

common/texture/texture_slots.cpp

@@ -42,7 +42,9 @@ std::vector<std::string> jak2_slots = {
     "cas-toxic-slime-scroll-dest",
 };
 
-std::vector<std::string> jak3_slots = {};
+std::vector<std::string> jak3_slots = {
+    "skull-gem-dest", "jakc-arm", "jakc-eyebrow", "jakc-face", "jakc-finger", "jakc-hair",
+};
 
 }  // namespace
 

common/type_system/Type.cpp

@@ -10,7 +10,7 @@
 #include "common/log/log.h"
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace {
 std::string reg_kind_to_string(RegClass kind) {

common/type_system/Type.h

@@ -28,8 +28,12 @@ struct MethodInfo {
   std::string name;
   TypeSpec type;
   std::string defined_in_type;
+  // TODO - this might be redundant, but preventing an unknown breaking change, clean it up later
+  std::string type_name;
   bool no_virtual = false;
+  // TODO - this is no longer used anymore...but was not removed from th4e struct?
   bool overrides_parent = false;
+  // TODO - this is no longer used anymore...but was not removed from th4e struct?
   bool only_overrides_docstring = false;
   std::optional<std::string> docstring;
   std::optional<std::string> overlay_name;

common/type_system/TypeFieldLookup.cpp

@@ -9,7 +9,7 @@
 
 #include "common/log/log.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace {
 // debug prints for the reverse lookup

common/type_system/TypeSpec.cpp

@@ -7,7 +7,7 @@
 
 #include <stdexcept>
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 bool TypeTag::operator==(const TypeTag& other) const {
   return name == other.name && value == other.value;

common/type_system/TypeSpec.h

@@ -124,10 +124,12 @@ class TypeSpec {
     ASSERT(m_arguments);
     return m_arguments->at(idx);
   }
+
   TypeSpec& get_arg(int idx) {
     ASSERT(m_arguments);
     return m_arguments->at(idx);
   }
+
   const TypeSpec& last_arg() const {
     ASSERT(m_arguments);
     ASSERT(!m_arguments->empty());

common/type_system/TypeSystem.cpp

@@ -14,8 +14,8 @@
 #include "common/util/Assert.h"
 #include "common/util/math_util.h"
 
-#include "third-party/fmt/color.h"
-#include "third-party/fmt/core.h"
+#include "fmt/color.h"
+#include "fmt/core.h"
 
 namespace {
 template <typename... Args>
@@ -548,6 +548,7 @@ MethodInfo TypeSystem::override_method(Type* type,
                            existing_info.name,
                            existing_info.type,
                            type->get_name(),
+                           type->get_name(),
                            existing_info.no_virtual,
                            false,
                            true,
@@ -607,6 +608,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
                              method_name,
                              ts,
                              type->get_name(),
+                             type->get_name(),
                              no_virtual,
                              true,
                              false,
@@ -645,6 +647,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
                                method_name,
                                ts,
                                type->get_name(),
+                               type->get_name(),
                                no_virtual,
                                false,
                                false,
@@ -678,8 +681,8 @@ MethodInfo TypeSystem::overlay_method(Type* type,
   }
 
   // use the existing ID.
-  return type->add_method({existing_info.id, method_name, ts, type->get_name(), false, true, false,
-                           docstring, std::make_optional(method_overlay_name)});
+  return type->add_method({existing_info.id, method_name, ts, type->get_name(), type->get_name(),
+                           false, true, false, docstring, std::make_optional(method_overlay_name)});
 }
 
 MethodInfo TypeSystem::define_method(const std::string& type_name,
@@ -709,8 +712,10 @@ MethodInfo TypeSystem::define_method(Type* type,
   // look up the method
   MethodInfo existing_info;
   bool got_existing = try_lookup_method(type, method_name, &existing_info);
-
   if (got_existing) {
+    // The lookup will return a parents method, but the type should be equal
+    // to the type in question (it's a child)
+    existing_info.type_name = type->get_name();
     // Update the docstring
     existing_info.docstring = docstring;
     int bad_arg_idx = -99;
@@ -758,7 +763,7 @@ MethodInfo TypeSystem::add_new_method(Type* type,
     return existing;
   } else {
     return type->add_new_method(
-        {0, "new", ts, type->get_name(), false, false, false, docstring, {}});
+        {0, "new", ts, type->get_name(), type->get_name(), false, false, false, docstring, {}});
   }
 }
 
@@ -1442,6 +1447,21 @@ std::vector<std::string> TypeSystem::search_types_by_parent_type(
   return results;
 }
 
+std::vector<std::string> TypeSystem::search_types_by_parent_type_strict(
+    const std::string& parent_type) {
+  std::vector<std::string> results = {};
+  for (const auto& [type_name, type_info] : m_types) {
+    // Only NullType's have no parent
+    if (!type_info->has_parent()) {
+      continue;
+    }
+    if (type_info->get_parent() == parent_type) {
+      results.push_back(type_name);
+    }
+  }
+  return results;
+}
+
 std::vector<std::string> TypeSystem::search_types_by_minimum_method_id(
     const int minimum_method_id,
     const std::optional<std::vector<std::string>>& existing_matches) {

common/type_system/TypeSystem.h

@@ -278,6 +278,7 @@ class TypeSystem {
   std::vector<std::string> search_types_by_parent_type(
       const std::string& parent_type,
       const std::optional<std::vector<std::string>>& existing_matches = {});
+  std::vector<std::string> search_types_by_parent_type_strict(const std::string& parent_type);
 
   std::vector<std::string> search_types_by_minimum_method_id(
       const int minimum_method_id,

common/type_system/defenum.cpp

@@ -13,7 +13,7 @@
 #include "common/util/BitUtils.h"
 #include "common/util/string_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 namespace {
 const goos::Object& car(const goos::Object* x) {
@@ -58,10 +58,11 @@ EnumType* parse_defenum(const goos::Object& defenum,
   auto& enum_name_obj = car(iter);
   iter = cdr(iter);
   // check for docstring
+  std::optional<std::string> maybe_docstring;
   if (iter->is_pair() && car(iter).is_string()) {
-    // TODO - docstring - store and use docstring if coming from the compiler
     if (symbol_metadata) {
-      symbol_metadata->docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
+      maybe_docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
+      symbol_metadata->docstring = maybe_docstring.value();
     }
     iter = cdr(iter);
   }
@@ -157,6 +158,7 @@ EnumType* parse_defenum(const goos::Object& defenum,
   if (is_type("integer", base_type, ts)) {
     auto parent = ts->get_type_of_type<ValueType>(base_type.base_type());
     auto new_type = std::make_unique<EnumType>(parent, name, is_bitfield, entries);
+    new_type->m_metadata.docstring = maybe_docstring;
     new_type->set_runtime_type(parent->get_runtime_name());
     return dynamic_cast<EnumType*>(ts->add_type(name, std::move(new_type)));
   } else {

common/type_system/deftype.cpp

@@ -13,7 +13,7 @@
 #include "common/type_system/state.h"
 #include "common/util/string_util.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 /*!
  * Missing Features
@@ -84,12 +84,10 @@ double get_float(const goos::Object& obj) {
   throw std::runtime_error(obj.print() + " was supposed to be an number, but isn't");
 }
 
-void add_field(
-    StructureType* structure,
-    TypeSystem* ts,
-    const goos::Object& def,
-    std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>&
-        constants) {
+void add_field(StructureType* structure,
+               TypeSystem* ts,
+               const goos::Object& def,
+               goos::EnvironmentMap* constants) {
   auto rest = &def;
 
   auto name = symbol_string(car(rest));
@@ -113,9 +111,8 @@ void add_field(
     if (car(rest).is_int()) {
       array_size = car(rest).integer_obj.value;
       rest = cdr(rest);
-    } else if (car(rest).is_symbol() &&
-               constants.find((car(rest)).as_symbol()) != constants.end()) {
-      array_size = get_int(constants[(car(rest)).as_symbol()]);
+    } else if (car(rest).is_symbol() && constants && constants->lookup((car(rest)).as_symbol())) {
+      array_size = get_int(*constants->lookup((car(rest)).as_symbol()));
       rest = cdr(rest);
     }
 
@@ -543,13 +540,11 @@ void declare_state(Type* type,
   });
 }
 
-StructureDefResult parse_structure_def(
-    StructureType* type,
-    TypeSystem* ts,
-    const goos::Object& fields,
-    const goos::Object& options,
-    std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>&
-        constants) {
+StructureDefResult parse_structure_def(StructureType* type,
+                                       TypeSystem* ts,
+                                       const goos::Object& fields,
+                                       const goos::Object& options,
+                                       goos::EnvironmentMap* constants) {
   StructureDefResult result;
   for_each_in_list(fields, [&](const goos::Object& o) { add_field(type, ts, o, constants); });
   TypeFlags flags;
@@ -811,18 +806,11 @@ TypeSpec parse_typespec(const TypeSystem* type_system, const goos::Object& src)
   return {};
 }
 
-DeftypeResult parse_deftype(
-    const goos::Object& deftype,
-    TypeSystem* ts,
-    std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>*
-        constants) {
+DeftypeResult parse_deftype(const goos::Object& deftype,
+                            TypeSystem* ts,
+                            goos::EnvironmentMap* constants) {
   DefinitionMetadata symbol_metadata;
-  std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>
-      no_consts;
-  auto& constants_to_use = no_consts;
-  if (constants != nullptr) {
-    constants_to_use = *constants;
-  }
+  goos::EnvironmentMap no_consts;
 
   auto iter = &deftype;
 
@@ -861,8 +849,7 @@ DeftypeResult parse_deftype(
     }
     new_type->inherit(pto);
     ts->forward_declare_type_as(name, pto->get_name());
-    auto sr =
-        parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
+    auto sr = parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants);
     result.flags = sr.flags;
     result.create_runtime_type = sr.generate_runtime_type;
     structure_result = sr;
@@ -891,8 +878,7 @@ DeftypeResult parse_deftype(
     ASSERT(pto);
     new_type->inherit(pto);
     ts->forward_declare_type_as(name, pto->get_name());
-    auto sr =
-        parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
+    auto sr = parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants);
     result.flags = sr.flags;
     result.create_runtime_type = sr.generate_runtime_type;
     structure_result = sr;

common/type_system/deftype.h

@@ -17,9 +17,7 @@ struct DeftypeResult {
   bool create_runtime_type = true;
 };
 
-DeftypeResult parse_deftype(
-    const goos::Object& deftype,
-    TypeSystem* ts,
-    std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>*
-        constants = nullptr);
+DeftypeResult parse_deftype(const goos::Object& deftype,
+                            TypeSystem* ts,
+                            goos::EnvironmentMap* constants = nullptr);
 TypeSpec parse_typespec(const TypeSystem* type_system, const goos::Object& src);

common/util/Assert.h

@@ -36,6 +36,9 @@
 
 #define ASSERT_NOT_REACHED_MSG(STR) \
   (void)((private_assert_failed("not reached", __FILE__, __LINE__, __PRETTY_FUNCTION__, STR), 0))
+
+#define ASSERT_EQ_IMM(EXPR, EXPECTED) \
+  ASSERT_MSG((EXPR) == (EXPECTED), fmt::format("result was {}, expected {}", (EXPR), (EXPECTED)))
 #else
 
 #define ASSERT(EX) ((void)0)

common/util/FileUtil.cpp

@@ -22,7 +22,7 @@
 // This disables the use of PCLMULQDQ which is probably ok, but let's just be safe and disable it
 // because nobody will care if png compression is 10% slower.
 #define FPNG_NO_SSE 1
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 #include "third-party/fpng/fpng.cpp"
 #include "third-party/fpng/fpng.h"
 #include "third-party/lzokay/lzokay.hpp"
@@ -57,6 +57,14 @@ fs::path get_user_home_dir() {
 #endif
 }
 
+struct {
+  bool initialized = false;
+  fs::path path_to_data_folder;
+  fs::path user_config_dir_override = "";
+  // by default - if the config dir is overridden, we don't use the default save location
+  bool use_overridden_config_dir_for_saves = true;
+} g_file_path_info;
+
 fs::path get_user_config_dir() {
   fs::path config_base_path;
 #ifdef _WIN32
@@ -80,35 +88,53 @@ fs::path get_user_config_dir() {
 
 fs::path get_user_settings_dir(GameVersion game_version) {
   auto game_version_name = game_version_names[game_version];
-  return get_user_config_dir() / game_version_name / "settings";
+  auto config_dir = get_user_config_dir();
+  if (!g_file_path_info.user_config_dir_override.empty()) {
+    config_dir = g_file_path_info.user_config_dir_override / "OpenGOAL";
+  }
+  return config_dir / game_version_name / "settings";
 }
 
 fs::path get_user_memcard_dir(GameVersion game_version) {
   auto game_version_name = game_version_names[game_version];
-  return get_user_config_dir() / game_version_name / "saves";
+  auto config_dir = get_user_config_dir();
+  if (!g_file_path_info.user_config_dir_override.empty() &&
+      g_file_path_info.use_overridden_config_dir_for_saves) {
+    config_dir = g_file_path_info.user_config_dir_override / "OpenGOAL";
+  }
+  return config_dir / game_version_name / "saves";
 }
 
 fs::path get_user_screenshots_dir(GameVersion game_version) {
   auto game_version_name = game_version_names[game_version];
-  return get_user_config_dir() / game_version_name / "screenshots";
+  auto config_dir = get_user_config_dir();
+  if (!g_file_path_info.user_config_dir_override.empty()) {
+    config_dir = g_file_path_info.user_config_dir_override / "OpenGOAL";
+  }
+  return config_dir / game_version_name / "screenshots";
 }
 
 fs::path get_user_misc_dir(GameVersion game_version) {
   auto game_version_name = game_version_names[game_version];
-  return get_user_config_dir() / game_version_name / "misc";
+  auto config_dir = get_user_config_dir();
+  if (!g_file_path_info.user_config_dir_override.empty()) {
+    config_dir = g_file_path_info.user_config_dir_override / "OpenGOAL";
+  }
+  return config_dir / game_version_name / "misc";
 }
 
 fs::path get_user_features_dir(GameVersion game_version) {
   auto game_version_name = game_version_names[game_version];
-  auto path = get_user_config_dir() / game_version_name / "features";
+  auto config_dir = get_user_config_dir();
+  if (!g_file_path_info.user_config_dir_override.empty()) {
+    config_dir = g_file_path_info.user_config_dir_override / "OpenGOAL";
+  }
+  auto path = config_dir / game_version_name / "features";
   file_util::create_dir_if_needed(path);
   return path;
 }
 
-struct {
-  bool initialized = false;
-  fs::path path_to_data;
-} gFilePathInfo;
+fs::path g_iso_data_directory = "";
 
 /*!
  * Get the path to the current executable.
@@ -159,9 +185,11 @@ std::optional<std::string> try_get_jak_project_path() {
   return try_get_project_path_from_path(get_current_executable_path());
 }
 
-std::optional<fs::path> try_get_data_dir() {
+std::optional<fs::path> try_get_data_dir(bool skip_logs) {
   fs::path my_path = get_current_executable_path();
-  lg::info("Current executable directory - {}", my_path.string());
+  if (!skip_logs) {
+    lg::debug("Current executable directory - {}", my_path.string());
+  }
   auto data_dir = my_path.parent_path() / "data";
   if (fs::exists(data_dir) && fs::is_directory(data_dir)) {
     return std::make_optional(data_dir);
@@ -170,31 +198,38 @@ std::optional<fs::path> try_get_data_dir() {
   }
 }
 
-bool setup_project_path(std::optional<fs::path> project_path_override) {
-  if (gFilePathInfo.initialized) {
+bool setup_project_path(std::optional<fs::path> project_path_override, bool skip_logs) {
+  if (g_file_path_info.initialized) {
     return true;
   }
 
   if (project_path_override) {
-    gFilePathInfo.path_to_data = fs::absolute(project_path_override.value());
-    gFilePathInfo.initialized = true;
-    lg::info("Using explicitly set project path: {}", gFilePathInfo.path_to_data.string());
+    g_file_path_info.path_to_data_folder = fs::absolute(project_path_override.value());
+    g_file_path_info.initialized = true;
+    if (!skip_logs) {
+      lg::debug("Using explicitly set project path: {}",
+                g_file_path_info.path_to_data_folder.string());
+    }
     return true;
   }
 
-  auto data_path = try_get_data_dir();
+  auto data_path = try_get_data_dir(skip_logs);
   if (data_path) {
-    gFilePathInfo.path_to_data = *data_path;
-    gFilePathInfo.initialized = true;
-    lg::info("Using data path: {}", data_path->string());
+    g_file_path_info.path_to_data_folder = *data_path;
+    g_file_path_info.initialized = true;
+    if (!skip_logs) {
+      lg::debug("Using data path: {}", data_path->string());
+    }
     return true;
   }
 
   auto development_repo_path = try_get_jak_project_path();
   if (development_repo_path) {
-    gFilePathInfo.path_to_data = *development_repo_path;
-    gFilePathInfo.initialized = true;
-    lg::info("Using development repo path: {}", *development_repo_path);
+    g_file_path_info.path_to_data_folder = *development_repo_path;
+    g_file_path_info.initialized = true;
+    if (!skip_logs) {
+      lg::debug("Using development repo path: {}", *development_repo_path);
+    }
     return true;
   }
 
@@ -202,9 +237,37 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
   return false;
 }
 
+void override_user_config_dir(fs::path user_config_dir_override,
+                              bool use_overridden_config_dir_for_saves) {
+  g_file_path_info.user_config_dir_override = user_config_dir_override;
+  g_file_path_info.use_overridden_config_dir_for_saves = use_overridden_config_dir_for_saves;
+}
+
 fs::path get_jak_project_dir() {
-  ASSERT(gFilePathInfo.initialized);
-  return gFilePathInfo.path_to_data;
+  ASSERT(g_file_path_info.initialized);
+  return g_file_path_info.path_to_data_folder;
+}
+
+fs::path get_iso_dir_for_game(GameVersion game_version) {
+  if (!g_iso_data_directory.empty()) {
+    return g_iso_data_directory;
+  }
+  // Find the location based on the game version
+  std::string expected_subdir = "jak1";
+  if (game_version == GameVersion::Jak2) {
+    expected_subdir = "jak2";
+  } else if (game_version == GameVersion::Jak3) {
+    expected_subdir = "jak3";
+  }
+  const auto temp_dir = get_jak_project_dir() / "iso_data" / expected_subdir;
+  if (fs::exists(temp_dir)) {
+    g_iso_data_directory = temp_dir;
+  }
+  return g_iso_data_directory;
+}
+
+void set_iso_data_dir(const fs::path& directory) {
+  g_iso_data_directory = directory;
 }
 
 std::string get_file_path(const std::vector<std::string>& input) {
@@ -736,4 +799,30 @@ std::string get_majority_file_line_endings(const std::string& file_contents) {
   return "\n";
 }
 
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents) {
+  size_t lf_count = 0;
+  size_t crlf_count = 0;
+
+  for (size_t i = 0; i < file_contents.size(); ++i) {
+    if (file_contents[i] == '\n') {
+      if (i > 0 && file_contents[i - 1] == '\r') {
+        crlf_count++;
+      } else {
+        lf_count++;
+      }
+    }
+  }
+
+  if (crlf_count > lf_count) {
+    return {lf_count + crlf_count, "\r\n"};
+  }
+  return {lf_count + crlf_count, "\n"};
+}
+
+bool is_dir_in_dir(const fs::path& parent, const fs::path& child) {
+  // Check if the parent path is a prefix of the child path
+  return child.has_parent_path() && child.parent_path().lexically_relative(parent) == fs::path(".");
+}
+
 }  // namespace file_util

common/util/FileUtil.h

@@ -35,12 +35,17 @@ fs::path get_user_screenshots_dir(GameVersion game_version);
 fs::path get_user_misc_dir(GameVersion game_version);
 fs::path get_user_features_dir(GameVersion game_version);
 fs::path get_jak_project_dir();
+fs::path get_iso_dir_for_game(GameVersion game_version);
+void set_iso_data_dir(const fs::path& directory);
 
 bool create_dir_if_needed(const fs::path& path);
 bool create_dir_if_needed_for_file(const std::string& path);
 bool create_dir_if_needed_for_file(const fs::path& path);
+std::string get_current_executable_path();
 std::optional<std::string> try_get_project_path_from_path(const std::string& path);
-bool setup_project_path(std::optional<fs::path> project_path_override);
+bool setup_project_path(std::optional<fs::path> project_path_override, bool skip_logs = false);
+void override_user_config_dir(fs::path user_config_dir_override,
+                              bool use_overridden_config_dir_for_saves);
 std::string get_file_path(const std::vector<std::string>& path);
 void write_binary_file(const std::string& name, const void* data, size_t size);
 void write_binary_file(const fs::path& name, const void* data, size_t size);
@@ -72,4 +77,7 @@ std::vector<fs::path> sort_filepaths(const std::vector<fs::path>& paths, const b
 void copy_file(const fs::path& src, const fs::path& dst);
 std::string make_screenshot_filepath(const GameVersion game_version, const std::string& name = "");
 std::string get_majority_file_line_endings(const std::string& file_contents);
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents);
+bool is_dir_in_dir(const fs::path& parent, const fs::path& child);
 }  // namespace file_util

common/util/FontUtils.cpp

@@ -16,8 +16,8 @@
 
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/format.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
 
 const std::unordered_map<std::string, GameTextVersion> sTextVerEnumMap = {
     {"jak1-v1", GameTextVersion::JAK1_V1},
@@ -985,6 +985,8 @@ static std::vector<ReplaceInfo> s_replace_info_jak2 = {
     {"o~Y~-26H~-4V'~Z", "ó"},
     {"U~Y~-24H~-3V'~Z", "Ú"},
     {"u~Y~-24H~-3V'~Z", "ú"},
+    {"Z~Y~-24H~-3V'~Z", "Ź"},
+    {"z~Y~-24H~-3V'~Z", "ź"},
 
     // circumflex
     {"A~Y~-20H~-4V^~Z", "Â"},
@@ -1015,6 +1017,7 @@ static std::vector<ReplaceInfo> s_replace_info_jak2 = {
     {"a~Y~-25H~-5V¨~Z", "ä"},
     {"E~Y~-20H~-5V¨~Z", "Ë"},
     {"I~Y~-19H~-5V¨~Z", "Ï"},
+    {"i~Y~-26H~-4V¨~Z", "ï"},
     {"O~Y~-26H~-8V¨~Z", "Ö"},
     {"o~Y~-26H~-4V¨~Z", "ö"},
     {"U~Y~-25H~-8V¨~Z", "Ü"},

common/util/Range.h

@@ -67,4 +67,4 @@ class Range {
  private:
   T m_start = {};
   T m_end = {};
-};
+};

common/util/Trie.h

@@ -12,7 +12,7 @@
  * It owns the memory for the objects it stores.
  * Doing an insert will create a copy of your object.
  *
- * Other that deleting the whole thing, there is no support for removing a node.
+ * Other than deleting the whole thing, there is no support for removing a node.
  */
 template <typename T>
 class Trie {
@@ -187,4 +187,4 @@ std::vector<T*> Trie<T>::get_all_nodes() const {
   std::vector<T*> result;
   m_root.get_all_children(result);
   return result;
-}
+}

common/util/ast_util.cpp

@@ -0,0 +1,31 @@
+#include "ast_util.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node) {
+  uint32_t start = ts_node_start_byte(node);
+  uint32_t end = ts_node_end_byte(node);
+  return source.substr(start, end - start);
+}
+
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results) {
+  if (ts_node_child_count(curr_node) == 0) {
+    return;
+  }
+  std::vector<std::string> node_elements;
+  bool added = false;
+  for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
+    const auto child_node = ts_node_child(curr_node, i);
+    const auto contents = get_source_code(source, child_node);
+    node_elements.push_back(contents);
+    // Check for a match
+    if (node_elements == prefix && !added) {
+      results.push_back(curr_node);
+      added = true;
+    }
+    search_for_forms_that_begin_with(source, child_node, prefix, results);
+  }
+}
+}  // namespace ast_util

common/util/ast_util.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "tree_sitter/api.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node);
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results);
+
+}  // namespace ast_util

common/util/compress.cpp

@@ -5,7 +5,7 @@
 
 #include "common/util/Assert.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 #include "third-party/zstd/lib/zstd.h"
 
 namespace compression {

common/util/crc32.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <cstdlib>
 
 #include "common/common_types.h"

common/util/dgo_util.cpp

@@ -5,7 +5,7 @@
 #include "common/util/Assert.h"
 #include "common/versions/versions.h"
 
-#include "third-party/fmt/core.h"
+#include "fmt/core.h"
 
 /*!
  * Assert false if the char[] has non-null data after the null terminated string.

common/util/gltf_util.cpp

@@ -0,0 +1,467 @@
+#include "gltf_util.h"
+
+#include "common/log/log.h"
+
+namespace gltf_util {
+
+/*!
+ * Convert a GLTF position buffer or similar to std::vector<Vec3f>
+ */
+std::vector<math::Vector3f> extract_vec3f(const u8* data, u32 count, u32 stride) {
+  std::vector<math::Vector3f> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    memcpy(&result.emplace_back(), data, sizeof(math::Vector3f));
+    data += stride;
+  }
+  return result;
+}
+
+std::vector<math::Vector2f> extract_vec2f(const u8* data, u32 count, u32 stride) {
+  std::vector<math::Vector2f> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    memcpy(&result.emplace_back(), data, sizeof(math::Vector2f));
+    data += stride;
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF color buffer (float format) to u8 colors.
+ */
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_float(const u8* data,
+                                                               u32 count,
+                                                               u32 stride) {
+  std::vector<math::Vector<u8, 4>> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    math::Vector<float, 4> temp;
+    memcpy(&temp, data, sizeof(math::Vector<float, 4>));
+    data += stride;
+    result.emplace_back(temp.x() * 255, temp.y() * 255, temp.z() * 255, temp.w() * 255);
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF color buffer (u16 format) to u8 colors.
+ */
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_u16(const u8* data,
+                                                             u32 count,
+                                                             u32 stride) {
+  std::vector<math::Vector<u8, 4>> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    math::Vector<u16, 4> temp;
+    memcpy(&temp, data, sizeof(math::Vector<u16, 4>));
+    data += stride;
+    result.emplace_back(temp.x() >> 8, temp.y() >> 8, temp.z() >> 8, temp.w() >> 8);
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF index buffer
+ */
+std::vector<u32> gltf_index_buffer(const tinygltf::Model& model,
+                                   int indices_idx,
+                                   u32 index_offset) {
+  const auto& indices_accessor = model.accessors[indices_idx];
+  const auto& buffer_view = model.bufferViews[indices_accessor.bufferView];
+  const auto& buffer = model.buffers[buffer_view.buffer];
+  const auto data_ptr = buffer.data.data() + buffer_view.byteOffset + indices_accessor.byteOffset;
+  const auto stride = indices_accessor.ByteStride(buffer_view);
+  const auto count = indices_accessor.count;
+
+  switch (indices_accessor.componentType) {
+    case TINYGLTF_COMPONENT_TYPE_BYTE:
+      return index_list_to_u32<s8>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE:
+      return index_list_to_u32<u8>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_SHORT:
+      return index_list_to_u32<s16>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT:
+      return index_list_to_u32<u16>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_INT:
+      return index_list_to_u32<s32>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT:
+      return index_list_to_u32<u32>(data_ptr, count, index_offset, stride);
+    default:
+      ASSERT_MSG(false, "unsupported component type");
+  }
+}
+
+/*!
+ * Extract positions, colors, and normals from a mesh.
+ */
+ExtractedVertices gltf_vertices(const tinygltf::Model& model,
+                                const std::map<std::string, int>& attributes,
+                                const math::Matrix4f& w_T_local,
+                                bool get_colors,
+                                bool get_normals,
+                                const std::string& debug_name) {
+  std::vector<tfrag3::PreloadedVertex> result;
+  std::vector<math::Vector<u8, 4>> vtx_colors;
+
+  {
+    const auto& position_attrib = attributes.find("POSITION");
+    ASSERT_MSG(position_attrib != attributes.end(), "Did not find position attribute.");
+
+    const auto attrib_accessor = model.accessors[position_attrib->second];
+    const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+    const auto& buffer = model.buffers[buffer_view.buffer];
+    const auto data_ptr = buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+    const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+    const auto count = attrib_accessor.count;
+
+    ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC3, "POSITION wasn't vec3");
+    ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+               "POSITION wasn't float");
+    // for (auto& attrib : attributes) {
+    // lg::print("attrib: {}\n", attrib.first);
+    //}
+    auto mesh_verts = extract_vec3f(data_ptr, count, byte_stride);
+    result.reserve(mesh_verts.size());
+    for (auto& vert : mesh_verts) {
+      auto& new_vert = result.emplace_back();
+      math::Vector4f v_in(vert.x(), vert.y(), vert.z(), 1);
+      math::Vector4f v_w = w_T_local * v_in;
+      new_vert.x = v_w.x() * 4096;
+      new_vert.y = v_w.y() * 4096;
+      new_vert.z = v_w.z() * 4096;
+    }
+  }
+
+  if (get_colors) {
+    const auto& color_attrib = attributes.find("COLOR_0");
+    if (color_attrib == attributes.end()) {
+      lg::error("Mesh {} didn't have any colors, using white", debug_name);
+      for (size_t i = 0; i < result.size(); i++) {
+        vtx_colors.emplace_back(0x80, 0x80, 0x80, 0xff);
+      }
+    } else {
+      const auto attrib_accessor = model.accessors[color_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC4, "COLOR_0 wasn't vec4");
+      std::vector<math::Vector<u8, 4>> colors;
+      switch (attrib_accessor.componentType) {
+        case TINYGLTF_COMPONENT_TYPE_FLOAT:
+          colors = extract_color_from_vec4_float(data_ptr, count, byte_stride);
+          break;
+        case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT:
+          colors = extract_color_from_vec4_u16(data_ptr, count, byte_stride);
+          break;
+        default:
+          lg::die("Unknown component type for COLOR_0: {}", attrib_accessor.componentType);
+      }
+      vtx_colors.insert(vtx_colors.end(), colors.begin(), colors.end());
+    }
+  }
+
+  bool got_texture = false;
+  {
+    const auto& texcoord_attrib = attributes.find("TEXCOORD_0");
+    if (texcoord_attrib != attributes.end()) {
+      const auto attrib_accessor = model.accessors[texcoord_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC2, "TEXCOORD wasn't vec2");
+      ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+                 "TEXCOORD wasn't float");
+      auto mesh_verts = extract_vec2f(data_ptr, count, byte_stride);
+      ASSERT(mesh_verts.size() == result.size());
+      got_texture = true;
+      for (size_t i = 0; i < mesh_verts.size(); i++) {
+        result[i].s = mesh_verts[i].x();
+        result[i].t = mesh_verts[i].y();
+      }
+    } else {
+      if (!get_normals) {
+        // don't warn if we're just getting collision
+        lg::warn("No texcoord attribute for mesh: {}", debug_name);
+      }
+    }
+  }
+
+  std::vector<math::Vector3f> normals;
+  if (get_normals) {
+    const auto& normal_attrib = attributes.find("NORMAL");
+    if (normal_attrib != attributes.end()) {
+      const auto attrib_accessor = model.accessors[normal_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC3, "NORMAL wasn't vec3");
+      ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+                 "NORMAL wasn't float");
+      normals = extract_vec3f(data_ptr, count, byte_stride);
+      for (auto& nrm : normals) {
+        math::Vector4f nrm4(nrm.x(), nrm.y(), nrm.z(), 0.f);
+        nrm = (w_T_local * nrm4).xyz();
+      }
+      ASSERT(normals.size() == result.size());
+    } else {
+      lg::error("No NORMAL attribute for mesh: {}", debug_name);
+    }
+  }
+
+  for (auto& v : result) {
+    v.color_index = 0;
+    if (!got_texture) {
+      v.s = 0;
+      v.t = 0;
+    }
+  }
+  // TODO: other properties
+  return {result, vtx_colors, normals};
+}
+
+DrawMode make_default_draw_mode() {
+  DrawMode mode;
+  mode.set_depth_write_enable(true);
+  mode.set_depth_test(GsTest::ZTest::GEQUAL);
+  mode.set_alpha_blend(DrawMode::AlphaBlend::DISABLED);
+  mode.set_aref(0);
+  mode.set_alpha_fail(GsTest::AlphaFail::KEEP);
+  mode.set_clamp_s_enable(false);
+  mode.set_clamp_t_enable(false);
+  mode.disable_filt();  // for checkerboard...
+  mode.enable_tcc();    // ?
+  mode.disable_at();
+  mode.enable_zt();
+  mode.disable_ab();
+  mode.disable_decal();
+  mode.enable_fog();
+  return mode;
+}
+
+int texture_pool_debug_checker(TexturePool* pool) {
+  const auto& existing = pool->textures_by_name.find("DEBUG_CHECKERBOARD");
+  if (existing == pool->textures_by_name.end()) {
+    size_t idx = pool->textures_by_idx.size();
+    pool->textures_by_name["DEBUG_CHECKERBOARD"] = idx;
+    auto& tex = pool->textures_by_idx.emplace_back();
+    tex.w = 16;
+    tex.h = 16;
+    tex.debug_name = "DEBUG_CHECKERBOARD";
+    tex.debug_tpage_name = "DEBUG";
+    tex.load_to_pool = false;
+    tex.combo_id = 0;  // doesn't matter, not a pool tex
+    tex.data.resize(16 * 16);
+    u32 c0 = 0xa0303030;
+    u32 c1 = 0xa0e0e0e0;
+    for (int i = 0; i < 16; i++) {
+      for (int j = 0; j < 16; j++) {
+        tex.data[i * 16 + j] = (((i / 4) & 1) ^ ((j / 4) & 1)) ? c1 : c0;
+      }
+    }
+    return idx;
+  } else {
+    return existing->second;
+  }
+}
+
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
+  const auto& existing = pool->textures_by_name.find(tex.name);
+  if (existing != pool->textures_by_name.end()) {
+    lg::info("Reusing image: {}", tex.name);
+    return existing->second;
+  } else {
+    lg::info("adding new texture: {}, size {} kB", tex.name, tex.width * tex.height * 4 / 1024);
+  }
+
+  ASSERT(tex.bits == 8);
+  ASSERT(tex.component == 4);
+  ASSERT(tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+
+  size_t idx = pool->textures_by_idx.size();
+  pool->textures_by_name[tex.name] = idx;
+  auto& tt = pool->textures_by_idx.emplace_back();
+  tt.w = tex.width;
+  tt.h = tex.height;
+  tt.debug_name = tex.name;
+  tt.debug_tpage_name = "custom-level";
+  tt.load_to_pool = false;
+  tt.combo_id = 0;  // doesn't matter, not a pool tex
+  tt.data.resize(tt.w * tt.h);
+  ASSERT(tex.image.size() >= tt.data.size());
+  memcpy(tt.data.data(), tex.image.data(), tt.data.size() * 4);
+  return idx;
+}
+
+math::Matrix4f affine_translation(const math::Vector3f& translation) {
+  math::Matrix4f result = math::Matrix4f::identity();
+  result(0, 3) = translation[0];
+  result(1, 3) = translation[1];
+  result(2, 3) = translation[2];
+  result(3, 3) = 1;
+  return result;
+}
+
+math::Matrix4f affine_scale(const math::Vector3f& scale) {
+  math::Matrix4f result = math::Matrix4f::zero();
+  result(0, 0) = scale[0];
+  result(1, 1) = scale[1];
+  result(2, 2) = scale[2];
+  result(3, 3) = 1;
+  return result;
+}
+
+math::Matrix4f affine_rot_qxyzw(const math::Vector4f& quat) {
+  math::Matrix4f result = math::Matrix4f::zero();
+  result(3, 3) = 1;
+  result(0, 0) = 1.0 - 2.0 * (quat.y() * quat.y() + quat.z() * quat.z());
+  result(0, 1) = 2.0 * (quat.x() * quat.y() - quat.z() * quat.w());
+  result(0, 2) = 2.0 * (quat.x() * quat.z() + quat.y() * quat.w());
+  result(1, 0) = 2.0 * (quat.x() * quat.y() + quat.z() * quat.w());
+  result(1, 1) = 1.0 - 2.0 * (quat.x() * quat.x() + quat.z() * quat.z());
+  result(1, 2) = 2.0 * (quat.y() * quat.z() - quat.x() * quat.w());
+  result(2, 0) = 2.0 * (quat.x() * quat.z() - quat.y() * quat.w());
+  result(2, 1) = 2.0 * (quat.y() * quat.z() + quat.x() * quat.w());
+  result(2, 2) = 1.0 - 2.0 * (quat.x() * quat.x() + quat.y() * quat.y());
+  return result;
+}
+
+math::Vector3f vector3f_from_gltf(const std::vector<double>& in) {
+  ASSERT(in.size() == 3);
+  return math::Vector3f{in[0], in[1], in[2]};
+}
+
+math::Vector4f vector4f_from_gltf(const std::vector<double>& in) {
+  ASSERT(in.size() == 4);
+  return math::Vector4f{in[0], in[1], in[2], in[3]};
+}
+
+math::Matrix4f matrix_from_node(const tinygltf::Node& node) {
+  if (!node.matrix.empty()) {
+    math::Matrix4f result;
+    for (int i = 0; i < 16; i++) {
+      result.data()[i] = node.matrix[i];
+    }
+    return result;
+  } else {
+    // from trs
+    math::Matrix4f t, r, s;
+    if (!node.translation.empty()) {
+      t = affine_translation(vector3f_from_gltf(node.translation));
+    } else {
+      t = math::Matrix4f::identity();
+    }
+
+    if (!node.rotation.empty()) {
+      r = affine_rot_qxyzw(vector4f_from_gltf(node.rotation));
+    } else {
+      r = math::Matrix4f::identity();
+    }
+
+    if (!node.scale.empty()) {
+      s = affine_scale(vector3f_from_gltf(node.scale));
+    } else {
+      s = math::Matrix4f::identity();
+    }
+
+    return t * r * s;
+  }
+}
+
+/*!
+ * Recursively walk the tree of nodes, flatten, and compute w_T_node for each.
+ */
+void node_find_helper(const tinygltf::Model& model,
+                      const math::Matrix4f& w_T_parent,
+                      int node_idx,
+                      std::vector<NodeWithTransform>* out) {
+  const auto& node = model.nodes.at(node_idx);
+  math::Matrix4f w_T_node = w_T_parent * matrix_from_node(node);
+  out->push_back({node_idx, w_T_node});
+  for (auto& child : node.children) {
+    node_find_helper(model, w_T_node, child, out);
+  }
+}
+
+std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Model& model) {
+  std::vector<NodeWithTransform> out;
+  for (auto& scene : model.scenes) {
+    for (auto& nidx : scene.nodes) {
+      math::Matrix4f identity = math::Matrix4f::identity();
+      node_find_helper(model, identity, nidx, &out);
+    }
+  }
+  return out;
+}
+
+void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
+                    std::vector<tfrag3::PreloadedVertex>& vertices_out,
+                    std::vector<u32>& old_to_new_out) {
+  ASSERT(vertices_out.empty());
+  ASSERT(old_to_new_out.empty());
+  old_to_new_out.resize(vertices_in.size(), -1);
+
+  std::unordered_map<tfrag3::PreloadedVertex, u32, tfrag3::PreloadedVertex::hash> vtx_to_new;
+
+  for (size_t in_idx = 0; in_idx < vertices_in.size(); in_idx++) {
+    auto& vtx = vertices_in[in_idx];
+    const auto& lookup = vtx_to_new.find(vtx);
+    if (lookup == vtx_to_new.end()) {
+      // first time seeing this one
+      size_t new_idx = vertices_out.size();
+      vertices_out.push_back(vtx);
+      old_to_new_out[in_idx] = new_idx;
+      vtx_to_new[vtx] = new_idx;
+    } else {
+      old_to_new_out[in_idx] = lookup->second;
+    }
+  }
+}
+
+DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler) {
+  DrawMode mode = make_default_draw_mode();
+  if (sampler.magFilter == TINYGLTF_TEXTURE_FILTER_NEAREST) {
+    ASSERT(sampler.minFilter == TINYGLTF_TEXTURE_FILTER_NEAREST);
+    mode.set_filt_enable(false);
+  } else {
+    ASSERT(sampler.minFilter != TINYGLTF_TEXTURE_FILTER_NEAREST);
+    mode.set_filt_enable(true);
+  }
+
+  switch (sampler.wrapS) {
+    case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
+      mode.set_clamp_s_enable(true);
+      break;
+    case TINYGLTF_TEXTURE_WRAP_REPEAT:
+      mode.set_clamp_s_enable(false);
+      break;
+    default:
+      ASSERT(false);
+  }
+
+  switch (sampler.wrapT) {
+    case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
+      mode.set_clamp_t_enable(true);
+      break;
+    case TINYGLTF_TEXTURE_WRAP_REPEAT:
+      mode.set_clamp_t_enable(false);
+      break;
+    default:
+      ASSERT(false);
+  }
+
+  return mode;
+}
+}  // namespace gltf_util

common/util/gltf_util.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/common_types.h"
+#include "common/custom_data/Tfrag3Data.h"
+#include "common/math/Vector.h"
+
+#include "third-party/tiny_gltf/tiny_gltf.h"
+
+namespace gltf_util {
+
+/*!
+ * Convert a GLTF index buffer to std::vector<u32>
+ */
+template <typename T>
+std::vector<u32> index_list_to_u32(const u8* data, u32 num_verts, u32 offset, u32 stride) {
+  std::vector<u32> result;
+  result.reserve(num_verts);
+  for (u32 i = 0; i < num_verts; i++) {
+    T val;
+    memcpy(&val, data, sizeof(T));
+    result.push_back(offset + val);
+    data += stride;
+  }
+  return result;
+}
+
+std::vector<math::Vector3f> extract_vec3f(const u8* data, u32 count, u32 stride);
+std::vector<math::Vector2f> extract_vec2f(const u8* data, u32 count, u32 stride);
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_u16(const u8* data, u32 count, u32 stride);
+std::vector<u32> gltf_index_buffer(const tinygltf::Model& model, int indices_idx, u32 index_offset);
+
+struct ExtractedVertices {
+  std::vector<tfrag3::PreloadedVertex> vtx;
+  std::vector<math::Vector<u8, 4>> vtx_colors;
+  std::vector<math::Vector3f> normals;
+};
+
+ExtractedVertices gltf_vertices(const tinygltf::Model& model,
+                                const std::map<std::string, int>& attributes,
+                                const math::Matrix4f& w_T_local,
+                                bool get_colors,
+                                bool get_normals,
+                                const std::string& debug_name);
+DrawMode make_default_draw_mode();
+
+struct TexturePool {
+  std::unordered_map<std::string, int> textures_by_name;
+  std::vector<tfrag3::Texture> textures_by_idx;
+};
+
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex);
+int texture_pool_debug_checker(TexturePool* pool);
+
+struct NodeWithTransform {
+  int node_idx;
+  math::Matrix4f w_T_node;
+};
+
+void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
+                    std::vector<tfrag3::PreloadedVertex>& vertices_out,
+                    std::vector<u32>& old_to_new_out);
+
+std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Model& model);
+
+DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler);
+
+}  // namespace gltf_util

common/util/os.cpp

@@ -3,20 +3,29 @@
 #include "common/common_types.h"
 #include "common/log/log.h"
 
-#ifdef __linux__
-
+#ifdef _WIN32
+// clang-format off
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <psapi.h>
+// clang-format on
+size_t get_peak_rss() {
+  HANDLE hProcess = GetCurrentProcess();
+  PROCESS_MEMORY_COUNTERS pmc;
+  if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) {
+    return pmc.PeakWorkingSetSize;
+  } else {
+    return 0;
+  }
+}
+#else
 #include <sys/resource.h>
-
 size_t get_peak_rss() {
   rusage x;
   getrusage(RUSAGE_SELF, &x);
   return x.ru_maxrss * 1024;
 }
-
-#else
-size_t get_peak_rss() {
-  return 0;
-}
 #endif
 
 #ifdef _WIN32

common/util/os.h

@@ -3,7 +3,6 @@
 #include <cstddef>
 #include <string>
 
-// Note: these are not implemented on windows and will return zero.
 size_t get_peak_rss();
 void setup_cpu_info();
 

common/util/print_float.cpp

@@ -5,8 +5,8 @@
 #include "common/goal_constants.h"
 #include "common/util/Assert.h"
 
+#include "fmt/core.h"
 #include "third-party/dragonbox.h"
-#include "third-party/fmt/core.h"
 
 /*!
  * Convert a float to a string. The string is _always_ in this format:
@@ -219,4 +219,4 @@ bool proper_float(float value) {
   } else {
     return true;
   }
-}
+}

common/util/string_util.cpp

@@ -247,7 +247,7 @@ std::string titlize(const std::string& str) {
 }
 
 std::string pad_right(const std::string& input, const int width, const char padding_char) {
-  if (input.length() >= width) {
+  if ((int)input.length() >= width) {
     return input;  // No need to pad if input length is already greater or equal to width
   } else {
     int padding_width = width - input.length();

common/util/trie_with_duplicates.h

@@ -0,0 +1,146 @@
+#pragma once
+
+#include <array>
+#include <memory>
+#include <string>
+#include <vector>
+
+// A normal Trie does not allow for duplicate keys, however this one does
+// It allows for insertion and removal
+template <typename T>
+class TrieWithDuplicates {
+ private:
+  struct TrieNode {
+    std::array<std::unique_ptr<TrieNode>, 256> children;
+    std::vector<std::unique_ptr<T>> elements;
+  };
+
+  std::unique_ptr<TrieNode> root = std::make_unique<TrieNode>();
+
+ public:
+  TrieWithDuplicates() {}
+
+  T* insert(const std::string& key, const T& element) {
+    std::unique_ptr<T> new_element = std::make_unique<T>(element);
+    TrieNode* curr_node = root.get();
+    for (const char character : key) {
+      auto& child = curr_node->children[(uint8_t)character];
+      if (!child) {
+        child = std::make_unique<TrieNode>();
+      }
+      curr_node = child.get();
+    }
+    curr_node->elements.push_back(std::move(new_element));
+    return curr_node->elements.back().get();
+  }
+
+  std::vector<T*> retrieve_with_prefix(const std::string& prefix, int max_count = -1) const {
+    std::vector<T*> results;
+    TrieNode* curr_node = root.get();
+    for (const char character : prefix) {
+      if (max_count >= 0 && (int)results.size() > max_count) {
+        return results;
+      }
+      const auto& child = curr_node->children.at((uint8_t)character);
+      if (child == nullptr) {
+        return results;  // tree ends, nothing found with that prefix
+      } else {
+        curr_node = child.get();
+      }
+    }
+    retrieve_elements(curr_node, results, max_count);
+    return results;
+  }
+
+  std::vector<T*> retrieve_with_exact(const std::string& key) const {
+    std::vector<T*> results;
+    TrieNode* curr_node = root.get();
+    for (const char character : key) {
+      const auto& child = curr_node->children.at((uint8_t)character);
+      if (child == nullptr) {
+        return results;  // tree ends, nothing found with that key
+      } else {
+        curr_node = child.get();
+      }
+    }
+    for (const auto& element : curr_node->elements) {
+      results.push_back(element.get());
+    }
+    return results;
+  }
+
+  bool remove(const std::string& key, const T* to_be_removed) {
+    TrieNode* curr_node = root.get();
+    for (const char character : key) {
+      const auto& child = curr_node->children.at((uint8_t)character);
+      if (child == nullptr) {
+        return false;  // tree ends, nothing found with that key
+      } else {
+        curr_node = child.get();
+      }
+    }
+    // Since the trie holds duplicates, we can't delete on the key alone
+    // now search to see which element is identical
+    auto it = curr_node->elements.begin();
+    while (it != curr_node->elements.end()) {
+      if (it->get() == to_be_removed) {
+        it = curr_node->elements.erase(it);
+        return true;  // we can assume that the same ptr isn't stored twice.
+      } else {
+        ++it;
+      }
+    }
+    return false;
+  }
+
+  // Return the total number of elements stored in the TrieMap
+  int size() const {
+    int count = 0;
+    count_elements(root.get(), count);
+    return count;
+  }
+
+  std::vector<T*> get_all_elements() const {
+    std::vector<T*> results;
+    get_all_elements_helper(root.get(), results);
+    return results;
+  }
+
+ private:
+  void retrieve_elements(const TrieNode* node, std::vector<T*>& results, int max_count = -1) const {
+    for (const auto& element : node->elements) {
+      if (max_count >= 0 && (int)results.size() > max_count) {
+        return;
+      }
+      results.push_back(element.get());
+    }
+    for (const auto& child : node->children) {
+      if (max_count >= 0 && (int)results.size() > max_count) {
+        return;
+      }
+      if (child.get() != nullptr) {
+        retrieve_elements(child.get(), results, max_count);
+      }
+    }
+  }
+
+  void count_elements(const TrieNode* node, int& count) const {
+    count += node->elements.size();
+    for (const auto& child : node->children) {
+      if (child.get() != nullptr) {
+        count_elements(child.get(), count);
+      }
+    }
+  }
+
+  void get_all_elements_helper(const TrieNode* node, std::vector<T*>& result) const {
+    for (const auto& element : node->elements) {
+      result.push_back(element.get());
+    }
+    for (const auto& child : node->children) {
+      if (child.get() != nullptr) {
+        get_all_elements_helper(child.get(), result);
+      }
+    }
+  }
+};

common/versions/versions.cpp

@@ -3,8 +3,8 @@
 #include "common/util/Assert.h"
 #include "common/versions/revision.h"
 
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/format.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
 
 GameVersion game_name_to_version(const std::string& name) {
   if (name == "jak1") {

custom_assets/README.md

@@ -5,13 +5,13 @@ Disclaimer: custom levels are still in development and are missing most features
 The first three steps are already done for "test zone", so this can be used as a starting point.
 
 # 1: File Setup
-To create a custom level, copy the layout of `custom_levels/test-zone`. See `test-zone.jsonc` for information on how to name things. The `.gd` file also contains the level name.
+To create a custom level, copy the layout of `custom_assets/jak1/levels/test-zone`. See `test-zone.jsonc` for information on how to name things. The `.gd` file also contains the level name.
 
 # 2: Modify the engine
-Modify `goal_src/engine/level/level-info.gc` to add level info for each custom level. There is level info for `test-zone` at the bottom that can be used as an example.
+Modify `goal_src/jak1/engine/level/level-info.gc` to add level info for each custom level. There is level info for `test-zone` at the bottom that can be used as an example.
 
 # 3: Modify the build system
-Modify `goal_src/game.gp` and add a custom level target:
+Modify `goal_src/jak1/game.gp` and add a custom level target:
 ```lisp
 (build-custom-level "test-zone")
 ;; the DGO file
@@ -21,7 +21,7 @@ Modify `goal_src/game.gp` and add a custom level target:
 # 4: Export the GLTF file from blender.
 For now, all meshes are displayed and treated as ground collision. This causes buggy collision because walls shouldn't use "floor" mode.
 
-Blender will create a `.glb` file, which must have the name specified in the `.jsonc` file and should be located in `custom_level/your_level`
+Blender will create a `.glb` file, which must have the name specified in the `.jsonc` file and should be located in `custom_assets/jak1/levels/your_level`
 
 # 5: Rebuild the game
 Any time the `.glb` file is changed, you must rebuild the game. Launch the compiler (`goalc`) and run `(mi)` to rebuild everything. It's recommended to leave the compiler open - it will remember files that haven't changed and skip rebuilding them.

custom_assets/jak1/levels/test-zone/test-zone.jsonc

@@ -10,7 +10,7 @@
   // Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
   // and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
   // only have 1.
-  "gltf_file": "custom_levels/jak1/test-zone/test-zone2.glb",
+  "gltf_file": "custom_assets/jak1/levels/test-zone/test-zone2.glb",
 
   // automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
   "automatic_wall_detection": true,
@@ -55,20 +55,22 @@
   // adds a 'symbol' tag (using the 'type' and 'string' lump types works the same way):
   // "symbol-list": ["symbol", "sym-1", "sym-2"]
 
-   // The base actor id for your custom level. If you have multiple levels this should be unique!
-   "base_id": 100,
+  // The base actor id for your custom level. If you have multiple levels, this should be unique!
+  "base_id": 100,
 
   // All art groups you want to use in your custom level. Will add their models and corresponding textures to the FR3 file.
   // Note: You will still have to add them to your level's .gd file.
-  // Removed so that the release builds don't have to double-decompile the game
   // "art_groups": ["plat-ag"],
-  "art_groups": [],
 
-   // Any textures you want to include in your custom level. This is mainly useful for things such as the zoomer HUD,
-   // which is not in the common level files and has no art group associated with it.
-   // To get a list of all the textures, you can extract all of the game's textures
-   // by setting "save_texture_pngs" to true in the decompiler config.
-   "textures": [
+  // If you have any custom models in the "custom_assets/jak1/models" folder that you want to use in your level, add them to this list.
+  // Note: Like with art groups, these should also be added to your level's .gd file.
+  "custom_models": ["test-actor"],
+
+  // Any textures you want to include in your custom level. This is mainly useful for things such as the zoomer HUD,
+  // which is not in the common level files and has no art group associated with it.
+  // To get a list of all the textures, you can extract all of the game's textures
+  // by setting "save_texture_pngs" to true in the decompiler config.
+  "textures": [
     // all textures required for the zoomer HUD
     // "zoomerhud",
     // "zoomerhud-dial",
@@ -130,6 +132,16 @@
       "lump": {
         "name": "test-eco"
       }
+    },
+    {
+      "trans": [-5.41, 3.5, 28.42], // translation
+      "etype": "test-actor",  // actor type
+      "game_task": 0, // associated game task (for powercells, etc)
+      "quat": [0, 0, 0, 1], // quaternion
+      "bsphere": [-7.41, 3.5, 28.42, 10], // bounding sphere
+      "lump": {
+        "name": "test-actor"
+      }
     }
     // {
     //   "trans": [-7.41, 3.5, 28.42], // translation

custom_assets/jak1/levels/test-zone/testzone.gd

@@ -4,6 +4,8 @@
 ;; the actual file name still needs to be 8.3
 ("TSZ.DGO"
  ("static-screen.o"
-  "test-zone.go"
+  "test-zone-obs.o"
   "plat-ag.go"
+  "test-actor-ag.go"
+  "test-zone.go"
  ))

custom_assets/jak2/levels/test-zone/test-zone.jsonc

@@ -10,7 +10,7 @@
   // Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
   // and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
   // only have 1.
-  "gltf_file": "custom_levels/jak2/test-zone/test-zone2.glb",
+  "gltf_file": "custom_assets/jak2/levels/test-zone/test-zone2.glb",
 
   // automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
   "automatic_wall_detection": true,