jak2: 90 fps menu option (#3245 )

Closes #3223
save & load game territory setting (#3243 )
2026-11-07 09:26:09 -10:00 · 2023-12-03 21:32:21 +01:00 · 2023-12-03 18:19:38 +00:00 · 2023-12-03 16:16:05 +00:00 · 2023-12-03 09:42:01 +00:00 · 2023-12-03 09:24:34 +00:00
11621 changed files with 3795655 additions and 610055 deletions
@@ -1,81 +0,0 @@
-name: "\U0001F41B Bug Report"
-description: Create a bug report.
-labels: ["bug"]
-body:
-  - type: checkboxes
-    attributes:
-      label: Acknowledgements
-      description: Take a couple minutes to help our maintainers work faster.
-      options:
-        - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+sort%3Aupdated-desc) for duplicate or closed bug reports
-          required: true
-        - label: I understand that I am supposed to provide my own legitimately obtained copy of the game
-          required: true
-
-  - type: textarea
-    attributes:
-      label: Describe the Bug
-      description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: How To Reproduce
-      description: Steps to reproduce the behavior. You can also post a video of it here.
-    validations:
-      required: true
-
-  - type: dropdown
-    attributes:
-      label: Does this problem occur on original hardware or PCSX2?
-      description: Some things that may seem like bugs are actually exactly how the original game behaved.
-      options:
-        - Yes, it's unique to OpenGOAL
-        - Didn't check
-        - Not needed, bug is obvious
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Expected Behavior
-      description: A clear and concise description of the expected behavior.
-      placeholder: When I do X, Y should happen.
-    validations:
-      required: true
-
-  - type: input
-    attributes:
-      label: Operating System
-      description: Windows version, Linux distribution, etc.
-    validations:
-      required: true
-
-  - type: input
-    attributes:
-      label: OpenGOAL Version
-      description: Either the release version (vX.X.X) or the commit sha if you've built from source
-    validations:
-      required: true
-
-  - type: dropdown
-    attributes:
-      label: Game Version
-      options:
-        - NTSC 1.0 (black label)
-        - NTSC Greatest Hits version (red label)
-        - PAL
-        - JP
-    validations:
-      required: true
-
-  - type: dropdown
-    attributes:
-      label: Are you using a high refresh-rate monitor (>60Hz)
-      options:
-        - "No"
-        - "Yes"
-    validations:
-      required: true
-
@@ -1 +1,8 @@
 blank_issues_enabled: true
+contact_links:
+  - name: Feature Requests and Ideas
+    url: https://github.com/open-goal/jak-project/discussions/new/choose
+    about: Please use the discussion board to bring up feature requests and ideas. If there is interest in them, they can then be converted to an issue(s).
+  - name: Discord Help Channel
+    url: https://discord.gg/dPRCfsju3N
+    about: The best place for general help and asking questions.
@@ -1,24 +0,0 @@
-name: "\U0001F680 Enhancement Request"
-description: Suggest an improvement for an existing feature.
-labels: ["enhancement"]
-body:
-  - type: textarea
-    attributes:
-      label: What feature is your idea related to?
-      description: A clear and concise description of what the problem is.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Describe the solution you'd like.
-      description: A clear and concise description of what you want to happen.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional context
-      description: Any other context or screenshots about the enhancement request here.
-    validations:
-      required: false
@@ -1,24 +0,0 @@
-name: "\U0001F3AE Game Feature Request"
-description: Suggest an idea for the game
-labels: ["game"]
-body:
-  - type: textarea
-    attributes:
-      label: Is your feature request related to a problem?
-      description: A clear and concise description of what the problem is.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Describe the solution you'd like.
-      description: A clear and concise description of what you want to happen.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional context
-      description: Add any other context or screenshots about the feature request here.
-    validations:
-      required: false
@@ -0,0 +1,74 @@
+name: "\U0001F41B Jak 1 Bug Report"
+description: Create a bug report for Jak 1.
+labels: ["bug", "jak1"]
+body:
+  - type: checkboxes
+    attributes:
+      label: Acknowledgements
+      description: Take a couple minutes to help our maintainers work faster.
+      options:
+        - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+label%3Ajak1+sort%3Aupdated-desc+) for duplicate or closed bug reports
+          required: true
+        - label: I understand that I am supposed to provide my own legitimately obtained copy of the game
+          required: true
+
+  - type: textarea
+    attributes:
+      label: Describe the Bug
+      description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: How To Reproduce
+      description: Steps to reproduce the behavior. You can also post a video of it here.
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Does this problem occur on original hardware or PCSX2?
+      description: Some things that may seem like bugs are actually exactly how the original game behaved.
+      options:
+        - Yes, it's unique to OpenGOAL
+        - Didn't check
+        - Not needed, bug is obvious
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of the expected behavior.
+      placeholder: When I do X, Y should happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Environment Information
+      description: "You can upload the [Support Package](https://github.com/open-goal/launcher#asking-for-help) provided by the Launcher here, or you can provide the following information: CPU, GPU, OS Version, OpenGOAL Version (found in the window's title bar)"
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Game Version
+      options:
+        - NTSC 1.0 (black label)
+        - NTSC Greatest Hits version (red label)
+        - PAL
+        - JP
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Have you set the game to something other than `60fps`?
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true
+
@@ -0,0 +1,74 @@
+name: "\U0001F41B Jak 2 Bug Report"
+description: Create a bug report for Jak 2.
+labels: ["bug", "jak2"]
+body:
+  - type: checkboxes
+    attributes:
+      label: Acknowledgements
+      description: Take a couple minutes to help our maintainers work faster.
+      options:
+        - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+label%3Ajak2+sort%3Aupdated-desc+) for duplicate or closed bug reports
+          required: true
+        - label: I understand that I am supposed to provide my own legitimately obtained copy of the game
+          required: true
+
+  - type: textarea
+    attributes:
+      label: Describe the Bug
+      description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: How To Reproduce
+      description: Steps to reproduce the behavior. You can also post a video of it here.
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Does this problem occur on original hardware or PCSX2?
+      description: Some things that may seem like bugs are actually exactly how the original game behaved.
+      options:
+        - Yes, it's unique to OpenGOAL
+        - Didn't check
+        - Not needed, bug is obvious
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of the expected behavior.
+      placeholder: When I do X, Y should happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Environment Information
+      description: "You can upload the [Support Package](https://github.com/open-goal/launcher#asking-for-help) provided by the Launcher here, or you can provide the following information: CPU, GPU, OS Version, OpenGOAL Version (found in the window's title bar)"
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Game Version
+      options:
+        - NTSC 1.0 (black label)
+        - NTSC Greatest Hits version (red label)
+        - PAL
+        - JP
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Have you set the game to something other than `60fps`?
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true
+
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-DEST=${1}
-BIN_SOURCE=${2}
-SOURCE=${3}
-
-mkdir -p $DEST
-
-PREP_BIN="${PREP_BIN:-true}"
-
-if [ "$PREP_BIN" = "true" ]; then
-  cp $BIN_SOURCE/game/gk $DEST
-  cp $BIN_SOURCE/goalc/goalc $DEST
-  cp $BIN_SOURCE/decompiler/extractor $DEST
-
-  strip $DEST/gk
-  strip $DEST/goalc
-  strip $DEST/extractor
-
-  chmod +x $DEST/gk
-  chmod +x $DEST/goalc
-  chmod +x $DEST/extractor
-fi
-
-mkdir -p $DEST/data
-mkdir -p $DEST/data/launcher/
-mkdir -p $DEST/data/decompiler/
-mkdir -p $DEST/data/assets
-mkdir -p $DEST/data/game
-mkdir -p $DEST/data/log
-mkdir -p $DEST/data/game/graphics/opengl_renderer/
-
-cp -r $SOURCE/.github/scripts/releases/error-code-metadata.json $DEST/data/launcher/error-code-metadata.json
-cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
-cp -r $SOURCE/goal_src $DEST/data
-cp -r $SOURCE/game/assets $DEST/data/game/
-cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
-cp -r $SOURCE/custom_levels $DEST/data
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+DEST=${1}
+BIN_SOURCE=${2}
+SOURCE=${3}
+
+mkdir -p $DEST
+
+PREP_BIN="${PREP_BIN:-true}"
+
+if [ "$PREP_BIN" = "true" ]; then
+  cp $BIN_SOURCE/game/gk $DEST
+  cp $BIN_SOURCE/goalc/goalc $DEST
+  cp $BIN_SOURCE/decompiler/extractor $DEST
+
+  chmod +x $DEST/gk
+  chmod +x $DEST/goalc
+  chmod +x $DEST/extractor
+fi
+
+mkdir -p $DEST/data
+mkdir -p $DEST/data/launcher/
+mkdir -p $DEST/data/decompiler/
+mkdir -p $DEST/data/game
+mkdir -p $DEST/data/log
+mkdir -p $DEST/data/game/graphics/opengl_renderer/
+
+cp -r $SOURCE/.github/scripts/releases/error-code-metadata.json $DEST/data/launcher/error-code-metadata.json
+cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
+cp -r $SOURCE/goal_src $DEST/data
+cp -r $SOURCE/game/assets $DEST/data/game/
+cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
+cp -r $SOURCE/custom_levels $DEST/data
@@ -15,7 +15,6 @@ cp $BIN_SOURCE/extractor.exe $DEST
 mkdir -p $DEST/data
 mkdir -p $DEST/data/launcher/
 mkdir -p $DEST/data/decompiler/
-mkdir -p $DEST/data/assets
 mkdir -p $DEST/data/game
 mkdir -p $DEST/data/log
 mkdir -p $DEST/data/game/graphics/opengl_renderer/
@@ -3,10 +3,11 @@ name: Build
 on:
  push:
    branches:
-      - '*'
+      - master
  pull_request:
    branches:
      - master
+  merge_group: {}

 jobs:
  # Windows
@@ -42,3 +43,20 @@ jobs:
      cmakePreset: "Release-linux-gcc"
      cachePrefix: ""
    secrets: inherit
+
+  # MacOS
+  build_macos_intel:
+    name: "🍎 MacOS"
+    uses: ./.github/workflows/macos-build.yaml
+    with:
+      cmakePreset: "Release-macos-clang"
+      cachePrefix: ""
+
+  # Q4 2023 there will hopefully be native arm64 runners 
+  # https://github.com/github/roadmap/issues/528
+  # build_macos_arm:
+  #   name: "🍎 MacOS"
+  #   uses: ./.github/workflows/macos-build-arm.yaml
+  #   with:
+  #     cmakePreset: "Release-macos-clang"
+  #     cachePrefix: ""
@@ -23,7 +23,7 @@ jobs:
      - name: Bump Version and Push Tag
        if: github.repository == 'open-goal/jak-project'
        id: tag_version
-        uses: mathieudutour/github-tag-action@v6.0
+        uses: mathieudutour/github-tag-action@v6.1
        with:
          github_token: ${{ secrets.BOT_PAT }}
          tag_prefix: v
@@ -6,17 +6,78 @@ on:
      - master
    paths:
      - 'goal_src/**'
+  workflow_dispatch: {}

 jobs:
-  inform:
-    name: Inform Pages Repo
+  gen-docs:
+    name: Generate Documentation
    if: github.repository == 'open-goal/jak-project'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
+    runs-on: ubuntu-20.04
+    timeout-minutes: 45
+    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
+      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
+      BUILDCACHE_COMPRESS_FORMAT: ZSTD
+      BUILDCACHE_COMPRESS_LEVEL: 19
+      BUILDCACHE_DIRECT_MODE: true
+      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Package Dependencies
+        run: |
+          sudo apt update
+          sudo apt install build-essential cmake \
+            clang gcc g++ lcov make nasm libxrandr-dev \
+            libxinerama-dev libxcursor-dev libpulse-dev \
+            libxi-dev zip ninja-build libgl1-mesa-dev
+
+      - name: Setup Buildcache
+        uses: mikehardy/[email protected]
+        with:
+          cache_key: linux-ubuntu-20.04--Release-linux-clang
+          buildcache_tag: v0.28.1
+
+      - name: CMake Generation
+        env:
+          CC: clang
+          CXX: clang++
+        run: |
+          cmake -B build --preset=Release-linux-clang \
+            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+
+      - name: Build Project
+        run: cmake --build build --parallel $((`nproc`)) --target goalc
+
+      - name: Generate Docs For Jak 1
+        run: |
+          ./build/goalc/goalc --cmd "(begin (make-group \"all-code\") (gen-docs \"${PWD}\"))" --game jak1
+
+      - name: Generate Docs For Jak 2
+        run: |
+          ./build/goalc/goalc --cmd "(begin (make-group \"all-code\") (gen-docs \"${PWD}\"))" --game jak2
+
+      - name: Append File Docstrings
+        run: |
+          python ./scripts/docs/append-file-docs.py
+
+      - name: Upload Docs Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: opengoal-docs
+          if-no-files-found: error
+          path: |
+            ./jak1*.json
+            ./jak2*.json
+
+      - name: Log Run ID
+        run: echo ${{ github.run_id }}
+
      - name: Send Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.BOT_PAT }}
          repository: 'open-goal/open-goal.github.io'
          event-type: updateProgress
+          client-payload: '{"docGenRunId": "${{ github.run_id }}"}'
@@ -3,19 +3,20 @@ name: Lint
 on:
  push:
    branches:
-      - '*'
+      - master
  pull_request:
    branches:
      - master
+  merge_group: {}

 jobs:
-  lint:
+  formatting:
    name: 📝 Formatting
    runs-on: ubuntu-20.04
    timeout-minutes: 10
    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      # wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
      # sudo add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main'
@@ -31,3 +32,38 @@ jobs:
        run: |
          chmod +x ./third-party/run-clang-format/run-clang-format.py
          ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp --color always
+
+  required-checks:
+    name: 📝 Required Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Check for Unresolved Conflicts
+        run: python ./scripts/gsrc/check-for-conflicts.py
+
+      - name: Check for Incorrect Asserts
+        run: python ./scripts/ci/check-for-asserts.py
+
+      - name: Check for Invalid Translation Characters
+        run: python ./scripts/ci/lint-characters.py
+
+  optional-checks:
+    name: 📝 Optional Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Python Dependencies
+        run: pip install unidiff colorama
+
+      - name: Check for Removed goal_src/ Code
+        # FYI - if you run this `git diff` on windows, it creates a utf-16 LE file
+        run: |
+          git fetch origin
+          git diff origin/master > lint-changes.diff
+          python ./scripts/ci/lint-gsrc-removals.py
@@ -9,12 +9,16 @@ on:
      cachePrefix:
        required: true
        type: string
+      uploadArtifacts:
+        required: false
+        type: boolean
+        default: false

 jobs:
  build:
    name: Clang
    runs-on: ubuntu-20.04
-    timeout-minutes: 45
+    timeout-minutes: 60

    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
@@ -25,20 +29,21 @@ jobs:

    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Install Package Dependencies
-        run: >
-          sudo apt install build-essential cmake
-          clang gcc g++ lcov make nasm libxrandr-dev
-          libxinerama-dev libxcursor-dev libpulse-dev
-          libxi-dev zip ninja-build
+        run: |
+          sudo apt update
+          sudo apt install build-essential cmake \
+            clang gcc g++ lcov make nasm libxrandr-dev \
+            libxinerama-dev libxcursor-dev libpulse-dev \
+            libxi-dev zip ninja-build libgl1-mesa-dev libssl-dev

      - name: Setup Buildcache
-        uses: xTVaser/buildcache-action@version-flag
+        uses: mikehardy/buildcache-action@v2.1.0
        with:
          cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.27.6
+          buildcache_tag: v0.28.1

      - name: CMake Generation
        env:
@@ -50,15 +55,24 @@ jobs:
            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache

      - name: Build Project
-        run: cmake --build build -j$((`nproc`+1))
+        run: cmake --build build --parallel $((`nproc`))

      - name: Run Tests
        env:
          GTEST_OUTPUT: "xml:opengoal-test-report.xml"
        run: ./test.sh

+      - name: Prepare artifacts
+        if: ${{ inputs.uploadArtifacts }}
+        run: |
+          strip ./build/goalc/goalc
+          strip ./build/decompiler/extractor
+          strip ./build/game/gk
+          strip ./build/lsp/lsp
+
      - name: Upload artifact
        uses: actions/upload-artifact@v3
+        if: ${{ inputs.uploadArtifacts }}
        with:
          name: opengoal-linux-${{ inputs.cachePrefix }}
          if-no-files-found: error
@@ -14,7 +14,7 @@ jobs:
  build:
    name: GCC
    runs-on: ubuntu-20.04
-    timeout-minutes: 45
+    timeout-minutes: 60

    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
@@ -25,20 +25,21 @@ jobs:

    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Install Package Dependencies
-        run: >
-          sudo apt install build-essential cmake
-          clang gcc g++ lcov make nasm libxrandr-dev
-          libxinerama-dev libxcursor-dev libpulse-dev
-          libxi-dev zip ninja-build
+        run: |
+          sudo apt update
+          sudo apt install build-essential cmake \
+            clang gcc g++ lcov make nasm libxrandr-dev \
+            libxinerama-dev libxcursor-dev libpulse-dev \
+            libxi-dev zip ninja-build libgl1-mesa-dev libssl-dev

      - name: Setup Buildcache
-        uses: xTVaser/buildcache-action@version-flag
+        uses: mikehardy/buildcache-action@v2.1.0
        with:
          cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
-          buildcache_tag: v0.27.6
+          buildcache_tag: v0.28.1

      - name: CMake Generation
        env:
@@ -46,22 +47,19 @@ jobs:
          CXX: g++
        run: |
          cmake -B build --preset=${{ inputs.cmakePreset }} \
-            -DCODE_COVERAGE=ON \
+            -DCODE_COVERAGE=OFF \
            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache

      - name: Build Project
-        run: cmake --build build -j$((`nproc`+1)) -- -w dupbuild=warn
+        run: cmake --build build --parallel $((`nproc`)) -- -w dupbuild=warn

-      - name: Run Tests - With Coverage
-        working-directory: ./build
-        env:
-          GTEST_OUTPUT: "xml:opengoal-test-report.xml"
-        run: ninja goalc-test_coverage -w dupbuild=warn
+      - name: Run Tests
+        run: ./test.sh

-      - name: Submit Coverage Report to Codacy
-        uses: codacy/codacy-coverage-reporter-action@v1
-        continue-on-error: true
-        with:
-          project-token: ${{ secrets.CODACY_PROJECT_KEY }}
-          coverage-reports: ./build/goalc-test_coverage.info
+      # - name: Submit Coverage Report to Codacy
+      #   uses: codacy/codacy-coverage-reporter-action@v1
+      #   continue-on-error: true
+      #   with:
+      #     project-token: ${{ secrets.CODACY_PROJECT_KEY }}
+      #     coverage-reports: ./build/goalc-test_coverage.info
@@ -0,0 +1,66 @@
+name: MacOS Build
+
+on:
+  workflow_call:
+    inputs:
+      cmakePreset:
+        required: true
+        type: string
+      cachePrefix:
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: ARM
+    runs-on: macos-latest
+    timeout-minutes: 120
+
+    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
+      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
+      BUILDCACHE_COMPRESS_FORMAT: ZSTD
+      BUILDCACHE_COMPRESS_LEVEL: 19
+      BUILDCACHE_DIRECT_MODE: true
+      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up ARM64 environment
+        run: sudo softwareupdate --install-rosetta --agree-to-license
+
+      - name: Install Package Dependencies
+        run: arch -arm64 brew install cmake ninja
+
+      - name: Setup Buildcache
+        uses: mikehardy/[email protected]
+        with:
+          cache_key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          buildcache_tag: v0.28.1
+
+      - name: CMake Generation
+        env:
+          CC: clang
+          CXX: clang++
+        run: |
+          cmake -B build --preset=${{ inputs.cmakePreset }} \
+            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+
+      - name: Build Project
+        run: cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))
+
+      - name: Run Tests
+        run: ./test.sh
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: opengoal-macos-${{ inputs.cachePrefix }}
+          if-no-files-found: error
+          path: |
+            ./build/goalc/goalc
+            ./build/decompiler/extractor
+            ./build/game/gk
+            ./build/lsp/lsp
@@ -0,0 +1,76 @@
+name: MacOS Build
+
+on:
+  workflow_call:
+    inputs:
+      cmakePreset:
+        required: true
+        type: string
+      cachePrefix:
+        required: true
+        type: string
+      uploadArtifacts:
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  build:
+    name: Intel
+    runs-on: macos-11
+    timeout-minutes: 120
+
+    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
+      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
+      BUILDCACHE_COMPRESS_FORMAT: ZSTD
+      BUILDCACHE_COMPRESS_LEVEL: 19
+      BUILDCACHE_DIRECT_MODE: true
+      BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Package Dependencies
+        run: brew install cmake nasm ninja
+
+      - name: Setup Buildcache
+        uses: mikehardy/[email protected]
+        with:
+          cache_key: macos-11-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
+          buildcache_tag: v0.28.1
+
+      - name: CMake Generation
+        env:
+          CC: clang
+          CXX: clang++
+        run: |
+          cmake -B build --preset=${{ inputs.cmakePreset }} \
+            -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
+
+      - name: Build Project
+        run: cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))
+
+      - name: Run Tests
+        run: ./test.sh
+
+      - name: Prepare artifacts
+        if: ${{ inputs.uploadArtifacts }}
+        run: |
+          strip ./build/goalc/goalc
+          strip ./build/decompiler/extractor
+          strip ./build/game/gk
+          strip ./build/lsp/lsp
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ inputs.uploadArtifacts }}
+        with:
+          name: opengoal-macos-${{ inputs.cachePrefix }}
+          if-no-files-found: error
+          path: |
+            ./build/goalc/goalc
+            ./build/decompiler/extractor
+            ./build/game/gk
+            ./build/lsp/lsp
@@ -13,6 +13,7 @@ jobs:
    with:
      cmakePreset: "Release-windows-clang-static"
      cachePrefix: "static"
+      uploadArtifacts: true
    secrets: inherit

  # Linux
@@ -22,6 +23,17 @@ jobs:
    with:
      cmakePreset: "Release-linux-clang-static"
      cachePrefix: "static"
+      uploadArtifacts: true
+    secrets: inherit
+
+  # macOS
+  build_macos_intel:
+    name: "🍎 MacOS"
+    uses: ./.github/workflows/macos-build.yaml
+    with:
+      cmakePreset: "Release-macos-clang-static"
+      cachePrefix: "static"
+      uploadArtifacts: true
    secrets: inherit

  # Upload the Artifacts
@@ -30,10 +42,11 @@ jobs:
    needs:
      - build_windows_clang
      - build_linux_clang
+      - build_macos_intel
    name: "Upload Artifacts"
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4

      - name: Prepare Artifact Folder
        run: mkdir -p ./ci-artifacts/final
@@ -49,12 +62,11 @@ jobs:
      - name: Prepare Linux Release Assets
        run: |
          mkdir -p ./ci-artifacts/linux
-          ./.github/scripts/releases/extract_build_linux.sh ./ci-artifacts/linux ./ci-artifacts/opengoal-linux-static ./
+          ./.github/scripts/releases/extract_build_unix.sh ./ci-artifacts/linux ./ci-artifacts/opengoal-linux-static ./
          pushd ci-artifacts/linux
          TAG_VAL=$(echo ${{ github.REF }} | awk -F'refs/tags/' '{print $2}')
          tar czf ../final/opengoal-linux-${TAG_VAL}.tar.gz .
          popd
-          strip ./ci-artifacts/opengoal-linux-static/lsp/lsp
          chmod +x ./ci-artifacts/opengoal-linux-static/lsp/lsp
          cp ./ci-artifacts/opengoal-linux-static/lsp/lsp ./ci-artifacts/final/opengoal-lsp-linux-${TAG_VAL}.bin

@@ -66,6 +78,17 @@ jobs:
          7z a -tzip ./ci-artifacts/final/opengoal-windows-${TAG_VAL}.zip ./ci-artifacts/windows/*
          cp ./ci-artifacts/opengoal-windows-static/lsp.exe ./ci-artifacts/final/opengoal-lsp-windows-${TAG_VAL}.exe

+      - name: Prepare macOS Build Assets
+        run: |
+          mkdir -p ./ci-artifacts/macos
+          ./.github/scripts/releases/extract_build_unix.sh ./ci-artifacts/macos ./ci-artifacts/opengoal-macos-static ./
+          pushd ci-artifacts/macos
+          TAG_VAL=$(echo ${{ github.REF }} | awk -F'refs/tags/' '{print $2}')
+          tar czf ../final/opengoal-macos-intel-${TAG_VAL}.tar.gz .
+          popd
+          chmod +x ./ci-artifacts/opengoal-macos-static/lsp/lsp
+          cp ./ci-artifacts/opengoal-macos-static/lsp/lsp ./ci-artifacts/final/opengoal-lsp-macos-intel-${TAG_VAL}.bin
+
      - name: Upload Assets
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -79,10 +102,3 @@ jobs:
        run: |
          TAG_VAL=$(echo ${{ github.REF }} | awk -F'refs/tags/' '{print $2}')
          gh release edit ${TAG_VAL} --draft=false --repo open-goal/jak-project
-
-      - name: Consume Release in the Launcher
-        uses: peter-evans/repository-dispatch@v2
-        with:
-          token: ${{ secrets.BOT_PAT }}
-          repository: 'open-goal/launcher'
-          event-type: releaseLauncher
@@ -3,13 +3,14 @@ name: Update Controller Database
 on:
  schedule:
    - cron: "0 16 * * 1" # every monday @ 12pm EST - https://crontab.guru/#0_16_*_*_1
+  workflow_dispatch: {}

 jobs:
  update-controller-db:
    if: github.repository == 'open-goal/jak-project'
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.BOT_PAT }}

@@ -17,10 +18,14 @@ jobs:
        run: |
          wget -O ./game/assets/sdl_controller_db.txt https://raw.githubusercontent.com/gabomdq/SDL_GameControllerDB/master/gamecontrollerdb.txt

-      - name: commit version bump
-        uses: EndBug/add-and-commit@v9
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
        with:
-          default_author: github_actor
-          author_name: 'OpenGOALBot'
-          author_email: '[email protected]'
-          message: "Updating Controller Database"
+          token: ${{ secrets.BOT_PAT }}
+          author: 'OpenGOALBot <[email protected]>'
+          committer: 'OpenGOALBot <OpenGOALBot@users.noreply.github.com>'
+          branch: 'bot/update-controller-db'
+          base: "master"
+          body: "Updating Controller Database"
+          commit-message: "ci: updated controller database"
+          title: "CI: Periodic Controller Database Update"
@@ -9,13 +9,17 @@ on:
      cachePrefix:
        required: true
        type: string
+      uploadArtifacts:
+        required: false
+        type: boolean
+        default: false

 jobs:
  build:
    name: Clang
    runs-on: windows-2022
    # Set some sort of timeout in the event of run-away builds.  We are limited on concurrent jobs so, get rid of them.
-    timeout-minutes: 45
+    timeout-minutes: 60

    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
@@ -26,25 +30,39 @@ jobs:

    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Install NASM
-        run: choco install ${{ github.workspace }}/third-party/nasm/nasm.2.15.05.nupkg
+        # TODO - Simplify this with just the first command once choco 2.0 rolls out everywhere
+        run: |
+          $chocoVersion = choco --version
+          if ($chocoVersion.StartsWith("2.")) {
+            choco install nasm --source="${{ github.workspace }}/third-party/nasm"
+          } else {
+            choco install ${{ github.workspace }}/third-party/nasm/old/nasm.2.15.05.nupkg
+          }

      - name: Setup Buildcache
-        uses: mikehardy/buildcache-action@v1.3.0
+        uses: mikehardy/buildcache-action@v2.1.0
        with:
          cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}

      - uses: ilammy/msvc-dev-cmd@v1

+      # https://github.com/actions/runner-images/issues/5459
+      # Causes binaries to be built differently, get rid of this garbage
+      - name: Remove Strawberry Perl from PATH
+        run: |
+          $env:PATH = $env:PATH -replace "C:\\Strawberry\\c\\bin;", ""
+          "PATH=$env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Append
+
      - name: CMake Generation
        shell: cmd
        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe .

      - name: Build Project
        shell: cmd
-        run: cmake --build build -j 2
+        run: cmake --build build --parallel %NUMBER_OF_PROCESSORS%

      - name: Run Tests
        timeout-minutes: 10
@@ -54,6 +72,7 @@ jobs:

      - name: Upload artifact
        uses: actions/upload-artifact@v3
+        if: ${{ inputs.uploadArtifacts }}
        with:
          name: opengoal-windows-${{ inputs.cachePrefix }}
          if-no-files-found: error
@@ -14,7 +14,7 @@ jobs:
  build:
    name: MSVC
    runs-on: windows-2022
-    timeout-minutes: 45
+    timeout-minutes: 60

    env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
      BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
@@ -25,18 +25,31 @@ jobs:

    steps:
      - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Install NASM
-        run: choco install ${{ github.workspace }}/third-party/nasm/nasm.2.15.05.nupkg
+        # TODO - Simplify this with just the first command once choco 2.0 rolls out everywhere
+        run: |
+          $chocoVersion = choco --version
+          if ($chocoVersion.StartsWith("2.")) {
+            choco install nasm --source="${{ github.workspace }}/third-party/nasm"
+          } else {
+            choco install ${{ github.workspace }}/third-party/nasm/old/nasm.2.15.05.nupkg
+          }

      - name: Setup Buildcache
-        uses: mikehardy/buildcache-action@v1.3.0
+        uses: mikehardy/buildcache-action@v2.1.0
        with:
          cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}

      - uses: ilammy/msvc-dev-cmd@v1

+      # Fixes an issue with the image causing builds to fail - https://github.com/actions/runner-images/issues/8598
+      - name: Remove Strawberry Perl from PATH
+        run: |
+          $env:PATH = $env:PATH -replace "C:\\Strawberry\\c\\bin;", ""
+          "PATH=$env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Append
+
      - name: CMake Generation - MSVC
        shell: cmd
        run: cmake -B build --preset=${{ inputs.cmakePreset }} -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache.exe .
@@ -45,7 +58,7 @@ jobs:
        shell: cmd
        run: |
          set CL=/MP
-          cmake --build build -j 2
+          cmake --build build --parallel %NUMBER_OF_PROCESSORS%

      - name: Run Tests
        timeout-minutes: 10
@@ -1,3 +1,7 @@
+# logs
+/log
+prof.json
+
 # for CMake
 /Testing

@@ -6,9 +10,13 @@ cmake-build-debug/*
 cmake-build-debug--o0/*
 .idea/*
 build/*
-decompiler_out/*
-decompiler_out2/*
+/decompiler_out*
 logs/*
+profile_data/*
+
+# for vscode/clangd
+.cache/*
+.DS_Store

 # wsl apparently builds to here?
 linux-default/
@@ -33,9 +41,10 @@ search-results.json
 /search-results.json

 # graphics debug
-debug_out/*
-gfx_dumps/*
-screenshots/*
+debug_out/
+glb_out/
+gfx_dumps/
+screenshots/

 # game stuff
 game_config/*
@@ -49,6 +58,7 @@ texture_replacements/*

 # generated cmake files
 svnrev.h
+common/versions/revision.h
 ci-artifacts/
 out/build/
 __pycache__/
@@ -59,3 +69,10 @@ __pycache__/

 # backup files from OpenMaya
 *.bak
+
+# docs
+/jak1-*.json
+/jak2-*.json
+/TODO.md
+unifont-15.0.03.ttf
+*.diff
@@ -0,0 +1 @@
+endOfLine: "auto"
@@ -68,6 +68,20 @@
        "jak2"
      ]
    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "offline-test.exe (bin\\offline-test.exe)",
+      "name": "Tests - Offline Tests - Jak 1 - Specific File",
+      "args": [
+        "--iso_data_path",
+        "${workspaceRoot}/iso_data/jak1",
+        "--game",
+        "jak1",
+        "--file",
+        "euler-h"
+      ]
+    },
    {
      "type": "default",
      "project": "CMakeLists.txt",
@@ -79,50 +93,43 @@
        "--game",
        "jak2",
        "--file",
-        "progress"
+        "oracle-texture"
      ]
    },
    {
      "type": "default",
      "project": "CMakeLists.txt",
      "projectTarget": "gk.exe (bin\\gk.exe)",
-      "name": "Game - Runtime (no kernel)",
-      "args": ["-fakeiso", "-debug", "-nokernel", "-v", "-nodisplay"]
+      "name": "Game - Jak 1 - Runtime",
+      "args": ["-v", "--game", "jak1", "--", "-fakeiso", "-debug"]
    },
    {
      "type": "default",
      "project": "CMakeLists.txt",
      "projectTarget": "gk.exe (bin\\gk.exe)",
-      "name": "Game - Runtime (with kernel)",
-      "args": ["-fakeiso", "-debug", "-v"]
-    },
-    {
-      "type": "default",
-      "project": "CMakeLists.txt",
-      "projectTarget": "gk.exe (bin\\gk.exe)",
-      "name": "Game - Runtime (boot)",
-      "args": ["-boot", "-fakeiso", "-debug", "-v"]
-    },
-    {
-      "type": "default",
-      "project": "CMakeLists.txt",
-      "projectTarget": "gk.exe (bin\\gk.exe)",
-      "name": "Game - Runtime (boot no debug)",
-      "args": ["-boot", "-fakeiso", "-v"]
-    },
-    {
-      "type": "default",
-      "project": "CMakeLists.txt",
-      "projectTarget": "gk.exe (bin\\gk.exe)",
-      "name": "Game - Jak 2 - Runtime (boot)",
-      "args": ["-boot", "-fakeiso", "-debug", "-v", "-jak2"]
+      "name": "Game - Jak 1 - Runtime (boot)",
+      "args": ["-v", "--game", "jak1", "--", "-boot", "-fakeiso", "-debug"]
    },
    {
      "type": "default",
      "project": "CMakeLists.txt",
      "projectTarget": "gk.exe (bin\\gk.exe)",
      "name": "Game - Jak 2 - Runtime (no boot)",
-      "args": ["-fakeiso", "-debug", "-v", "-jak2"]
+      "args": ["-v", "--game", "jak2", "--", "-fakeiso", "-debug"]
+    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "gk.exe (bin\\gk.exe)",
+      "name": "Game - Jak 2 - Runtime (boot)",
+      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso", "-debug"]
+    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "gk.exe (bin\\gk.exe)",
+      "name": "Game - Jak 2 - Runtime (release)",
+      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso"]
    },
    {
      "type": "default",
@@ -135,15 +142,15 @@
      "type": "default",
      "project": "CMakeLists.txt",
      "projectTarget": "goalc.exe (bin\\goalc.exe)",
-      "name": "REPL - Jak 2",
-      "args": ["--user-auto", "--game", "jak2"]
+      "name": "REPL - Jak 1",
+      "args": ["--user-auto", "--game", "jak1"]
    },
    {
      "type": "default",
      "project": "CMakeLists.txt",
      "projectTarget": "goalc.exe (bin\\goalc.exe)",
-      "name": "REPL - Auto Listen",
-      "args": ["--user-auto", "--auto-lt"]
+      "name": "REPL - Jak 2",
+      "args": ["--user-auto", "--game", "jak2"]
    },
    {
      "type": "default",
@@ -151,9 +158,24 @@
      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
      "name": "Decompiler - Jak 1",
      "args": [
-        "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc",
+        "${workspaceRoot}/decompiler/config/jak1/jak1_config.jsonc",
        "${workspaceRoot}/iso_data",
-        "${workspaceRoot}/decompiler_out"
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "ntsc_v1"
+      ]
+    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
+      "name": "Decompiler - Jak 1 NTSC-U v2",
+      "args": [
+        "${workspaceRoot}/decompiler/config/jak1/jak1_config.jsonc",
+        "${workspaceRoot}/iso_data",
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "ntsc_v2"
      ]
    },
    {
@@ -162,9 +184,11 @@
      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
      "name": "Decompiler - Jak 1 PAL",
      "args": [
-        "${workspaceRoot}/decompiler/config/jak1_pal.jsonc",
+        "${workspaceRoot}/decompiler/config/jak1/jak1_config.jsonc",
        "${workspaceRoot}/iso_data",
-        "${workspaceRoot}/decompiler_out"
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "pal"
      ]
    },
    {
@@ -173,9 +197,11 @@
      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
      "name": "Decompiler - Jak 2",
      "args": [
-        "${workspaceRoot}/decompiler/config/jak2_ntsc_v1.jsonc",
+        "${workspaceRoot}/decompiler/config/jak2/jak2_config.jsonc",
        "${workspaceRoot}/iso_data",
-        "${workspaceRoot}/decompiler_out"
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "ntsc_v1"
      ]
    },
    {
@@ -184,10 +210,12 @@
      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
      "name": "Decompiler - Jak 2 - Extract",
      "args": [
-        "${workspaceRoot}/decompiler/config/jak2_ntsc_v1.jsonc",
+        "${workspaceRoot}/decompiler/config/jak2/jak2_config.jsonc",
        "${workspaceRoot}/iso_data",
        "${workspaceRoot}/decompiler_out",
-        "--config-override \"{\\\"decompile_code\\\": false}\""
+        "--version",
+        "ntsc_v1",
+        "--config-override \"{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, \\\"allowed_objects\\\": []}\""
      ]
    },
    {
@@ -233,7 +261,21 @@
      "project" : "CMakeLists.txt",
      "projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
      "name" : "Tools - Type Searcher",
-      "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--fields", "[{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":2},{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":4}]"]
+      "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--size", 255, "--fields", "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"]
+    },
+    {
+      "type" : "default",
+      "project" : "CMakeLists.txt",
+      "projectTarget" : "formatter.exe (bin\\formatter.exe)",
+      "name" : "Tools - Formatter - Inplace",
+      "args" : ["--write", "--file", "C:\\Users\\xtvas\\Repos\\opengoal\\jak-project\\goal_src\\jak2\\pc\\util\\popup-menu-h.gc"]
+    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
+      "name": "Tests - Formatter",
+      "args": ["--gtest_brief=0", "--gtest_filter=\"*FormatterTests*\""]
    }
  ]
 }
@@ -3,7 +3,7 @@
  // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp

  // List of extensions which should be recommended for users of this workspace.
-  "recommendations": ["opengoal.opengoal", "naumovs.color-highlight"],
+  "recommendations": ["opengoal.opengoal"],
  // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
  "unwantedRecommendations": []
 }
@@ -5,13 +5,13 @@
  "version": "0.2.0",
  "configurations": [
    {
-      "name": "Python",
+      "name": "Append File Docs",
      "type": "python",
      "request": "launch",
-      "program": "${workspaceFolder}/scripts/gsrc/update-from-decomp.py",
+      "program": "${workspaceFolder}/scripts/ci/lint-characters.py",
      "console": "integratedTerminal",
      "cwd": "${workspaceFolder}",
-      "args": ["--game", "jak2", "--file", "sparticle-h"]
+      "args": ["--fix"]
    },
  ]
 }
@@ -17,4 +17,10 @@
    "body": [";; og:ignore-form:${1:substr}"],
    "description": "If the provided substr is found in the starting line of a form, the entire form will be omitted when copying decompiled code into the file"
  },
+  "og:preserve-this": {
+		"scope": "opengoal",
+    "prefix": ["og:preserve-this"],
+    "body": [";; og:preserve-this"],
+    "description": "When updating GOAL source files from decomp, CI will pick up if this comment is deleted in the diff. Use this to keep track of manually patched sections of code."
+  },
 }
@@ -9,4 +9,97 @@
    "editor.snippetSuggestions": "top"
  },
  "python.formatting.provider": "black",
+  "files.associations": {
+    "optional": "cpp",
+    "algorithm": "cpp",
+    "any": "cpp",
+    "array": "cpp",
+    "atomic": "cpp",
+    "bit": "cpp",
+    "bitset": "cpp",
+    "cctype": "cpp",
+    "charconv": "cpp",
+    "chrono": "cpp",
+    "cinttypes": "cpp",
+    "clocale": "cpp",
+    "cmath": "cpp",
+    "codecvt": "cpp",
+    "compare": "cpp",
+    "concepts": "cpp",
+    "condition_variable": "cpp",
+    "cstdarg": "cpp",
+    "cstddef": "cpp",
+    "cstdint": "cpp",
+    "cstdio": "cpp",
+    "cstdlib": "cpp",
+    "cstring": "cpp",
+    "ctime": "cpp",
+    "cwchar": "cpp",
+    "cwctype": "cpp",
+    "deque": "cpp",
+    "exception": "cpp",
+    "filesystem": "cpp",
+    "format": "cpp",
+    "forward_list": "cpp",
+    "fstream": "cpp",
+    "functional": "cpp",
+    "future": "cpp",
+    "initializer_list": "cpp",
+    "iomanip": "cpp",
+    "ios": "cpp",
+    "iosfwd": "cpp",
+    "iostream": "cpp",
+    "istream": "cpp",
+    "iterator": "cpp",
+    "limits": "cpp",
+    "list": "cpp",
+    "locale": "cpp",
+    "map": "cpp",
+    "memory": "cpp",
+    "mutex": "cpp",
+    "new": "cpp",
+    "numeric": "cpp",
+    "ostream": "cpp",
+    "queue": "cpp",
+    "random": "cpp",
+    "ranges": "cpp",
+    "ratio": "cpp",
+    "regex": "cpp",
+    "set": "cpp",
+    "shared_mutex": "cpp",
+    "span": "cpp",
+    "sstream": "cpp",
+    "stack": "cpp",
+    "stdexcept": "cpp",
+    "stop_token": "cpp",
+    "streambuf": "cpp",
+    "string": "cpp",
+    "system_error": "cpp",
+    "thread": "cpp",
+    "tuple": "cpp",
+    "type_traits": "cpp",
+    "typeinfo": "cpp",
+    "unordered_map": "cpp",
+    "unordered_set": "cpp",
+    "utility": "cpp",
+    "valarray": "cpp",
+    "variant": "cpp",
+    "vector": "cpp",
+    "xfacet": "cpp",
+    "xhash": "cpp",
+    "xiosbase": "cpp",
+    "xlocale": "cpp",
+    "xlocbuf": "cpp",
+    "xlocinfo": "cpp",
+    "xlocmes": "cpp",
+    "xlocmon": "cpp",
+    "xlocnum": "cpp",
+    "xloctime": "cpp",
+    "xmemory": "cpp",
+    "xstddef": "cpp",
+    "xstring": "cpp",
+    "xtr1common": "cpp",
+    "xtree": "cpp",
+    "xutility": "cpp"
+  }
 }
@@ -18,14 +18,20 @@ option(STATICALLY_LINK "Build for release purposes (statically link everything)"
 message(STATUS "Statically Link? ${STATICALLY_LINK}")
 if(STATICALLY_LINK)
    set(BUILD_SHARED_LIBS OFF)
+    set(BUILD_STATIC_LIBS ON)
 else()
    set(BUILD_SHARED_LIBS ON)
+    set(BUILD_STATIC_LIBS OFF)
+endif()
+
+# For clangd
+if (EXISTS "${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json" )
+    configure_file(
+        "${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json"
+        "${PROJECT_SOURCE_DIR}/build/compile_commands.json")
 endif()

 # Setup compiler flags
-# TODO - consider moving most of the configuration into presets
-# - https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html
-# - requires atleast CMake 3.19
 if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
    message(STATUS "Clang Detected - Setting Defaults")
    set(CMAKE_CXX_FLAGS
@@ -38,13 +44,13 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
        -Wall \
        -Wno-c++11-narrowing \
        -Wno-c++98-compat \
-        -O3")
+        -O3 \
+        -D_CRT_SECURE_NO_WARNINGS")

    # Increase stack size for windows, who's default is too low
    if(WIN32)
        # Increase the reserved stack size for all threads to 16MB
        # Note: this is only _reserved_ memory, not necessarily _committed_ memory
-        # TODO - test with add_link_options instead
        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LDFLAGS} -Xlinker /STACK:16000000")
    else()
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -g -Wextra")
@@ -56,7 +62,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
    endif()

    if(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
-        # TODO - test with add_link_options instead
        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LDFLAGS} -Xlinker /debug")
    endif()

@@ -94,6 +99,34 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
    endif()
    set(THIRDPARTY_IGNORED_WARNINGS "-w")
+elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+    message(STATUS "AppleClang detected - Setting Defaults")
+    set(CMAKE_CXX_FLAGS
+        "${CMAKE_CXX_FLAGS} \
+        -march=native \
+        -Wall \
+        -Winit-self \
+        -ggdb \
+        -Wextra \
+        -Wno-cast-align \
+        -Wcast-qual \
+        -Wdisabled-optimization \
+        -Wformat \
+        -Wextra \
+        -Wmissing-include-dirs \
+        -Woverloaded-virtual \
+        -Wredundant-decls \
+        -Wshadow \
+        -Wsign-promo \
+        -fdiagnostics-color=always"
+        )
+
+    # additional c++ flags for release mode for our projects
+    if(CMAKE_BUILD_TYPE MATCHES "Release")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
+    endif()
+    set(THIRDPARTY_IGNORED_WARNINGS "-w")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-stack_size -Wl,0x20000000")
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
    message(STATUS "MSVC detected - Setting Defaults")

@@ -122,8 +155,6 @@ if(WIN32)
    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-    set(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION 7.1.7600.0.30514) # win7.1, supports xp
-    message("Windows SDK version: ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}")
 endif()

 # Code Coverage
@@ -135,17 +166,64 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CODE_COVERAGE)
    message(STATUS "Code Coverage build is enabled!")
 endif()

+function(build_third_party_lib dir_name target_name)
+    add_subdirectory(third-party/${dir_name} EXCLUDE_FROM_ALL)
+    if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
+        # For Microsoft Visual C++ Compiler (MSVC)
+        target_compile_options(${target_name} PRIVATE /w)
+    elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+        # For GNU Compiler Collection (GCC) or Clang
+        target_compile_options(${target_name} PRIVATE -w)
+    endif()
+endfunction()
+
+
 # Dependencies and Libraries
 # includes relative to top level jak-project folder
 include_directories(./)

 # build templating engine library
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THIRDPARTY_IGNORED_WARNINGS} ")
 include_directories(SYSTEM third-party/inja)

+# libcurl for HTTP requests
+# Enable SSL Support, most URLs now-a-days use SSL!
+# TODO - probably integrate with ZSTD since we have it already
+set(CURL_USE_LIBSSH2 OFF)
+set(CURL_ZLIB OFF)
+if(WIN32)
+    set(CURL_USE_SCHANNEL ON) # native Windows SSL support
+elseif(APPLE)
+    set(CURL_USE_SECTRANSP ON) # native macOS SSL support
+else()
+    if(STATICALLY_LINK)
+        set(OPENSSL_USE_STATIC_LIBS TRUE)
+    endif()
+    set(CURL_USE_OPENSSL ON) # not native, but seems to be the best choice for linux
+endif()
+include_directories(third-party/curl/include)
+if(STATICALLY_LINK)
+    build_third_party_lib(curl libcurl_static)
+else()
+    build_third_party_lib(curl libcurl_shared)
+endif()
+
 # build repl library
-add_subdirectory(third-party/replxx EXCLUDE_FROM_ALL)
-string(REPLACE " ${THIRDPARTY_IGNORED_WARNINGS} " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+build_third_party_lib(replxx replxx)
+
+# SQLite - Jak 2/3's built in editor
+add_definitions(-DHAVE_USLEEP=1)
+add_definitions(-DSQLITE_THREADSAFE=1)
+add_definitions(-DSQLITE_ENABLE_JSON1)
+add_definitions(-DSQLITE_ENABLE_RTREE)
+build_third_party_lib(sqlite3 sqlite3)
+
+# build tree-sitter parser
+include_directories(third-party/tree-sitter/tree-sitter/lib/include)
+include_directories(third-party/tree-sitter/tree-sitter-opengoal/include)
+build_third_party_lib(tree-sitter tree-sitter)
+
+# native OS dialogs for error messages
+build_third_party_lib(libtinyfiledialogs libtinyfiledialogs)

 # build common library
 add_subdirectory(common)
@@ -153,24 +231,28 @@ add_subdirectory(common)
 # build decompiler
 add_subdirectory(decompiler)

-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THIRDPARTY_IGNORED_WARNINGS} ")
-add_subdirectory(third-party/cubeb EXCLUDE_FROM_ALL)
+# cubeb - audio
+build_third_party_lib(cubeb cubeb)

 # build LSP
 add_subdirectory(lsp)

-# build libco
-add_subdirectory(third-party/libco)
+# build libco and zstd
+build_third_party_lib(libco libco)
+build_third_party_lib(zstd libzstd_static)

-# build glfw library
-add_subdirectory(third-party/glfw EXCLUDE_FROM_ALL)
-add_subdirectory(third-party/zstd EXCLUDE_FROM_ALL)
+# build SDL
+include(SDLOptions)
+if(STATICALLY_LINK)
+    build_third_party_lib(SDL SDL2-static)
+else()
+    build_third_party_lib(SDL SDL2)
+endif()

 # build imgui
 include_directories(third-party/glad/include)
-include_directories(third-party/glfw/include)
-add_subdirectory(third-party/imgui EXCLUDE_FROM_ALL)
-string(REPLACE " ${THIRDPARTY_IGNORED_WARNINGS} " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+include_directories(third-party/SDL/include)
+build_third_party_lib(imgui imgui)

 # build the game code in C++
 add_subdirectory(game)
@@ -186,28 +268,25 @@ if(WIN32)
    set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 endif()

-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THIRDPARTY_IGNORED_WARNINGS} ")
-add_subdirectory(third-party/googletest EXCLUDE_FROM_ALL)
+build_third_party_lib(googletest gtest)
 include(GoogleTest)
-string(REPLACE " ${THIRDPARTY_IGNORED_WARNINGS} " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")

 # build tests
 include(test/CMakeLists.txt)

 # build lzokay library
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THIRDPARTY_IGNORED_WARNINGS} ")
-add_subdirectory(third-party/lzokay EXCLUDE_FROM_ALL)
+build_third_party_lib(lzokay lzokay)

 # build format library
-add_subdirectory(third-party/fmt EXCLUDE_FROM_ALL)
+build_third_party_lib(fmt fmt)

-add_subdirectory(third-party/stb_image EXCLUDE_FROM_ALL)
-add_subdirectory(third-party/tiny_gltf EXCLUDE_FROM_ALL)
-add_subdirectory(third-party/xdelta3 EXCLUDE_FROM_ALL)
+build_third_party_lib(stb_image stb_image)
+build_third_party_lib(tiny_gltf tiny_gltf)
+build_third_party_lib(xdelta3 xdelta3)

 # discord rich presence
 include_directories(third-party/discord-rpc/include)
-add_subdirectory(third-party/discord-rpc EXCLUDE_FROM_ALL)
+build_third_party_lib(discord-rpc discord-rpc)

 # build zydis third party library for disassembling x86
 # TODO - Once under CMake 3.13's policy CMP0077, override with `set()` instead
@@ -222,10 +301,9 @@ else()
    option(ZYDIS_BUILD_SHARED_LIB "Zydis: Build shared library" ON)
 endif()

-add_subdirectory(third-party/zydis EXCLUDE_FROM_ALL)
+build_third_party_lib(zydis Zydis)

 # windows memory management lib
 if(WIN32)
-    add_subdirectory(third-party/mman)
+    build_third_party_lib(mman mman)
 endif()
-string(REPLACE " ${THIRDPARTY_IGNORED_WARNINGS} " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
@@ -14,7 +14,8 @@
        "ASAN_BUILD": "OFF",
        "STATICALLY_LINK": "OFF",
        "ZYDIS_BUILD_SHARED_LIB": "ON",
-        "CODE_COVERAGE": "OFF"
+        "CODE_COVERAGE": "OFF",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
      }
    },
    {
@@ -57,6 +58,26 @@
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
      }
    },
+    {
+      "name": "base-linux-debug",
+      "hidden": true,
+      "inherits": "base",
+      "binaryDir": "${sourceDir}/build/Release/bin",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
+      }
+    },
+    {
+      "name": "base-macos-release",
+      "hidden": true,
+      "inherits": "base",
+      "binaryDir": "${sourceDir}/build/Release/bin",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
+      }
+    },
    {
      "name": "base-clang",
      "hidden": true,
@@ -127,13 +148,29 @@
      "description": "Build with Clang as Release without Debug Symbols",
      "inherits": ["base-linux-release", "base-clang"]
    },
+    {
+      "name": "Release-macos-clang",
+      "displayName": "MacOS Release (clang)",
+      "description": "Build with Clang as Release without Debug Symbols",
+      "inherits": ["base-macos-release", "base-clang"]
+    },
+    {
+      "name": "Release-macos-clang-static",
+      "displayName": "MacOS Static Release (clang)",
+      "description": "Build with Clang as Release without Debug Symbols but statically linked",
+      "inherits": ["base-macos-release", "base-clang"],
+      "cacheVariables": {
+        "STATICALLY_LINK": "true",
+        "ZYDIS_BUILD_SHARED_LIB": "OFF"
+      }
+    },
    {
      "name": "Release-linux-clang-asan",
      "displayName": "Linux Release (clang-asan)",
      "description": "Build with Clang as Release without Debug Symbols and ASAN Fuzzing",
      "inherits": ["base-linux-release", "base-clang"],
      "cacheVariables": {
-        "ASAN_BUILD": "ON"
+        "ASAN_BUILD": "OFF"
      }
    },
    {
@@ -146,6 +183,16 @@
        "ZYDIS_BUILD_SHARED_LIB": "OFF"
      }
    },
+    {
+      "name": "Debug-linux-clang-static",
+      "displayName": "Linux Static Release (clang)",
+      "description": "Build with Clang as Release without Debug Symbols but statically linked",
+      "inherits": ["base-linux-debug", "base-clang"],
+      "cacheVariables": {
+        "STATICALLY_LINK": "true",
+        "ZYDIS_BUILD_SHARED_LIB": "OFF"
+      }
+    },
    {
      "name": "Release-linux-gcc",
      "displayName": "Linux Release (gcc)",
@@ -4,15 +4,26 @@

 <p align="center">
  <a href="https://opengoal.dev/docs/intro" rel="nofollow"><img src="https://img.shields.io/badge/Documentation-Here-informational" alt="Documentation Badge" style="max-width:100%;"></a>
-  <a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/workflows/Build/badge.svg"><img src="https://github.com/open-goal/jak-project/workflows/Build/badge.svg" alt="Linux and Windows Build" style="max-width:100%;"></a>
-  <a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&utm_medium=referral&utm_content=open-goal/jak-project&utm_campaign=Badge_Coverage" rel="nofollow"><img src="https://app.codacy.com/project/badge/Coverage/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
+  <a title="Crowdin" target="_blank" href="https://crowdin.com/project/opengoal"><img src="https://badges.crowdin.net/opengoal/localized.svg"></a>
+  <a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/actions/workflows/build-matrix.yaml"><img src="https://github.com/open-goal/jak-project/actions/workflows/build-matrix.yaml/badge.svg" alt="Linux and Windows Build" style="max-width:100%;"></a>
  <a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=open-goal/jak-project&amp;utm_campaign=Badge_Grade" rel="nofollow"><img src="https://app.codacy.com/project/badge/Grade/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
  <a href="https://discord.gg/VZbXMHXzWv"><img src="https://img.shields.io/discord/756287461377703987" alt="Discord"></a>
-  <a href="https://makeapullrequest.com"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square" alt=PRs Welcome></a>
 </p>

- [Please read first](#please-read-first)
-  - [Quick Start](#quick-start)
+## Please read first <!-- omit from toc -->
+
+> [!IMPORTANT]
+> Our repositories on GitHub are primarily for development of the project and tracking active issues. Most of the information you will find here pertains to setting up the project for development purposes and is not relevant to the end-user.
+
+For a setup guide on how to install and play the game there is the following video that you can check out: https://youtu.be/K84UUMnkJc4
+
+For questions or additional information pertaining to the project, we have a Discord for discussion here: https://discord.gg/VZbXMHXzWv
+
+Additionally, you can find further documentation and answers to **frequently asked questions** on the project's main website: https://opengoal.dev
+
+> [!WARNING]
+> **Do not use this decompilation project without providing your own legally purchased copy of the game.** We do not distribute any assets from the game - you must use your own legitimately obtained PS2 copy of the game. We support every retail PAL, NTSC, and NTSC-J build, including Greatest Hits copies.
+
 - [Project Description](#project-description)
  - [Current Status](#current-status)
  - [Methodology](#methodology)
@@ -25,6 +36,11 @@
  - [Windows](#windows)
    - [Required Software](#required-software)
    - [Using Visual Studio](#using-visual-studio)
+  - [MacOS](#macos)
+    - [Intel Based](#intel-based)
+    - [Apple Silicon](#apple-silicon)
+  - [VSCode](#vscode)
+    - [Building and Debugging](#building-and-debugging)
  - [Building and Running the Game](#building-and-running-the-game)
    - [Extract Assets](#extract-assets)
    - [Build the Game](#build-the-game)
@@ -40,25 +56,9 @@
  - [`goal_src/`](#goal_src)
  - [`game` runtime](#game-runtime)

-## Please read first
-
-Our repositories on GitHub are primarily for development of the project and tracking active issues. Most of the information you will find here pertains to setting up the project for development purposes and is not relevant to the end-user.
-
-For questions or additional information pertaining to the project, we have a Discord for discussion here: https://discord.gg/VZbXMHXzWv
-
-Additionally, you can find further documentation and answers to **frequently asked questions** on the project's main website: https://opengoal.dev
-
-**Do not use this decompilation project without providing your own legally purchased copy of the game.**
-
-### Quick Start
-
-If you just want to play the game, you can follow the steps in this video: https://www.youtube.com/watch?v=p8I9NfuZOgE
-
-We do not distribute any assets from the game - you must use your own legitimately obtained PS2 copy of the game. We support every retail PAL, NTSC, and NTSC-J build, including Greatest Hits copies. 
-
 ## Project Description

-This project is to port Jak 1 (NTSC, "black label" version) to PC. Over 98% of this game is written in GOAL, a custom Lisp language developed by Naughty Dog. Our strategy is:
+This project is to port the original Jak and Daxter and Jak II to PC. Over 98% of the games are written in GOAL, a custom Lisp language developed by Naughty Dog. Our strategy is:
 - decompile the original game code into human-readable GOAL code
 - develop our own compiler for GOAL and recompile game code for x86-64
 - create a tool to extract game assets into formats that can be easily viewed or modified
@@ -76,7 +76,7 @@ We support both Linux and Windows on x86-64.

 ### Current Status

-Jak 1 is largely playable from start to finish with a handful of bugs that are continually being ironed out.
+Jak 1 is largely playable from start to finish with a handful of bugs that are continually being ironed out. Jak 2 is in development.

 ![](./docs/img/promosmall1.png)
 ![](./docs/img/promosmall2.png)
@@ -127,7 +127,7 @@ Unfortunately you'll still need task runner on your local machine to run the gam
 Install packages and init repository:

 ```sh
-sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libpulse-dev libxi-dev python
+sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libpulse-dev libxi-dev python libgl1-mesa-dev libssl-dev
 sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 ```

@@ -183,7 +183,7 @@ Run tests:
 Install packages and init repository:

 ```sh
-sudo dnf install cmake python lld clang nasm libX11-devel libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel pulseaudio-libs-devel
+sudo dnf install cmake python lld clang nasm libX11-devel libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel pulseaudio-libs-devel mesa-libGL-devel
 sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 ```

@@ -233,6 +233,51 @@ Then build the entire project as `Windows Release (clang)`. You can also press C
 ![](./docs/img/windows/release-build.png)
 ![](./docs/img/windows/build-all.png)

+### MacOS
+
+> NOTE: At this time you can only run the game on macOS if you have an Intel processor.
+
+Ensure that you have Xcode command line tools installed (this installs things like Apple Clang).  If you don't, you can run the following command:
+
+```bash
+xcode-select --install
+```
+
+#### Intel Based
+
+```bash
+brew install go-task/tap/go-task
+brew install cmake nasm ninja go-task clang-format
+cmake -B build --preset=Release-macos-clang
+cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))
+```
+
+#### Apple Silicon
+
+**Not Supported at This Time**
+
+```bash
+brew install go-task/tap/go-task
+brew install cmake ninja go-task clang-format
+cmake -B build --preset=Release-macos-clang
+cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))
+```
+
+You may have to add the MacOS SDK to your `LIBRARY_PATH`:
+- `export LIBRARY_PATH="$LIBRARY_PATH:/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"`
+
+### VSCode
+
+If you either don't want to or cannot use Visual Studio for working with the C++ project, VSCode is a good alternatively.
+
+The `clangd` extension is [recommended](https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd) and requires `clangd` to be on your `$PATH`.  If you can run `clangd` in a terminal successfully then you should be good to go.
+
+Once you generate your CMake for the first time the clangd LSP should be able to index the project and give you intellisense.
+
+#### Building and Debugging
+
+TODO - Consider Contributing Documentation :)
+
 ### Building and Running the Game

 Getting a running game involves 4 steps:
@@ -299,11 +344,11 @@ Run the following to build the game:
 g > (mi)
 ```

-> IMPORTANT NOTE! If you're not using the black label version, you may hit issues trying to run `(mi)` in this step. An example error might include something like:
+> IMPORTANT NOTE! If you're not using the non-default version of the game, you may hit issues trying to run `(mi)` in this step. An example error might include something like:
 >
 > `Input file iso_data/jak1/MUS/TWEAKVAL.MUS does not exist.`
 >
-> This is because other version paths are not currently accounted for in the build. A quick workaround is to rename both your `decompiler_out` and `iso_data` folders to use the black label naming, for example changing `decompiler_out/jak1_pal` to `decompiler_out/jak1` and `iso_data/jak1_pal` to `iso_data/jak1`, then running `(mi)` again.
+> This is because the decompiler inputs/outputs using the `gameName` JSON field in the decompiler config. For example if you are using Jak 1 PAL, it will assume `iso_data/jak1_pal` and `decompiler_out/jak1_pal`.  Therefore, you can inform the REPL/compiler of this via the `gameVersionFolder` config field described [here](./goal_src/user/README.md)

 #### Run the Game

@@ -1,4 +1,4 @@
-version: "3.13"
+version: 3

 includes:
  build: ./scripts/tasks/Taskfile_{{OS}}.yml
@@ -15,6 +15,8 @@ tasks:
    - 'python ./scripts/tasks/update-env.py --game jak1'
  set-game-jak2:
    - 'python ./scripts/tasks/update-env.py --game jak2'
+  set-game-jak3:
+    - 'python ./scripts/tasks/update-env.py --game jak3'
  set-decomp-ntscv1:
    desc: "aka black label"
    cmds:
@@ -31,6 +33,10 @@ tasks:
    desc: "NTSC-J region version"
    cmds:
      - 'python ./scripts/tasks/update-env.py --decomp_config ntscjp'
+  set-decomp-ntscko:
+    desc: "NTSC-K region version"
+    cmds:
+      - 'python ./scripts/tasks/update-env.py --decomp_config ntscko'
  # GENERAL
  extract:
    desc: "Extracts the game's assets from './iso_data' with the set decompiler config"
@@ -38,28 +44,28 @@ tasks:
      - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}}
        msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
    cmds:
-      - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" ./iso_data ./decompiler_out --config-override "{\\\"decompile_code\\\":false,\\\"allowed_objects\\\":[]}"'
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"allowed_objects\": []}'"
  boot-game:
    desc: "Boots the game, it will fail if it's not already compiled!"
    preconditions:
      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
    cmds:
-      - "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -debug -v"
+      - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso -debug"
  boot-game-retail:
    desc: "Boots the game without debug mode, it will fail if it's not already compiled!"
    preconditions:
      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
    cmds:
-      - "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -v"
+      - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso"
  run-game:
    desc: "Start the game's runtime, to start the game itself the REPL is required"
    preconditions:
      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
    cmds:
-      - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -{{.GAME}} -v"
+      - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -fakeiso -debug"
  # DEVELOPMENT
  repl:
    desc: "Start the REPL"
@@ -67,29 +73,32 @@ tasks:
      - sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
        msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
    cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --game {{.GAME}}"
-  repl-lt:
-    cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt --game {{.GAME}}"
+      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --user-auto --game {{.GAME}}"
  format:
    desc: "Format code"
    cmds:
+      - cmd: python ./scripts/cpp/format-includes.py
      - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
      # npm install -g prettier
-      - cmd: npx prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
+      - cmd: npx prettier --write ./decompiler/config/jak1/**/*.jsonc
        ignore_error: true
-      - cmd: npx prettier --write ./decompiler/config/jak2/*.jsonc
+      - cmd: npx prettier --write ./decompiler/config/jak2/**/*.jsonc
        ignore_error: true
-  run-game-headless:
+  format-json:
+    desc: "Format code"
    cmds:
-      - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -nodisplay"
+      # npm install -g prettier
+      - cmd: npx prettier --write ./decompiler/config/jak1/**/*.jsonc
+        ignore_error: true
+      - cmd: npx prettier --write ./decompiler/config/jak2/**/*.jsonc
+        ignore_error: true
  # DECOMPILING
  decomp:
    cmds:
-      - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out"'
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": true, \"levels_extract\": false}'"
  decomp-file:
    cmds:
-      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --config-override '{\"allowed_objects\": [\"{{.FILE}}\"]}'"
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": true, \"levels_extract\": false, \"allowed_objects\": [\"{{.FILE}}\"]}'"
  decomp-clean:
    cmds:
      - python ./scripts/tasks/clean-decomp.py --game "{{.GAME}}"
@@ -98,7 +107,10 @@ tasks:
      - python ./scripts/gsrc/lint-gsrc-file.py --game {{.GAME}} --file {{.FILE}}
  update-gsrc:
    cmds:
-      - python ./scripts/gsrc/update-gsrc-via-refs.py --game "{{.GAME}}" --decompiler "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler" --decompiler_config {{.DECOMP_CONFIG}}
+      - python ./scripts/gsrc/update-gsrc-via-refs.py --game "{{.GAME}}" --decompiler "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler" --decompiler_config {{.DECOMP_CONFIG}} --version {{.DECOMP_CONFIG_VERSION}}
+  update-gsrc-glob:
+    cmds:
+      - python ./scripts/gsrc/update-gsrc-via-refs.py --game "{{.GAME}}" --file_pattern "{{.GLOB}}" --decompiler "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler" --decompiler_config {{.DECOMP_CONFIG}} --version {{.DECOMP_CONFIG_VERSION}}
  update-gsrc-file:
    cmds:
      - task: decomp-file
@@ -110,6 +122,7 @@ tasks:
      - python ./scripts/tasks/extract-zip.py --file "{{.FILE}}" --out ./savestate_out/
      - '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool ./savestate_out/eeMemory.bin --output-path ./ --game {{.GAME}} > ee-analysis.log'
  watch-pcsx2:
+    # python -m pip install -U "watchdog[watchmedo]"
    cmds:
      - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
    vars:
@@ -118,31 +131,44 @@ tasks:
    desc: Just an example to show it running
    cmds:
      - "{{.TYPESEARCH_BIN_RELEASE_DIR}}/type_searcher --output-path ./search-results.json --game {{.GAME}} --fields '[{\"type\":\"int16\",\"offset\":2},{\"type\":\"int16\",\"offset\":4}]'"
+  update-treesitter:
+    desc: Updates locally built tree-sitter rules
+    cmds:
+      - cd ../tree-sitter-opengoal && yarn gen
+      - python ./scripts/tasks/cp.py --src "../tree-sitter-opengoal/src/*" --dest "./third-party/tree-sitter/tree-sitter-opengoal"
+      - python ./scripts/tasks/cp.py --src "../tree-sitter-opengoal/grammar.js" --dest "./third-party/tree-sitter/tree-sitter-opengoal"
+  fix-translations:
+    desc: Finds and tries to fix invalid translation Characters
+    cmds:
+      - python ./scripts/ci/lint-characters.py --fix
  # TESTS
  offline-tests: # ran by jenkins
    cmds:
-      - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}}'
-  offline-tests-fast:
-    cmds:
-      - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --dump_current_output --num_threads 32'
+      - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --fail-on-cmp'
  offline-test-file:
    cmds:
      - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --file {{.FILE}}'
+  offline-tests-fast:
+    cmds:
+      - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --pretty-print --num_threads 32 --dump_current_output --fail-on-cmp'
  # TODO - amalgamate offline-tests and this task, run twice if the previous step fails
  update-ref-tests:
    cmds:
      - cmd: python ./scripts/tasks/delete-file-or-folder.py --path failures
-      - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --dump_current_output --num_threads 32'
+      - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --pretty-print --num_threads 32 --dump_current_output --fail-on-cmp'
        ignore_error: true
      - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
      - task: offline-tests-fast
  update-ref-file:
    cmds:
      - cmd: python ./scripts/tasks/delete-file-or-folder.py --path failures
-      - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --file {{.FILE}} --game {{.GAME}} --dump_current_output'
+      - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --file {{.FILE}} --game {{.GAME}} --dump_current_output --fail-on-cmp'
        ignore_error: true
      - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
      - task: offline-test-file
+  type-test-jak1:
+    cmds:
+      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak1TypeConsistency*" --gtest_break_on_failure'
  type-test:
    cmds:
-      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_break_on_failure --gtest_filter="*Jak2TypeConsistency*"'
+      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak2TypeConsistency*" --gtest_break_on_failure'
@@ -1,29 +1,63 @@
+function(write_revision_h)
+    find_package(Git)
+    set(GIT_SHORT_SHA "")
+    set(GIT_TAG "")
+    if(NOT GIT_FOUND)
+      MESSAGE(STATUS "write_revision_h(): git was not found")
+    endif()
+    MESSAGE(STATUS "write_revision_h(): ${CMAKE_SOURCE_DIR}")
+    if (GIT_FOUND AND EXISTS ${CMAKE_SOURCE_DIR}/.git)
+        MESSAGE(STATUS "Git found, using it to get revision info")
+        EXECUTE_PROCESS(WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
+                OUTPUT_VARIABLE GIT_SHORT_SHA
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+        EXECUTE_PROCESS(WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} tag --points-at HEAD
+                OUTPUT_VARIABLE GIT_TAG
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+                ERROR_QUIET)
+    endif()
+    file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/versions/revision.h "#define BUILT_TAG \"${GIT_TAG}\"\n#define BUILT_SHA \"${GIT_SHORT_SHA}\"\n")
+endfunction()
+
+write_revision_h()
+
 add_library(common
-        versions.cpp
        audio/audio_formats.cpp
        cross_os_debug/xdbg.cpp
        cross_sockets/XSocket.cpp
-        cross_sockets/XSocketServer.cpp
        cross_sockets/XSocketClient.cpp
+        cross_sockets/XSocketServer.cpp
        custom_data/pack_helpers.cpp
        custom_data/TFrag3Data.cpp
-        dma/dma.cpp
        dma/dma_copy.cpp
+        dma/dma.cpp
        dma/gs.cpp
+        formatter/formatter_tree.cpp
+        formatter/formatter.cpp
+        formatter/formatter_tree.cpp
+        formatter/rules/formatting_rules.cpp
+        formatter/rules/rule_config.cpp
        global_profiler/GlobalProfiler.cpp
        goos/Interpreter.cpp
        goos/Object.cpp
        goos/ParseHelpers.cpp
-        goos/Printer.cpp
        goos/PrettyPrinter.cpp
        goos/PrettyPrinter2.cpp
+        goos/Printer.cpp
        goos/Reader.cpp
        goos/TextDB.cpp
-        goos/ReplUtils.cpp
        log/log.cpp
        math/geometry.cpp
-        nrepl/ReplClient.cpp
-        nrepl/ReplServer.cpp
+        repl/config.cpp
+        repl/nrepl/ReplClient.cpp
+        repl/nrepl/ReplServer.cpp
+        repl/util.cpp
+        serialization/subtitles/subtitles_v1.cpp
+        serialization/subtitles/subtitles_v2.cpp
+        serialization/subtitles/subtitles.cpp
+        serialization/text/text_ser.cpp
+        sqlite/sqlite.cpp
+        texture/texture_slots.cpp
        type_system/defenum.cpp
        type_system/deftype.cpp
        type_system/state.cpp
@@ -31,8 +65,6 @@ add_library(common
        type_system/TypeFieldLookup.cpp
        type_system/TypeSpec.cpp
        type_system/TypeSystem.cpp
-        serialization/subtitles/subtitles_ser.cpp
-        serialization/subtitles/subtitles_deser.cpp
        util/Assert.cpp
        util/BitUtils.cpp
        util/compress.cpp
@@ -40,22 +72,29 @@ add_library(common
        util/dgo_util.cpp
        util/DgoReader.cpp
        util/DgoWriter.cpp
+        util/dialogs.cpp
        util/diff.cpp
        util/FileUtil.cpp
        util/FontUtils.cpp
+        util/FrameLimiter.cpp
        util/json_util.cpp
-        util/read_iso_file.cpp
-        util/SimpleThreadGroup.cpp
-        util/Timer.cpp
        util/os.cpp
        util/print_float.cpp
-        util/FrameLimiter.cpp
-        util/unicode_util.cpp  )
+        util/read_iso_file.cpp
+        util/SimpleThreadGroup.cpp
+        util/string_util.cpp
+        util/term_util.cpp
+        util/Timer.cpp
+        util/unicode_util.cpp
+        versions/versions.cpp
+        )

-target_link_libraries(common fmt lzokay replxx libzstd_static)
+target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs)

 if(WIN32)
    target_link_libraries(common wsock32 ws2_32 windowsapp)
+elseif(APPLE)
+    # don't need anything special
 else()
    target_link_libraries(common stdc++fs)
 endif()
@@ -8,62 +8,87 @@
 /*!
 * Write a wave file from a vector of samples.
 */
-void write_wave_file_mono(const std::vector<s16>& samples, s32 sample_rate, const fs::path& name) {
+void write_wave_file(const std::vector<s16>& left_samples,
+                     const std::vector<s16>& right_samples,
+                     s32 sample_rate,
+                     const fs::path& name) {
  WaveFileHeader header;
  memcpy(header.chunk_id, "RIFF", 4);
-  header.chunk_size = 36 + samples.size() * sizeof(s16);
+  header.chunk_size = 36 + ((left_samples.size() + right_samples.size()) * sizeof(s16));
  memcpy(header.format, "WAVE", 4);

  // now the format
  memcpy(header.subchunk1_id, "fmt ", 4);
  header.subchunk1_size = 16;
  header.aud_format = 1;
-  header.num_channels = 1;  // mono
+  if (right_samples.empty()) {
+    header.num_channels = 1;  // mono
+  } else {
+    header.num_channels = 2;  // stereo
+  }
+
  header.sample_rate = sample_rate;
  header.byte_rate = sample_rate * header.num_channels * sizeof(s16);
  header.block_align = header.num_channels * sizeof(s16);
  header.bits_per_sample = 16;

  memcpy(header.subchunk2_id, "data", 4);
-  header.subchunk2_size = samples.size() * sizeof(s16);
+  header.subchunk2_size = (left_samples.size() + right_samples.size()) * sizeof(s16);

  BinaryWriter writer;
  writer.add(header);

-  for (auto& samp : samples) {
-    writer.add(samp);
+  if (right_samples.empty()) {
+    for (const auto& sample : left_samples) {
+      writer.add(sample);
+    }
+  } else {
+    for (size_t i = 0; i < left_samples.size(); i++) {
+      writer.add(left_samples.at(i));
+      if (i < right_samples.size()) {
+        writer.add(right_samples.at(i));
+      } else {
+        writer.add(0);
+      }
+    }
  }

  writer.write_to_file(name);
 }

-std::vector<s16> decode_adpcm(BinaryReader& reader) {
-  std::vector<s16> decoded_samples;
-  s32 sample_prev[2] = {0, 0};
+std::pair<std::vector<s16>, std::vector<s16>> decode_adpcm(BinaryReader& reader, const bool mono) {
+  std::vector<s16> left_samples;
+  std::vector<s16> right_samples;
+  s32 left_sample_prev[2] = {0, 0};
+  s32 right_sample_prev[2] = {0, 0};
  constexpr s32 f1[5] = {0, 60, 115, 98, 122};
  constexpr s32 f2[5] = {0, 0, -52, -55, -60};

-  int block_idx = 0;
+  [[maybe_unused]] int block_idx = 0;
+  // 16 byte blocks
+  int bytes_read = reader.get_seek();  // we've already read n bytes into the file
+  // Jak VAG's don't interleave the samples because of course they don't
+  // instead they are partitioned into contiguous 8kb (thats 8192 bytes) chunks
+  // alternating left/right
+  bool processing_left_chunk = true;
  while (true) {
    if (!reader.bytes_left()) {
      break;
    }
+
+    if (bytes_read == 0x2000) {
+      // switch streams
+      processing_left_chunk = !processing_left_chunk;
+      bytes_read = 0;
+    }
+
    u8 shift_filter = reader.read<u8>();
-    u8 flags = reader.read<u8>();
    u8 shift = shift_filter & 0b1111;
    u8 filter = shift_filter >> 4;
+    u8 flags = reader.read<u8>();
+    (void)flags;

-    if (shift > 12) {
-      ASSERT(false);
-    }
-
-    if (filter > 4) {
-      ASSERT(false);
-    }
-
-    if (flags == 7) {
-      break;
-    }
+    // removed assertions here (and that's probably why the audio doesn't sound right)

    u8 input_buffer[14];

@@ -81,25 +106,42 @@ std::vector<s16> decode_adpcm(BinaryReader& reader) {

      s32 sample = (s32)(s16)(nibble << 12);
      sample >>= shift;
-      sample += (sample_prev[0] * f1[filter] + sample_prev[1] * f2[filter] + 32) / 64;

-      if (sample > 0x7fff) {
-        sample = 0x7fff;
+      if (mono || processing_left_chunk) {
+        sample += (left_sample_prev[0] * f1[filter] + left_sample_prev[1] * f2[filter] + 32) / 64;
+
+        if (sample > 0x7fff) {
+          sample = 0x7fff;
+        }
+
+        if (sample < -0x8000) {
+          sample = -0x8000;
+        }
+
+        left_sample_prev[1] = left_sample_prev[0];
+        left_sample_prev[0] = sample;
+        left_samples.push_back(sample);
+      } else {
+        sample += (right_sample_prev[0] * f1[filter] + right_sample_prev[1] * f2[filter] + 32) / 64;
+
+        if (sample > 0x7fff) {
+          sample = 0x7fff;
+        }
+
+        if (sample < -0x8000) {
+          sample = -0x8000;
+        }
+
+        right_sample_prev[1] = right_sample_prev[0];
+        right_sample_prev[0] = sample;
+        right_samples.push_back(sample);
      }
-
-      if (sample < -0x8000) {
-        sample = -0x8000;
-      }
-
-      sample_prev[1] = sample_prev[0];
-      sample_prev[0] = sample;
-
-      decoded_samples.push_back(sample);
    }
+    bytes_read += 16;
    block_idx++;
  }

-  return decoded_samples;
+  return {left_samples, right_samples};
 }

 // I attempted to write an encoder below, which works, but has some limitations.
@@ -29,8 +29,11 @@ struct WaveFileHeader {
  s32 subchunk2_size;
 };

-void write_wave_file_mono(const std::vector<s16>& samples, s32 sample_rate, const fs::path& name);
+void write_wave_file(const std::vector<s16>& left_samples,
+                     const std::vector<s16>& right_samples,
+                     s32 sample_rate,
+                     const fs::path& name);

-std::vector<s16> decode_adpcm(BinaryReader& reader);
+std::pair<std::vector<s16>, std::vector<s16>> decode_adpcm(BinaryReader& reader, const bool mono);

 std::vector<u8> encode_adpcm(const std::vector<s16>& samples);
@@ -30,3 +30,7 @@ struct u128 {
  };
 };
 static_assert(sizeof(u128) == 16, "u128");
+
+#if defined __linux || defined __linux__ || defined __APPLE__
+#define OS_POSIX
+#endif
@@ -392,7 +392,7 @@ bool cont_now(const ThreadID& tid) {
 DEBUG_EVENT debugEvent;
 void ignore_debug_event() {
  if (!ContinueDebugEvent(debugEvent.dwProcessId, debugEvent.dwThreadId, DBG_CONTINUE)) {
-    win_print_last_error("ContinueDebugEvent");
+    win_print_last_error("ContinueDebugEvent ignore_debug_event");
  }
  cont_status = -1;
 }
@@ -456,7 +456,7 @@ bool check_stopped(const ThreadID& tid, SignalInfo* out) {
    if (cont_status != -1) {
      cv.wait(lk, [&] { return cont_status == 1; });
      if (!ContinueDebugEvent(debugEvent.dwProcessId, debugEvent.dwThreadId, DBG_CONTINUE)) {
-        win_print_last_error("ContinueDebugEvent");
+        win_print_last_error("ContinueDebugEvent check_stopped");
      }
      cont_status = -1;
    }
@@ -505,18 +505,12 @@ bool check_stopped(const ThreadID& tid, SignalInfo* out) {
          }
        }
      } break;
+      case CREATE_THREAD_DEBUG_EVENT:   // 2
      case CREATE_PROCESS_DEBUG_EVENT:  // 3
-        if (debugEvent.u.CreateProcessInfo.hProcess != NULL &&
-            GetProcessId(debugEvent.u.CreateProcessInfo.hProcess) == debugEvent.dwProcessId) {
-        }
-        // out->kind = SignalInfo::NOTHING;
-        ignore_debug_event();
-        break;
-      case CREATE_THREAD_DEBUG_EVENT:  // 2
-      case EXIT_THREAD_DEBUG_EVENT:    // 4
-      case LOAD_DLL_DEBUG_EVENT:       // 6
-      case UNLOAD_DLL_DEBUG_EVENT:     // 7
-      case OUTPUT_DEBUG_STRING_EVENT:  // 8
+      case EXIT_THREAD_DEBUG_EVENT:     // 4
+      case LOAD_DLL_DEBUG_EVENT:        // 6
+      case UNLOAD_DLL_DEBUG_EVENT:      // 7
+      case OUTPUT_DEBUG_STRING_EVENT:   // 8
        // don't care about these
        // out->kind = SignalInfo::NOTHING;
        ignore_debug_event();
@@ -554,7 +548,7 @@ bool read_goal_memory(u8* dest_buffer,
  HANDLE hProc = OpenProcess(PROCESS_VM_READ, FALSE, context.tid.pid);

  if (hProc == NULL) {
-    win_print_last_error("OpenProcess");
+    win_print_last_error("OpenProcess read_goal_memory");
    return false;
  }

@@ -579,7 +573,7 @@ bool write_goal_memory(const u8* src_buffer,
  HANDLE hProc = OpenProcess(PROCESS_VM_WRITE, FALSE, context.tid.pid);

  if (hProc == NULL) {
-    win_print_last_error("OpenProcess");
+    win_print_last_error("OpenProcess write_goal_memory");
    return false;
  }

@@ -601,7 +595,7 @@ bool get_regs_now(const ThreadID& tid, Regs* out) {
  HANDLE hThr = OpenThread(THREAD_GET_CONTEXT, FALSE, tid.tid);

  if (hThr == NULL) {
-    win_print_last_error("OpenThread");
+    win_print_last_error("OpenThread get_regs_now");
    return false;
  }

@@ -609,7 +603,7 @@ bool get_regs_now(const ThreadID& tid, Regs* out) {
  CloseHandle(hThr);

  if (!result) {
-    win_print_last_error("GetThreadContext");
+    win_print_last_error("GetThreadContext get_regs_now");
    return false;
  }

@@ -641,7 +635,7 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) {
  HANDLE hThr = OpenThread(THREAD_GET_CONTEXT, FALSE, tid.tid);

  if (hThr == NULL) {
-    win_print_last_error("OpenThread");
+    win_print_last_error("OpenThread set_regs_now");
    return false;
  }

@@ -649,7 +643,7 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) {
  CloseHandle(hThr);

  if (!result) {
-    win_print_last_error("GetThreadContext");
+    win_print_last_error("GetThreadContext set_regs_now");
    return false;
  }

@@ -674,7 +668,7 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) {
  hThr = OpenThread(THREAD_SET_CONTEXT, FALSE, tid.tid);

  if (hThr == NULL) {
-    win_print_last_error("OpenThread");
+    win_print_last_error("OpenThread set_regs_now set");
    return false;
  }

@@ -682,12 +676,67 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) {
  CloseHandle(hThr);

  if (!result) {
-    win_print_last_error("SetThreadContext");
+    win_print_last_error("SetThreadContext set_regs_now set");
    return false;
  }
  // todo, set fprs.
  return true;
 }
+#elif __APPLE__
+ThreadID::ThreadID(const std::string& str) {}
+
+std::string ThreadID::to_string() const {
+  return "invalid";
+}
+
+ThreadID get_current_thread_id() {
+  return ThreadID("not implemented on macOS");
+}
+
+bool attach_and_break(const ThreadID& tid);
+
+void allow_debugging() {
+  printf("allow_debugging not implemented on macOS\n");
+}
+
+bool detach_and_resume(const ThreadID& tid) {
+  return false;
+}
+bool get_regs_now(const ThreadID& tid, Regs* out) {
+  return false;
+}
+bool set_regs_now(const ThreadID& tid, const Regs& in) {
+  return false;
+}
+bool break_now(const ThreadID& tid) {
+  return false;
+}
+bool cont_now(const ThreadID& tid) {
+  return false;
+}
+bool open_memory(const ThreadID& tid, MemoryHandle* out);
+bool close_memory(const ThreadID& tid, MemoryHandle* handle) {
+  return false;
+}
+bool read_goal_memory(u8* dest_buffer,
+                      int size,
+                      u32 goal_addr,
+                      const DebugContext& context,
+                      const MemoryHandle& mem) {
+  return false;
+}
+
+bool write_goal_memory(const u8* src_buffer,
+                       int size,
+                       u32 goal_addr,
+                       const DebugContext& context,
+                       const MemoryHandle& mem) {
+  return false;
+}
+
+bool check_stopped(const ThreadID& tid, SignalInfo* out) {
+  return false;
+}
 #endif

 const char* gpr_names[] = {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
@@ -20,7 +20,7 @@
 #endif

 namespace xdbg {
-#ifdef __linux
+#ifdef OS_POSIX

 /*!
 * Identification for a thread.
@@ -4,7 +4,9 @@
 */

 // clang-format off
-#ifdef __linux
+#include "common/common_types.h"
+#ifdef OS_POSIX
+#include <arpa/inet.h>
 #include <netinet/tcp.h>
 #include <sys/socket.h>
 #include <unistd.h>
@@ -22,7 +24,7 @@
 // clang-format on

 int open_socket(int af, int type, int protocol) {
-#ifdef __linux
+#ifdef OS_POSIX
  return socket(af, type, protocol);
 #elif _WIN32
  WSADATA wsaData = {0};
@@ -45,7 +47,7 @@ int connect_socket(int socket, sockaddr* addr, int nameLen) {
  return result;
 }

-#ifdef __linux
+#ifdef OS_POSIX
 int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen) {
  return accept(socket, addr, addrLen);
 }
@@ -106,7 +108,7 @@ void close_socket(int sock) {
  if (sock < 0) {
    return;
  }
-#ifdef __linux
+#ifdef OS_POSIX
  close(sock);
 #elif _WIN32
  closesocket(sock);
@@ -130,7 +132,7 @@ int set_socket_option(int socket, int level, int optname, const void* optval, in
 }

 int set_socket_timeout(int socket, long microSeconds) {
-#ifdef __linux
+#ifdef OS_POSIX
  struct timeval timeout = {};
  timeout.tv_sec = 0;
  timeout.tv_usec = microSeconds;
@@ -142,13 +144,15 @@ int set_socket_timeout(int socket, long microSeconds) {
  return ret;
 #elif _WIN32
  DWORD timeout = microSeconds / 1000;  // milliseconds
+  // TODO - NOTE this might be bad / unreliable if the socket ends up being in a bad state
+  // select() instead should be used, will do so if ends up being an issue in practice
  return set_socket_option(socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
 #endif
 }

 int write_to_socket(int socket, const char* buf, int len) {
  int bytes_wrote = 0;
-#ifdef __linux
+#ifdef OS_POSIX
  bytes_wrote = send(socket, buf, len, MSG_NOSIGNAL);
 #elif _WIN32
  bytes_wrote = send(socket, buf, len, 0);
@@ -160,7 +164,7 @@ int write_to_socket(int socket, const char* buf, int len) {
 }

 int read_from_socket(int socket, char* buf, int len) {
-#ifdef __linux
+#ifdef OS_POSIX
  return read(socket, buf, len);
 #elif _WIN32
  return recv(socket, buf, len, 0);
@@ -168,10 +172,20 @@ int read_from_socket(int socket, char* buf, int len) {
 }

 bool socket_timed_out() {
-#ifdef __linux
+#ifdef OS_POSIX
  return errno == EAGAIN;
 #elif _WIN32
  auto err = WSAGetLastError();
  return err == WSAETIMEDOUT;
 #endif
 }
+
+std::string address_to_string(const sockaddr_in& addr) {
+  char ip_str[INET_ADDRSTRLEN];  // Buffer to hold the IP address string
+  // Convert binary IP address to human-readable string using inet_ntop
+  if (inet_ntop(AF_INET, &(addr.sin_addr), ip_str, INET_ADDRSTRLEN) == nullptr) {
+    // Handle the error (e.g., log, throw an exception, etc.)
+    return "Error: Unable to convert IP address";
+  }
+  return ip_str;
+}
@@ -6,7 +6,8 @@
 */

 // clang-format off
-#ifdef __linux
+#include "common/common_types.h"
+#ifdef OS_POSIX
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
@@ -24,11 +25,13 @@
 const int TCP_SOCKET_LEVEL = SOL_TCP;
 #elif _WIN32
 const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
+#elif __APPLE__
+const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
 #endif

 int open_socket(int af, int type, int protocol);
 int connect_socket(int socket, sockaddr* addr, int nameLen);
-#ifdef __linux
+#ifdef OS_POSIX
 int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen);
 int select_and_accept_socket(int socket, sockaddr* addr, socklen_t* addrLen, int microSeconds);
 #elif _WIN32
@@ -41,3 +44,4 @@ int set_socket_timeout(int socket, long microSeconds);
 int write_to_socket(int socket, const char* buf, int len);
 int read_from_socket(int socket, char* buf, int len);
 bool socket_timed_out();
+std::string address_to_string(const sockaddr_in& addr);
@@ -3,6 +3,7 @@
 #include <string>

 #include "common/cross_sockets/XSocket.h"
+#include "common/log/log.h"

 // clang-format off
 #ifdef _WIN32
@@ -12,7 +13,7 @@
 #include <WinSock2.h>
 #include <WS2tcpip.h>
 #endif
-#include "common/nrepl/ReplServer.h"
+#include "common/repl/nrepl/ReplServer.h"

 #include "third-party/fmt/core.h"
 // clang-format on
@@ -41,7 +42,11 @@ bool XSocketClient::connect() {
  }

  addr.sin_family = AF_INET;
-  addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+  // Convert IP address from text to binary form using inet_pton()
+  if (inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr) <= 0) {
+    lg::error("inet_pton() failed");
+    return false;
+  }
  addr.sin_port = htons(tcp_port);

  // Connect to server
@@ -2,6 +2,7 @@
 #include "XSocketServer.h"

 #include "common/cross_sockets/XSocket.h"
+#include "common/common_types.h"

 #include "third-party/fmt/core.h"

@@ -41,19 +42,26 @@ bool XSocketServer::init_server() {
    return false;
  }

-#ifdef __linux
-  int server_socket_opt = SO_REUSEADDR | SO_REUSEPORT;
-#elif _WIN32
-  int server_socket_opt = SO_EXCLUSIVEADDRUSE;
-#endif
+  int yes = 1;

-  int opt = 1;
-  if (set_socket_option(listening_socket, SOL_SOCKET, server_socket_opt, &opt, sizeof(opt)) < 0) {
+#ifdef OS_POSIX
+  if (set_socket_option(listening_socket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0) {
+    close_server_socket();
+    return false;
+  }
+  // macOS doesn't support setting multiple options at once, so we have to do this separately.
+  if (set_socket_option(listening_socket, SOL_SOCKET, SO_REUSEPORT, &yes, sizeof(yes)) < 0) {
+    close_server_socket();
+    return false;
+  }
+#elif _WIN32
+  if (set_socket_option(listening_socket, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, &yes, sizeof(yes)) < 0) {
    close_server_socket();
    return false;
  };
+#endif

-  if (set_socket_option(listening_socket, TCP_SOCKET_LEVEL, TCP_NODELAY, &opt, sizeof(opt)) < 0) {
+  if (set_socket_option(listening_socket, TCP_SOCKET_LEVEL, TCP_NODELAY, &yes, sizeof(yes)) < 0) {
    close_server_socket();
    return false;
  }
@@ -3,6 +3,7 @@
 #include <functional>
 #include <mutex>
 #include <thread>
+#include <vector>

 #include "common/common_types.h"
 #include "common/cross_sockets/XSocket.h"
@@ -3,6 +3,12 @@
 #include <algorithm>
 #include <functional>

+#ifndef __aarch64__
+#include "xmmintrin.h"
+#else
+#include "third-party/sse2neon/sse2neon.h"
+#endif
+
 #include "common/util/Assert.h"

 namespace tfrag3 {
@@ -36,6 +42,7 @@ void ShrubDraw::serialize(Serializer& ser) {
  ser.from_ptr(&num_triangles);
  ser.from_ptr(&first_index_index);
  ser.from_ptr(&num_indices);
+  ser.from_ptr(&proto_idx);
 }

 void InstancedStripDraw::serialize(Serializer& ser) {
@@ -72,6 +79,150 @@ void TfragTree::serialize(Serializer& ser) {
  ser.from_ptr(&use_strips);
 }

+math::Vector3f vopmula(math::Vector3f a, math::Vector3f b) {
+  return math::Vector3f(a.y() * b.z(), a.z() * b.x(), a.x() * b.y());
+}
+
+math::Vector3f vopmsub(math::Vector3f acc, math::Vector3f a, math::Vector3f b) {
+  return acc - vopmula(a, b);
+}
+
+/*!
+ * Compute the normal transformation for a TIE from the TIE matrix.
+ * Note that this isn't identical to the original game - we're missing the vf14 scaling factor
+ * For now, I just set this to 1, then normalize in the shader. Though I think we could avoid
+ * this by figuring out the value of vf14 here (I am just too lazy right now).
+ */
+std::array<math::Vector3f, 3> tie_normal_transform_v2(const std::array<math::Vector4f, 4>& m) {
+  // let:
+  // vf10, vf11, vf12, vf13 be the input matrix m
+  std::array<math::Vector3f, 3> result;
+  auto& vf10 = m[0];
+  auto& vf11 = m[1];
+  // auto& vf12 = m[2];
+
+  //  lui t6, 16256
+  //  mtc1 f1, t6 ;; 1.0
+  //
+  //  qmfc2.i s1, vf10
+  //  mtc1 f12, s1
+  float f12 = vf10.x();
+  //  dsra32 s2, s1, 0
+  //  mtc1 f13, s2
+  float f13 = vf10.y();
+  //  pextuw s2, r0, s2
+  //  mtc1 f14, s2
+  float f14 = vf10.z();
+  //  mula.s f12, f12
+  //  madda.s f13, f13
+  //  madd.s f15, f14, f14
+  float f15 = f12 * f12 + f13 * f13 + f14 * f14;
+  float scale = 1.f / sqrtf(f15);
+  //  rsqrt.s f15, f1, f15
+  //  mfc1 s1, f15
+  //  qmtc2.i vf14, s1
+  //  vmulx.xyz vf16, vf10, vf14
+
+  // vmulx.xyz vf16, vf10, vf14
+  math::Vector3f vf16 = vf10.xyz() * scale;
+
+  // vopmula.xyz acc, vf11, vf16
+  math::Vector3f acc = vopmula(vf11.xyz(), vf16);
+
+  // vopmsub.xyz vf17, vf16, vf11
+  math::Vector3f vf17 = vopmsub(acc, vf16, vf11.xyz());
+
+  // vopmula.xyz acc, vf16, vf17
+  acc = vopmula(vf16, vf17);
+
+  // vopmsub.xyz vf17, vf17, vf16
+  vf17 = vopmsub(acc, vf17, vf16);
+
+  // vmul.xyz vf14, vf17, vf17
+  math::Vector3f vf14 = vf17.elementwise_multiply(vf17);
+
+  // vmulax.w acc, vf0, vf14
+  // vmadday.w acc, vf0, vf14
+  // vmaddz.w vf14, vf0, vf14
+  float sum = vf14.x() + vf14.y() + vf14.z();
+
+  // vrsqrt Q, vf0.w, vf14.w
+  float Q = 1.f / std::sqrt(sum);
+
+  // vmulax.xyzw acc, vf24, vf16
+  // vmadday.xyzw acc, vf25, vf16
+  // vmaddz.xyzw vf10, vf26, vf16
+  // vf10 = vf16; // assume cam is identity here.
+  result[0] = vf16;
+
+  // vwaitq
+  // vmulq.xyz vf17, vf17, Q
+  vf17 *= Q;
+
+  // vopmula.xyz acc, vf16, vf17
+  acc = vopmula(vf16, vf17);
+  // vopmsub.xyz vf18, vf17, vf16
+  math::Vector3f vf18 = vopmsub(acc, vf17, vf16);
+
+  // vmulax.xyzw acc, vf24, vf17
+  // vmadday.xyzw acc, vf25, vf17
+  // vmaddz.xyzw vf11, vf26, vf17
+  result[1] = vf17;
+
+  // vmulax.xyzw acc, vf24, vf18
+  // vmadday.xyzw acc, vf25, vf18
+  // vmaddz.xyzw vf12, vf26, vf18
+  result[2] = vf18;
+
+  return result;
+  //
+  // sqc2 vf10, -112(t8)
+  // sqc2 vf11, -96(t8)
+  // sqc2 vf12, -80(t8)
+}
+
+u32 pack_to_gl_normal(s16 nx, s16 ny, s16 nz) {
+  ASSERT(nx >= -512 && nx <= 511);
+  ASSERT(ny >= -512 && ny <= 511);
+  ASSERT(nz >= -512 && nz <= 511);
+  return (nx & 0x3ff) | ((ny & 0x3ff) << 10) | ((nz & 0x3ff) << 20);
+}
+
+/*!
+ * Unpack tie normal by transforming and converting to s16 for OpenGL.
+ */
+u32 unpack_tie_normal(const std::array<math::Vector3f, 3>& mat, s8 nx, s8 ny, s8 nz) {
+  // rotate the normal
+  math::Vector3f nrm = math::Vector3f::zero();
+  nrm += mat[0] * nx;
+  nrm += mat[1] * ny;
+  nrm += mat[2] * nz;
+  // convert to s16 for OpenGL renderer
+  // nrm /= 0x100;  // number from EE asm
+  // nrm *= 0x200;  // for normalized s10 -> float conversion by OpenGL.
+  nrm *= 2;  // for normalized s10 -> float conversion by OpenGL.
+
+  auto as_int = nrm.cast<s16>();
+
+  return pack_to_gl_normal(as_int.x(), as_int.y(), as_int.z());
+}
+
+/*
+void tie_normal_v3(__m128* out, const std::array<math::Vector4f, 4>& in) {
+  math::Vector3f x_row = in[0].xyz();
+  math::Vector3f y_row = in[1].xyz();
+  math::Vector3f z_row = in[2].xyz();
+
+  x_row.normalize();
+  y_row = x_row.cross(y_row.cross(x_row)).normalized();
+  z_row = x_row.cross(y_row);
+
+  out[0] = _mm_setr_ps(x_row[0], x_row[1], x_row[2], 0);
+  out[1] = _mm_setr_ps(y_row[0], y_row[1], y_row[2], 0);
+  out[2] = _mm_setr_ps(z_row[0], z_row[1], z_row[2], 0);
+}
+ */
+
 void TieTree::unpack() {
  unpacked.vertices.resize(packed_vertices.color_indices.size());
  size_t i = 0;
@@ -84,25 +235,64 @@ void TieTree::unpack() {
        vtx.x = proto_vtx.x;
        vtx.y = proto_vtx.y;
        vtx.z = proto_vtx.z;
-        vtx.q_unused = 1.f;
        vtx.s = proto_vtx.s;
        vtx.t = proto_vtx.t;
+        vtx.nor = pack_to_gl_normal(proto_vtx.nx << 1, proto_vtx.ny << 1, proto_vtx.nz << 1);
+        vtx.r = proto_vtx.r;
+        vtx.g = proto_vtx.g;
+        vtx.b = proto_vtx.b;
+        vtx.a = proto_vtx.a;
        i++;
      }
    } else {
      const auto& mat = packed_vertices.matrices[grp.matrix_idx];
-      for (u32 src_idx = grp.start_vert; src_idx < grp.end_vert; src_idx++) {
-        auto& vtx = unpacked.vertices[i];
-        vtx.color_index = packed_vertices.color_indices[i];
-        const auto& proto_vtx = packed_vertices.vertices[src_idx];
-        auto temp = mat[0] * proto_vtx.x + mat[1] * proto_vtx.y + mat[2] * proto_vtx.z + mat[3];
-        vtx.x = temp.x();
-        vtx.y = temp.y();
-        vtx.z = temp.z();
-        vtx.q_unused = 1.f;
-        vtx.s = proto_vtx.s;
-        vtx.t = proto_vtx.t;
-        i++;
+
+      __m128 mat0 = _mm_loadu_ps(mat[0].data());
+      __m128 mat1 = _mm_loadu_ps(mat[1].data());
+      __m128 mat2 = _mm_loadu_ps(mat[2].data());
+      __m128 mat3 = _mm_loadu_ps(mat[3].data());
+
+      if (grp.has_normals) {
+        auto nmat = tie_normal_transform_v2(mat);
+        for (u32 src_idx = grp.start_vert; src_idx < grp.end_vert; src_idx++) {
+          auto& vtx = unpacked.vertices[i];
+          vtx.color_index = packed_vertices.color_indices[i];
+          const auto& proto_vtx = packed_vertices.vertices[src_idx];
+          // auto temp = mat[0] * proto_vtx.x + mat[1] * proto_vtx.y + mat[2] * proto_vtx.z +
+          // mat[3];
+          __m128 transformed = mat3;
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.x), mat0));
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.y), mat1));
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.z), mat2));
+          _mm_storeu_ps(&vtx.x, transformed);
+          vtx.s = proto_vtx.s;
+          vtx.t = proto_vtx.t;
+          vtx.nor = unpack_tie_normal(nmat, proto_vtx.nx, proto_vtx.ny, proto_vtx.nz);
+          vtx.r = proto_vtx.r;
+          vtx.g = proto_vtx.g;
+          vtx.b = proto_vtx.b;
+          vtx.a = proto_vtx.a;
+          i++;
+        }
+      } else {
+        for (u32 src_idx = grp.start_vert; src_idx < grp.end_vert; src_idx++) {
+          auto& vtx = unpacked.vertices[i];
+          vtx.color_index = packed_vertices.color_indices[i];
+          const auto& proto_vtx = packed_vertices.vertices[src_idx];
+          __m128 transformed = mat3;
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.x), mat0));
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.y), mat1));
+          transformed = _mm_add_ps(transformed, _mm_mul_ps(_mm_set1_ps(proto_vtx.z), mat2));
+          _mm_storeu_ps(&vtx.x, transformed);
+          vtx.s = proto_vtx.s;
+          vtx.t = proto_vtx.t;
+          vtx.nor = 0;
+          vtx.r = proto_vtx.r;
+          vtx.g = proto_vtx.g;
+          vtx.b = proto_vtx.b;
+          vtx.a = proto_vtx.a;
+          i++;
+        }
      }
    }
  }
@@ -159,7 +349,6 @@ void TfragTree::unpack() {
    o.z = cz + in.zoff * rescale;
    o.s = in.s / (1024.f);
    o.t = in.t / (1024.f);
-    o.q_unused = 1.f;
    o.color_index = in.color_index;
  }

@@ -188,6 +377,8 @@ void TieTree::serialize(Serializer& ser) {
    draw.serialize(ser);
  }

+  ser.from_ptr(&category_draw_indices);
+
  if (ser.is_saving()) {
    ser.save<size_t>(instanced_wind_draws.size());
  } else {
@@ -209,6 +400,9 @@ void TieTree::serialize(Serializer& ser) {
  packed_vertices.serialize(ser);
  ser.from_pod_vector(&colors);
  bvh.serialize(ser);
+
+  ser.from_ptr(&has_per_proto_visibility_toggle);
+  ser.from_string_vector(&proto_names);
 }

 void ShrubTree::serialize(Serializer& ser) {
@@ -223,6 +417,9 @@ void ShrubTree::serialize(Serializer& ser) {
  for (auto& draw : static_draws) {
    draw.serialize(ser);
  }
+
+  ser.from_ptr(&has_per_proto_visibility_toggle);
+  ser.from_string_vector(&proto_names);
 }

 void BVH::serialize(Serializer& ser) {
@@ -244,6 +441,17 @@ void Texture::serialize(Serializer& ser) {
  ser.from_ptr(&load_to_pool);
 }

+void IndexTexture::serialize(Serializer& ser) {
+  ser.from_ptr(&w);
+  ser.from_ptr(&h);
+  ser.from_ptr(&combo_id);
+  ser.from_pod_vector(&index_data);
+  ser.from_ptr(&color_table);
+  ser.from_str(&name);
+  ser.from_str(&tpage_name);
+  ser.from_string_vector(&level_names);
+}
+
 void CollisionMesh::serialize(Serializer& ser) {
  ser.from_pod_vector(&vertices);
 }
@@ -251,20 +459,59 @@ void CollisionMesh::serialize(Serializer& ser) {
 void MercDraw::serialize(Serializer& ser) {
  ser.from_ptr(&mode);
  ser.from_ptr(&tree_tex_id);
+  ser.from_ptr(&eye_id);
  ser.from_ptr(&first_index);
  ser.from_ptr(&index_count);
  ser.from_ptr(&num_triangles);
 }

-void MercEffect::serialize(Serializer& ser) {
+void Blerc::serialize(Serializer& ser) {
+  ser.from_pod_vector(&float_data);
+  ser.from_pod_vector(&int_data);
+}
+
+void MercModifiableDrawGroup::serialize(Serializer& ser) {
  if (ser.is_saving()) {
-    ser.save<size_t>(draws.size());
+    ser.save<size_t>(mod_draw.size());
  } else {
-    draws.resize(ser.load<size_t>());
+    mod_draw.resize(ser.load<size_t>());
  }
-  for (auto& draw : draws) {
+  for (auto& draw : mod_draw) {
    draw.serialize(ser);
  }
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(fix_draw.size());
+  } else {
+    fix_draw.resize(ser.load<size_t>());
+  }
+  for (auto& draw : fix_draw) {
+    draw.serialize(ser);
+  }
+  ser.from_pod_vector(&vertices);
+  ser.from_pod_vector(&vertex_lump4_addr);
+  ser.from_pod_vector(&fragment_mask);
+  ser.from_ptr(&expect_vidx_end);
+
+  blerc.serialize(ser);
+}
+
+void MercEffect::serialize(Serializer& ser) {
+  if (ser.is_saving()) {
+    ser.save<size_t>(all_draws.size());
+  } else {
+    all_draws.resize(ser.load<size_t>());
+  }
+  for (auto& draw : all_draws) {
+    draw.serialize(ser);
+  }
+
+  mod.serialize(ser);
+
+  ser.from_ptr(&envmap_mode);
+  ser.from_ptr(&envmap_texture);
+  ser.from_ptr(&has_envmap);
+  ser.from_ptr(&has_mod_draw);
 }

 void MercModel::serialize(Serializer& ser) {
@@ -279,6 +526,9 @@ void MercModel::serialize(Serializer& ser) {
  }
  ser.from_ptr(&max_draws);
  ser.from_ptr(&max_bones);
+  ser.from_ptr(&st_vif_add);
+  ser.from_ptr(&xyz_scale);
+  ser.from_ptr(&st_magic);
 }

 void MercModelGroup::serialize(Serializer& ser) {
@@ -314,6 +564,15 @@ void Level::serialize(Serializer& ser) {
    tex.serialize(ser);
  }

+  if (ser.is_saving()) {
+    ser.save<size_t>(index_textures.size());
+  } else {
+    index_textures.resize(ser.load<size_t>());
+  }
+  for (auto& tex : index_textures) {
+    tex.serialize(ser);
+  }
+
  for (int geom = 0; geom < 3; ++geom) {
    if (ser.is_saving()) {
      ser.save<size_t>(tfrag_trees[geom].size());
@@ -356,109 +615,175 @@ void Level::serialize(Serializer& ser) {
  }
 }

-std::array<int, MemoryUsageCategory::NUM_CATEGORIES> Level::get_memory_usage() const {
-  std::array<int, MemoryUsageCategory::NUM_CATEGORIES> result;
-  result.fill(0);
+void MercModifiableDrawGroup::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::MERC_MOD_VERT, sizeof(MercVertex) * vertices.size());
+  tracker->add(MemoryUsageCategory::MERC_MOD_DRAW_1, sizeof(MercDraw) * fix_draw.size());
+  tracker->add(MemoryUsageCategory::MERC_MOD_DRAW_2, sizeof(MercDraw) * mod_draw.size());
+  tracker->add(MemoryUsageCategory::MERC_MOD_TABLE, sizeof(u16) * vertex_lump4_addr.size());
+  tracker->add(MemoryUsageCategory::BLERC, sizeof(BlercFloatData) * blerc.float_data.size());
+  tracker->add(MemoryUsageCategory::BLERC, sizeof(u32) * blerc.int_data.size());
+}

-  // textures
-  for (const auto& tex : textures) {
-    result[TEXTURE] += tex.data.size() * sizeof(u32);
+void MercEffect::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::MERC_DRAW, sizeof(MercDraw) * all_draws.size());
+  mod.memory_usage(tracker);
+}
+
+void MercModel::memory_usage(MemoryUsageTracker* tracker) const {
+  for (auto& effect : effects) {
+    effect.memory_usage(tracker);
  }
+}

-  // tfrag
-  for (const auto& tfrag_tree_geoms : tfrag_trees) {
-    for (const auto& tfrag_tree : tfrag_tree_geoms) {
-      for (const auto& draw : tfrag_tree.draws) {
-        result[TFRAG_INDEX] += draw.runs.size() * sizeof(StripDraw::VertexRun);
-        result[TFRAG_INDEX] += draw.plain_indices.size() * sizeof(u32);
-        result[TFRAG_VIS] += draw.vis_groups.size() * sizeof(StripDraw::VisGroup);
-      }
-      result[TFRAG_VERTS] +=
-          tfrag_tree.packed_vertices.vertices.size() * sizeof(PackedTfragVertices::Vertex);
-      result[TFRAG_CLUSTER] +=
-          tfrag_tree.packed_vertices.cluster_origins.size() * sizeof(math::Vector<u16, 3>);
-      result[TFRAG_TIME_OF_DAY] += tfrag_tree.colors.size() * sizeof(TimeOfDayColor);
-      result[TFRAG_BVH] += tfrag_tree.bvh.vis_nodes.size() * sizeof(VisNode);
+void MercModelGroup::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::MERC_VERT, sizeof(MercVertex) * vertices.size());
+  tracker->add(MemoryUsageCategory::MERC_INDEX, sizeof(u32) * indices.size());
+  for (auto& model : models) {
+    model.memory_usage(tracker);
+  }
+}
+
+void CollisionMesh::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::COLLISION, sizeof(Vertex) * vertices.size());
+}
+
+void PackedShrubVertices::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::SHRUB_VERT, 64 * matrices.size());
+  tracker->add(MemoryUsageCategory::SHRUB_VERT, sizeof(InstanceGroup) * instance_groups.size());
+  tracker->add(MemoryUsageCategory::SHRUB_VERT, sizeof(Vertex) * vertices.size());
+}
+
+void ShrubTree::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY,
+               sizeof(TimeOfDayColor) * time_of_day_colors.size());
+  packed_vertices.memory_usage(tracker);
+  tracker->add(MemoryUsageCategory::SHRUB_DRAW, sizeof(ShrubDraw) * static_draws.size());
+  tracker->add(MemoryUsageCategory::SHRUB_IND, sizeof(u32) * indices.size());
+}
+
+void InstancedStripDraw::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::TIE_INST_INDEX, sizeof(u32) * vertex_index_stream.size());
+  tracker->add(MemoryUsageCategory::TIE_INST_VIS, sizeof(InstanceGroup) * instance_groups.size());
+}
+
+void PackedTieVertices::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::TIE_CIDX, sizeof(u16) * color_indices.size());
+  tracker->add(MemoryUsageCategory::TIE_MATRICES, 64 * matrices.size());
+  tracker->add(MemoryUsageCategory::TIE_GRPS, sizeof(MatrixGroup) * matrix_groups.size());
+  tracker->add(MemoryUsageCategory::TIE_VERTS, sizeof(Vertex) * vertices.size());
+}
+
+void TieTree::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::TIE_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
+  for (auto& draw : static_draws) {
+    tracker->add(MemoryUsageCategory::TIE_DEINST_INDEX,
+                 draw.runs.size() * sizeof(StripDraw::VertexRun));
+    tracker->add(MemoryUsageCategory::TIE_DEINST_INDEX, draw.plain_indices.size() * sizeof(u32));
+    tracker->add(MemoryUsageCategory::TIE_DEINST_VIS,
+                 draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
+  }
+  packed_vertices.memory_usage(tracker);
+  tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+
+  for (auto& draw : instanced_wind_draws) {
+    draw.memory_usage(tracker);
+  }
+  tracker->add(MemoryUsageCategory::TIE_WIND_INSTANCE_INFO,
+               sizeof(TieWindInstance) * wind_instance_info.size());
+}
+
+void PackedTfragVertices::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::TFRAG_VERTS,
+               sizeof(PackedTfragVertices::Vertex) * vertices.size());
+  tracker->add(MemoryUsageCategory::TFRAG_CLUSTER,
+               sizeof(math::Vector<u16, 3>) * cluster_origins.size());
+}
+
+void TfragTree::memory_usage(MemoryUsageTracker* tracker) const {
+  for (auto& draw : draws) {
+    tracker->add(MemoryUsageCategory::TFRAG_INDEX, draw.runs.size() * sizeof(StripDraw::VertexRun));
+    tracker->add(MemoryUsageCategory::TFRAG_INDEX, draw.plain_indices.size() * sizeof(u32));
+    tracker->add(MemoryUsageCategory::TFRAG_VIS,
+                 draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
+  }
+  packed_vertices.memory_usage(tracker);
+  tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+  tracker->add(MemoryUsageCategory::TFRAG_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
+}
+
+void Texture::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::TEXTURE, data.size() * sizeof(u32));
+}
+
+void IndexTexture::memory_usage(MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::SPECIAL_TEXTURE, index_data.size());
+  tracker->add(MemoryUsageCategory::SPECIAL_TEXTURE, 256 * 4);  // clut
+}
+
+void Level::memory_usage(MemoryUsageTracker* tracker) const {
+  for (const auto& texture : textures) {
+    texture.memory_usage(tracker);
+  }
+  for (const auto& texture : index_textures) {
+    texture.memory_usage(tracker);
+  }
+  for (const auto& tftk : tfrag_trees) {
+    for (const auto& tree : tftk) {
+      tree.memory_usage(tracker);
    }
  }
-
-  // tie
-  for (const auto& tie_tree_geoms : tie_trees) {
-    for (const auto& tie_tree : tie_tree_geoms) {
-      result[TIE_BVH] += tie_tree.bvh.vis_nodes.size();
-      for (const auto& draw : tie_tree.static_draws) {
-        result[TIE_DEINST_INDEX] += draw.runs.size() * sizeof(StripDraw::VertexRun);
-        result[TIE_DEINST_VIS] += draw.vis_groups.size() * sizeof(StripDraw::VisGroup);
-      }
-      result[TIE_VERTS] +=
-          tie_tree.packed_vertices.vertices.size() * sizeof(PackedTieVertices::Vertex);
-      result[TIE_CIDX] += tie_tree.packed_vertices.color_indices.size() * sizeof(u16);
-      result[TIE_MATRICES] += tie_tree.packed_vertices.matrices.size() * 4 * 4 * 4;
-      result[TIE_GRPS] +=
-          tie_tree.packed_vertices.matrix_groups.size() * sizeof(PackedTieVertices::MatrixGroup);
-      result[TIE_TIME_OF_DAY] += tie_tree.colors.size() * sizeof(TimeOfDayColor);
-
-      for (const auto& draw : tie_tree.instanced_wind_draws) {
-        result[TIE_INST_INDEX] += draw.vertex_index_stream.size() * sizeof(u32);
-        result[TIE_INST_VIS] +=
-            draw.instance_groups.size() * sizeof(InstancedStripDraw::InstanceGroup);
-      }
-      result[TIE_WIND_INSTANCE_INFO] +=
-          tie_tree.wind_instance_info.size() * sizeof(TieWindInstance);
+  for (const auto& ttk : tie_trees) {
+    for (const auto& tree : ttk) {
+      tree.memory_usage(tracker);
    }
  }
-
-  // shrub
-  for (const auto& shrub_tree : shrub_trees) {
-    result[SHRUB_TIME_OF_DAY] += shrub_tree.time_of_day_colors.size() * sizeof(TimeOfDayColor);
-    result[SHRUB_VERT] += shrub_tree.packed_vertices.matrices.size() * 4 * 4 * 4;
-    result[SHRUB_VERT] +=
-        shrub_tree.packed_vertices.vertices.size() * sizeof(PackedShrubVertices::Vertex);
-    result[SHRUB_VERT] += shrub_tree.packed_vertices.instance_groups.size() *
-                          sizeof(PackedShrubVertices::InstanceGroup);
-    result[SHRUB_IND] += sizeof(u32) * shrub_tree.indices.size();
+  for (const auto& tree : shrub_trees) {
+    tree.memory_usage(tracker);
  }
-
-  // merc
-  result[MERC_INDEX] += merc_data.indices.size() * sizeof(u32);
-  result[MERC_VERT] += merc_data.vertices.size() * sizeof(MercVertex);
-
-  // collision
-  result[COLLISION] += sizeof(CollisionMesh::Vertex) * collision.vertices.size();
-
-  return result;
+  collision.memory_usage(tracker);
+  merc_data.memory_usage(tracker);
 }

 void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
  int total_accounted = 0;
-  auto memory_use_by_category = lev.get_memory_usage();
+  MemoryUsageTracker mem_use;
+  lev.memory_usage(&mem_use);

  std::vector<std::pair<std::string, int>> known_categories = {
-      {"texture", memory_use_by_category[tfrag3::MemoryUsageCategory::TEXTURE]},
-      {"tie-deinst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_VIS]},
-      {"tie-deinst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_INDEX]},
-      {"tie-inst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_VIS]},
-      {"tie-inst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_INDEX]},
-      {"tie-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_BVH]},
-      {"tie-verts", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_VERTS]},
-      {"tie-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_TIME_OF_DAY]},
-      {"tie-wind-inst-info",
-       memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_WIND_INSTANCE_INFO]},
-      {"tie-cidx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_CIDX]},
-      {"tie-mats", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_MATRICES]},
-      {"tie-grps", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_GRPS]},
-      {"tfrag-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VIS]},
-      {"tfrag-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_INDEX]},
-      {"tfrag-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VERTS]},
-      {"tfrag-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_TIME_OF_DAY]},
-      {"tfrag-cluster", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_CLUSTER]},
-      {"tfrag-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_BVH]},
-      {"shrub-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_TIME_OF_DAY]},
-      {"shrub-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_VERT]},
-      {"shrub-ind", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_IND]},
-      {"collision", memory_use_by_category[tfrag3::MemoryUsageCategory::COLLISION]},
-      {"merc-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_VERT]},
-      {"merc-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_INDEX]}};
+      {"texture", mem_use.data[tfrag3::MemoryUsageCategory::TEXTURE]},
+      {"special-texture", mem_use.data[tfrag3::MemoryUsageCategory::SPECIAL_TEXTURE]},
+      {"tie-deinst-vis", mem_use.data[tfrag3::MemoryUsageCategory::TIE_DEINST_VIS]},
+      {"tie-deinst-idx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_DEINST_INDEX]},
+      {"tie-inst-vis", mem_use.data[tfrag3::MemoryUsageCategory::TIE_INST_VIS]},
+      {"tie-inst-idx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_INST_INDEX]},
+      {"tie-bvh", mem_use.data[tfrag3::MemoryUsageCategory::TIE_BVH]},
+      {"tie-verts", mem_use.data[tfrag3::MemoryUsageCategory::TIE_VERTS]},
+      {"tie-colors", mem_use.data[tfrag3::MemoryUsageCategory::TIE_TIME_OF_DAY]},
+      {"tie-wind-inst-info", mem_use.data[tfrag3::MemoryUsageCategory::TIE_WIND_INSTANCE_INFO]},
+      {"tie-cidx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_CIDX]},
+      {"tie-mats", mem_use.data[tfrag3::MemoryUsageCategory::TIE_MATRICES]},
+      {"tie-grps", mem_use.data[tfrag3::MemoryUsageCategory::TIE_GRPS]},
+      {"tfrag-vis", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_VIS]},
+      {"tfrag-idx", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_INDEX]},
+      {"tfrag-vert", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_VERTS]},
+      {"tfrag-colors", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_TIME_OF_DAY]},
+      {"tfrag-cluster", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_CLUSTER]},
+      {"tfrag-bvh", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_BVH]},
+      {"shrub-colors", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_TIME_OF_DAY]},
+      {"shrub-vert", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_VERT]},
+      {"shrub-ind", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_IND]},
+      {"shrub-draw", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_DRAW]},
+      {"collision", mem_use.data[tfrag3::MemoryUsageCategory::COLLISION]},
+      {"merc-vert", mem_use.data[tfrag3::MemoryUsageCategory::MERC_VERT]},
+      {"merc-idx", mem_use.data[tfrag3::MemoryUsageCategory::MERC_INDEX]},
+      {"merc-draw", mem_use.data[tfrag3::MemoryUsageCategory::MERC_DRAW]},
+      {"merc-mod-vert", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_VERT]},
+      {"merc-mod-ind", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_IND]},
+      {"merc-mod-table", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_TABLE]},
+      {"merc-mod-draw-1", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_1]},
+      {"merc-mod-draw-2", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_2]},
+      {"blerc", mem_use.data[tfrag3::MemoryUsageCategory::BLERC]},
+  };
  for (auto& known : known_categories) {
    total_accounted += known.second;
  }
@@ -469,8 +794,10 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
            [](const auto& a, const auto& b) { return a.second > b.second; });

  for (const auto& x : known_categories) {
-    fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
-               100.f * (float)x.second / uncompressed_data_size);
+    if (x.second) {
+      fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
+                 100.f * (float)x.second / uncompressed_data_size);
+    }
  }
 }

@@ -18,9 +18,13 @@ namespace tfrag3 {
 // - if changing any large things (vertices, vis, bvh, colors, textures) update get_memory_usage
 // - if adding a new category to the memory usage, update extract_level to print it.

+constexpr int TFRAG3_VERSION = 39;
+
 enum MemoryUsageCategory {
  TEXTURE,

+  SPECIAL_TEXTURE,
+
  TIE_DEINST_VIS,
  TIE_DEINST_INDEX,
  TIE_INST_VIS,
@@ -44,26 +48,51 @@ enum MemoryUsageCategory {
  SHRUB_TIME_OF_DAY,
  SHRUB_VERT,
  SHRUB_IND,
+  SHRUB_DRAW,

  MERC_VERT,
  MERC_INDEX,
+  MERC_DRAW,
+
+  MERC_MOD_DRAW_1,
+  MERC_MOD_DRAW_2,
+  MERC_MOD_VERT,
+  MERC_MOD_IND,
+  MERC_MOD_TABLE,
+  BLERC,

  COLLISION,

  NUM_CATEGORIES
 };

-constexpr int TFRAG3_VERSION = 21;
+struct MemoryUsageTracker {
+  u32 data[MemoryUsageCategory::NUM_CATEGORIES];
+
+  MemoryUsageTracker() {
+    for (auto& x : data) {
+      x = 0;
+    }
+  }
+
+  void add(MemoryUsageCategory category, u32 size_bytes) { data[category] += size_bytes; }
+};

 // These vertices should be uploaded to the GPU at load time and don't change
 struct PreloadedVertex {
  // the vertex position
-  float x, y, z;
+  float x = 0, y = 0, z = 0;
+  // envmap tint color, not used in == or hash.
+  u8 r = 0, g = 0, b = 0, a = 0;
  // texture coordinates
-  float s, t, q_unused;
+  float s = 0, t = 0;
+
+  // not used in == or hash!!
+  // note that this is a 10-bit 3-element field packed into 32-bits.
+  u32 nor = 0;
+
  // color table index
-  u16 color_index;
-  u16 pad[3];
+  u16 color_index = 0;

  struct hash {
    std::size_t operator()(const PreloadedVertex& x) const;
@@ -80,12 +109,15 @@ struct PackedTieVertices {
  struct Vertex {
    float x, y, z;
    float s, t;
+    s8 nx, ny, nz;
+    u8 r, g, b, a;
  };

  struct MatrixGroup {
    s32 matrix_idx;
    u32 start_vert;
    u32 end_vert;
+    bool has_normals = false;
  };

  std::vector<u16> color_indices;
@@ -93,6 +125,7 @@ struct PackedTieVertices {
  std::vector<MatrixGroup> matrix_groups;  // todo pack
  std::vector<Vertex> vertices;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 struct PackedTfragVertices {
@@ -102,7 +135,7 @@ struct PackedTfragVertices {
    s16 s, t;
    u16 color_index;
  };
-
+  void memory_usage(MemoryUsageTracker* tracker) const;
  std::vector<Vertex> vertices;
  std::vector<math::Vector<u16, 3>> cluster_origins;
 };
@@ -135,7 +168,7 @@ struct PackedShrubVertices {
  std::vector<InstanceGroup> instance_groups;  // todo pack
  std::vector<Vertex> vertices;
  u32 total_vertex_count;
-
+  void memory_usage(MemoryUsageTracker* tracker) const;
  void serialize(Serializer& ser);
 };

@@ -147,7 +180,7 @@ struct PackedShrubVertices {
 // check visibility.
 struct StripDraw {
  DrawMode mode;        // the OpenGL draw settings.
-  u32 tree_tex_id = 0;  // the texture that should be bound for the draw
+  s32 tree_tex_id = 0;  // the texture that should be bound for the draw (negative for anim slot)

  struct {
    u32 idx_of_first_idx_in_full_buffer = 0;
@@ -167,7 +200,8 @@ struct StripDraw {
  struct VisGroup {
    u32 num_inds = 0;           // number of vertex indices in this group
    u32 num_tris = 0;           // number of triangles
-    u32 vis_idx_in_pc_bvh = 0;  // the visibility group they belong to (in BVH)
+    u16 vis_idx_in_pc_bvh = 0;  // the visibility group they belong to (in BVH)
+    u16 tie_proto_idx = 0;      // index of tie proto (tie only)
  };
  std::vector<VisGroup> vis_groups;

@@ -185,6 +219,8 @@ struct ShrubDraw {

  // for debug counting.
  u32 num_triangles = 0;
+
+  u16 proto_idx = 0;
  void serialize(Serializer& ser);
 };

@@ -207,6 +243,7 @@ struct InstancedStripDraw {
  // for debug counting.
  u32 num_triangles = 0;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 // node in the BVH.
@@ -256,6 +293,19 @@ struct Texture {
  std::string debug_tpage_name;
  bool load_to_pool = false;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
+};
+
+struct IndexTexture {
+  u16 w, h;
+  u32 combo_id = 0;
+  std::vector<u8> index_data;
+  std::vector<std::string> level_names;
+  std::string name;
+  std::string tpage_name;
+  std::array<math::Vector4<u8>, 256> color_table;
+  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 // Tfrag trees have several kinds:
@@ -279,6 +329,7 @@ struct TfragTree {
  } unpacked;
  void unpack();
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 struct TieWindInstance {
@@ -288,10 +339,66 @@ struct TieWindInstance {
  void serialize(Serializer& ser);
 };

+// Tie draws are split into categories.
+enum class TieCategory {
+  // normal tie buckets
+  NORMAL,
+  TRANS,  // also called alpha
+  WATER,
+
+  // first draw (normal base draw) for envmapped stuff
+  NORMAL_ENVMAP,
+  TRANS_ENVMAP,
+  WATER_ENVMAP,
+
+  // second draw (shiny) for envmapped ties.
+  NORMAL_ENVMAP_SECOND_DRAW,
+  TRANS_ENVMAP_SECOND_DRAW,
+  WATER_ENVMAP_SECOND_DRAW,
+};
+constexpr int kNumTieCategories = 9;
+
+constexpr bool is_envmap_first_draw_category(tfrag3::TieCategory category) {
+  switch (category) {
+    case tfrag3::TieCategory::NORMAL_ENVMAP:
+    case tfrag3::TieCategory::WATER_ENVMAP:
+    case tfrag3::TieCategory::TRANS_ENVMAP:
+      return true;
+    default:
+      return false;
+  }
+}
+
+constexpr bool is_envmap_second_draw_category(tfrag3::TieCategory category) {
+  switch (category) {
+    case tfrag3::TieCategory::NORMAL_ENVMAP_SECOND_DRAW:
+    case tfrag3::TieCategory::WATER_ENVMAP_SECOND_DRAW:
+    case tfrag3::TieCategory::TRANS_ENVMAP_SECOND_DRAW:
+      return true;
+    default:
+      return false;
+  }
+}
+
+constexpr TieCategory get_second_draw_category(tfrag3::TieCategory category) {
+  switch (category) {
+    case TieCategory::NORMAL_ENVMAP:
+      return TieCategory::NORMAL_ENVMAP_SECOND_DRAW;
+    case TieCategory::TRANS_ENVMAP:
+      return TieCategory::TRANS_ENVMAP_SECOND_DRAW;
+    case TieCategory::WATER_ENVMAP:
+      return TieCategory::WATER_ENVMAP_SECOND_DRAW;
+    default:
+      return TieCategory::NORMAL_ENVMAP;
+  }
+}
+
 // A tie model
 struct TieTree {
  BVH bvh;
-  std::vector<StripDraw> static_draws;  // the actual topology and settings
+  std::vector<StripDraw> static_draws;
+  // Category n uses draws: static_draws[cdi[n]] to static_draws[cdi[n + 1]]
+  std::array<u32, kNumTieCategories + 1> category_draw_indices;

  PackedTieVertices packed_vertices;
  std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
@@ -299,12 +406,17 @@ struct TieTree {
  std::vector<InstancedStripDraw> instanced_wind_draws;
  std::vector<TieWindInstance> wind_instance_info;

+  // jak 2 and later can toggle on and off visibility per proto by name
+  bool has_per_proto_visibility_toggle = false;
+  std::vector<std::string> proto_names;
+
  struct {
    std::vector<PreloadedVertex> vertices;  // mesh vertices
    std::vector<u32> indices;
  } unpacked;

  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
  void unpack();
 };

@@ -320,7 +432,12 @@ struct ShrubTree {
    std::vector<ShrubGpuVertex> vertices;  // mesh vertices
  } unpacked;

+  // jak 2 and later can toggle on and off visibility per proto by name
+  bool has_per_proto_visibility_toggle = false;
+  std::vector<std::string> proto_names;
+
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
  void unpack();
 };

@@ -336,12 +453,13 @@ struct CollisionMesh {
  static_assert(sizeof(Vertex) == 32);
  std::vector<Vertex> vertices;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 // MERC

 struct MercVertex {
-  float pos[3];
+  alignas(32) float pos[3];
  float pad0;

  float normal[3];
@@ -360,17 +478,59 @@ static_assert(sizeof(MercVertex) == 64);

 struct MercDraw {
  DrawMode mode;
-  u32 tree_tex_id = 0;  // the texture that should be bound for the draw
-
+  s32 tree_tex_id = 0;  // the texture that should be bound for the draw (negative for anim slot)
+  u8 eye_id = 0xff;     // 0xff if not eyes, (slot << 1) | (is_r)
  u32 first_index;
  u32 index_count;
  u32 num_triangles;
  void serialize(Serializer& ser);
 };

-struct MercEffect {
-  std::vector<MercDraw> draws;
+struct BlercFloatData {
+  // [x, y, z, pad, nx, ny, nz, pad]
+  // note that this should match the layout of the merc vertex above
+  alignas(32) float v[8];
+};
+
+/*!
+ * Data to modify vertices based on blend shapes.
+ */
+struct Blerc {
+  std::vector<BlercFloatData> float_data;
+  std::vector<u32> int_data;
+  static constexpr u32 kTargetIdxTerminator = UINT32_MAX;
  void serialize(Serializer& ser);
+
+  // int data, per vertex:
+  // [tgt0_idx, tgt1_idx, ..., terminator, dest]
+  // float data, per vertex:
+  // [base, tgt0, tgt1, ...]
+
+  // final vertex position is:
+  // base + sum(tgtn * weights[tgtn_idx])
+};
+
+struct MercModifiableDrawGroup {
+  std::vector<MercVertex> vertices;
+  std::vector<u16> vertex_lump4_addr;
+  std::vector<MercDraw> fix_draw, mod_draw;
+  std::vector<u8> fragment_mask;
+  Blerc blerc;
+  u32 expect_vidx_end = 0;
+
+  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
+};
+
+struct MercEffect {
+  std::vector<MercDraw> all_draws;
+  MercModifiableDrawGroup mod;
+  DrawMode envmap_mode;
+  u32 envmap_texture;
+  bool has_envmap = false;
+  bool has_mod_draw = false;
+  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 struct MercModel {
@@ -378,7 +538,11 @@ struct MercModel {
  std::vector<MercEffect> effects;
  u32 max_draws;
  u32 max_bones;
+  u32 st_vif_add;
+  float xyz_scale;
+  float st_magic;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 struct MercModelGroup {
@@ -386,6 +550,7 @@ struct MercModelGroup {
  std::vector<u32> indices;
  std::vector<MercModel> models;
  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 //
@@ -397,6 +562,7 @@ struct Level {
  u16 version = TFRAG3_VERSION;
  std::string level_name;
  std::vector<Texture> textures;
+  std::vector<IndexTexture> index_textures;
  std::array<std::vector<TfragTree>, TFRAG_GEOS> tfrag_trees;
  std::array<std::vector<TieTree>, TIE_GEOS> tie_trees;
  std::vector<ShrubTree> shrub_trees;
@@ -404,8 +570,7 @@ struct Level {
  MercModelGroup merc_data;
  u16 version2 = TFRAG3_VERSION;
  void serialize(Serializer& ser);
-
-  std::array<int, MemoryUsageCategory::NUM_CATEGORIES> get_memory_usage() const;
+  void memory_usage(MemoryUsageTracker* tracker) const;
 };

 void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size);
@@ -113,6 +113,7 @@ struct VifCode {
    ITOP = 0b100,
    STMOD = 0b101,
    PC_PORT = 0b1000,  // not a valid PS2 VIF code, but we use this to signal PC-PORT specific stuff
+    PC_PORT2 = 0b1001,
    MSK3PATH = 0b110,
    MARK = 0b111,
    FLUSHE = 0b10000,
@@ -106,6 +106,17 @@ class DmaFollower {
    return result;
  }

+  DmaTransfer advance_and_print_dma(DmaFollower& dma) {
+    auto data = dma.read_and_advance();
+    printf(
+        "dma transfer:\n%ssize: %d\nvif0: %s, data: %d\nvif1: %s, data: %d, imm: "
+        "%d\n\n",
+        dma.current_tag().print().c_str(), data.size_bytes, data.vifcode0().print().c_str(),
+        data.vif0(), data.vifcode1().print().c_str(), data.vifcode1().num,
+        data.vifcode1().immediate);
+    return data;
+  }
+
  DmaTag current_tag() const { return DmaTag(read_val<u64>(m_tag_offset)); }
  u32 current_tag_vif0() const { return read_val<u32>(m_tag_offset + 8); }
  u32 current_tag_vif1() const { return read_val<u32>(m_tag_offset + 12); }
@@ -3,6 +3,7 @@
 #include "common/util/Assert.h"

 #include "third-party/fmt/core.h"
+#include "third-party/fmt/format.h"

 std::string reg_descriptor_name(GifTag::RegisterDescriptor reg) {
  switch (reg) {
@@ -275,6 +276,7 @@ std::string GsTest::print() const {
        ASSERT(false);
    }
  }
+  result += '\n';
  return result;
 }

@@ -352,15 +354,17 @@ std::string GsTexa::print() const {
 std::string GsTex0::print() const {
  return fmt::format(
      "tbp0: {} tbw: {} psm: {} tw: {} th: {} tcc: {} tfx: {} cbp: {} cpsm: {} csm: {}\n", tbp0(),
-      tbw(), psm(), tw(), th(), tcc(), tfx(), cbp(), cpsm(), csm());
+      tbw(), fmt::underlying(psm()), tw(), th(), tcc(), fmt::underlying(tfx()), cbp(), cpsm(),
+      csm());
 }

 std::string GsPrim::print() const {
-  return fmt::format("0x{:x}, kind {}\n", data, kind());
+  return fmt::format("0x{:x}, kind {}\n", data, fmt::underlying(kind()));
 }

 std::string GsFrame::print() const {
-  return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), psm(), fbmsk());
+  return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), fmt::underlying(psm()),
+                     fbmsk());
 }

 std::string GsXYOffset::print() const {
@@ -424,7 +424,7 @@ class DrawMode {
    SRC_DST_FIX_DST = 4,  // fix = 64
    ZERO_SRC_SRC_DST = 5,
    SRC_SRC_SRC_SRC = 6,
-    SRC_0_DST_DST = 7
+    SRC_0_DST_DST = 7  // Note: requires color_mult tricks
  };

  enum class AlphaTest {
@@ -0,0 +1,186 @@
+# OpenGOAL Formatter Documentation
+
+> [!IMPORTANT]
+> This is still in a highly experimental state, adjust your expectations accordingly.
+
+Despite LISP being very simple, and taking the stance of a highly opinionated formatter that doesn't concern itself with line-length in most situations writing a code formatter is still an incredibly annoying and difficult thing to do.  This documentation serves as a place for me to collect my thoughts and explain how it works (or more likely, how it does not).
+
+Also in general I found it _really_ hard to find modern and easy to understand articles on how to get started (with the exception of zprint's usage docs, but that doesn't really cover implementation ideas).  This area is either dominated by 40 year old journal articles, or discussions around really difficult formatting projects.  There isn't a whole lot in the vain of _here's how I made a decent, simple formatter_, so maybe it'll help someone else (even if it's teaching them potentially what not to do!)
+
+## Architecture / Overview
+
+Atleast for me, it helps to understand the path code takes as it flows through the formatting process.  Originally I tried to do as much at once for sake of efficiency but this just makes things incredibly hard to reason about and fix bugs.  So like any problem that is complex, break it down, the formatter goes through many distinct phases chipping away at the problem:
+
+```mermaid
+%%{init: {'theme': 'dark', "flowchart" : { "curve" : "basis" } } }%%
+flowchart TB
+  subgraph top1 [Build a Formatting Tree]
+    direction TB
+    a1(AST Parse) --> a2(Consolidate AST Nodes)
+    a1 --> a3(Collect Metadata)
+  end
+
+  subgraph top2 [Apply Formatting Configuration]
+    direction TB
+    b1(Recursively Iterate each Node) --> b2{Form head has\na predefined config?}
+    b2 --> |yes|b3(Fetch it and set the config on the node)
+    b2 --> |no|b4(Apply a default configuration)
+  end
+
+  subgraph top3 [Apply Formatting]
+    direction TB
+    c1(For each node, recursively build up a\nlist of source code lines, no indentation) --> c2(Consolidate the first two lines if hanging\nor if the config demands it)
+    c2 --> c3(Append indentation to each line as appropriate\nand surround each form with relevant parens)
+  end
+
+  subgraph top4 [Finalize Lines]
+    direction TB
+    d1(Concatenate all lines with whatever\nfile ending the file started with) --> d2(If we're formatting a file append a trailing new-line\n otherwise we are done)
+  end
+
+  top1-->|FormatterTree|top2
+  top2-->|FormatterTree with Configurations|top3
+  top3-->|String List|top4
+```
+
+### Build a Formatting Tree
+
+At the end of the day, formatting is about changing the location and amounts of white-space in a file.  Meaning it would be ideal if we could start with a representation that is whitespace-agnostic.  Some formatters (like cljfmt) choose to use the original whitespace as an input into the formatting process, essentially meaning that your code should already be somewhat-formatted.  This is somewhat advantageous as it can simplify the formatter's implementation or be a bit more flexible as to what the original programmer intended (assuming they knew what they were doing).  But of course it has one fatal flaw -- the code has to already be well formatted.
+
+For our formatter I wanted it to be able to take in horribly, ugly, minified code and output an optimal result -- especially since so much of our code gets spat out by another tool.  To do that, we need to be able to format the code without any existing whitespace helping us, adding in the whitespace from the ground up.
+
+The easiest way to do this is to construct an AST of the language which has a lot of other benefits for building language features but for here all we care about is getting rid of the whitespace and having some basic interpretation of the source code structure.  The easiest way now-a-days to make an AST for a language is to create a [treesitter grammar](https://github.com/open-goal/tree-sitter-opengoal).
+
+#### Consolidate AST Nodes and Collect Metadata
+
+One downside with using an AST, atleast with the way the grammar is currently implemented is it creates a somewhat difficult tree to work with from a formatting perspective.  For example in a LISP, everything is a list.  In our AST it does not look like how you may intuitively expect:
+
+`(+ 1 2)`
+
+```mermaid
+flowchart TB
+    a("source\n(+ 1 2)") --> b("list_lit\n")
+    b --> c("(") --> ca("(")
+    b --> d("sym_lit\n+") --> da("+")
+    b --> e("sym_lit\n1") --> ea("1")
+    b --> f("sym_lit\n2") --> fa("2")
+    b --> g(")") --> ga(")")
+```
+
+vs what you would probably prefer:
+
+```mermaid
+flowchart TB
+    a("top-level") --> b("list\n- surrounded in ()")
+    b --> da("+")
+    b --> ea("1")
+    b --> fa("2")
+```
+
+As you can see from even this simple example, the AST representation can be very verbose and in a sense has duplication.  Each node is aware of it's original location in the parsed source-code, but it's the leaf-node tokens that we care about for formatting and perhaps the form types they were a part of (ie. were they comments or in a list, etc).
+
+So to keep things brief, the AST representation is a bit too cumbersome and superfluous to use for formatting directly.  So the first step is recurse through it and optimize it, removing unneeded height and capturing crucial metadata that is actually relevant for formatting.
+
+We build a `FormatterTree` which consists of `FormatterNode`s.  Each `FormatterNode` has either a single token, or a list of `refs`.  A `ref` is just another `FormatterNode`.  Each node also has a bunch of metadata and configuration on how it should be formatted (just initialized for now).
+
+Having a dead-simple representation of the code like this is really only possible because it's a LISP, but it's also future proofed for us.  If OpenGOAL ever decides to support different brackets -- for example in Clojure a vector uses `[]` or a map uses `{}` it would be easy for update the formatter to wrap each list node in the right bracket type.
+
+### Applying Formatting Configuration
+
+Now that we have our simplified tree, we can get to actually formatting.  The first step is to figure out _how_ we intend to format things.  By separating this from the actual formatting it's a lot easier to reason about and find bugs.  You can also do potentially interesting things, where you do a full traversal of your source-code tree to figure out an optimal decision without trying to build up the final formatted string at the same time.
+
+It is lists, or `form`s that we care about figuring out how to format.  Each `form` has a head element (unless it's empty) that we can use to specify how it would be formatted.  For example we cannot specify the formatting for every single function name, but we can format a `defun` or a `let` in a special way.  With this mechanism, the formatter is easily extensible and configurable.  There is even the ability to specify formatting configuration for a list's children by index -- for example you may want to format the first element in a `let` (the binding list) differently than the rest of the code.
+
+There are also some forms that don't have a head-item, that we still identify and set some default formatting rules:
+- The top-level node - indentation is disabled
+- Lists of only literals, called `constant lists` - These are indented with `1` space instead of the default `2` for a `flow`'d form.
+  - If these lists literals are paired with keywords we call these `constant pairs` - We strive to put the pairs on the same line, if they are too long we spill to the next with `2` space indentation
+- TODO - cond/case pairs (similar to constant pairs)
+
+This brings up two important terms that you will see in the formatter implementation a lot -- `flow` and `hang`.  The difference is as such:
+
+```clj
+(println "this"
+         "form"
+         "is"
+         "hung / hanged")
+```
+
+```clj
+(println
+  "this"
+  "form"
+  "is"
+  "flowed")
+```
+
+The difference between the two other than visually is that hanging always results in 1 less line spent _because we don't care about line-length_.  This is great because it means we can always default to hanging unless overridden.  Flowing trades off that 1 vertical line for horizontal space and looks great for block-style forms.  Like the top-level of a `defun` or the content of a `let`.
+
+### Applying Formatting
+
+The final real-step in the process, actually using the configuration and generating the formatting code.  This phase is actually broken up into 3 sub-phases as we recurse through the `FormatterTree`
+
+#### Build Lines
+
+First we build the source code lines without indentation, it's a lot easier to figure out the indentation after all tokens are on the correct lines instead of doing it in tandem.  Most tokens are given their own line, but in the case of a hung form we combine the first two elements on the first line.  Other forms are formatted in even more complicated ways, or comments that were originally inline should remain next to that token.
+
+Secondly, once we have all our consolidated lines we can apply the indentation.  This is most often based on the configuration of the list node itself.  For example a `hung` form must be indented the width of a `(` and the length of the head-form, plus the whitespace separator -- conversely a `flowed` form only needs to be indented by `2` spaces, regardless of token length.  Because all of these algorithms are recursive we can hyper-focus on getting the indentation and formatting correct from the bottom-up.  In otherwords, we don't care if a form is 200 spaces indented into a function, once the lines are returned recursively, they will get another layer of indentation added on until we hit the top-level node.
+
+### Finalize Lines
+
+Finally, we concatenate our list of strings, respecting the original file's line-endings.
+
+As it is best practice, if we are formatting an entire file we should terminate with a trailing new-line.
+
+Both of these items are technically a TODO.
+
+## Configuration
+
+TODO
+
+## Reference
+
+This formatter is highly influenced by a lot of other existing formatters and writings.  The most significant of which is Clojure's [zprint](https://github.com/kkinnear/zprint/tree/main/doc/options) which has incredible documentation around it's capabilities and why it chooses what to do.
+
+The formatter I've built basically copies zprint in terms of basic functionality, but simplifies things by being even more opinionated (for example, doing far less in terms of trying to adhere to a line-length limit).
+
+### Other references
+
+- [The Hardest Program I’ve Ever Written - Dart Formatter](https://journal.stuffwithstuff.com/2015/09/08/the-hardest-program-ive-ever-written/)
+- [cljfmt](https://github.com/weavejester/cljfmt)
+
+### Why not respect a line-length limit
+
+The OpenGOAL formatter will respect line-length in only very specific situations:
+- When determining if a form can be fit entirely on a single line
+- Probably in the future for multi-line comments
+
+Besides it adding a massive amount of complexity, even zprint explains that a line-length limit won't always be followed
+
+> zprint will work very hard to fit the formatted output into this width, though there are limits to its effort.  For instance, it will not reduce the minimum indents in order to satisfy a particular width requirement.
+
+Let's illustrate with an example:
+
+```clj
+(defun some-func
+  (+ 123
+     456
+     (+ 123
+        456
+        ;; imagine this form exceeds our width, how can you "fix" this without
+        ;; messing up the indentation levels of the entire expression
+        ;; - you can't inline it (you're already past the width)
+        ;; - you can't insert a line break because it'll still be at the same horizontal level
+        (+ 123
+           456
+           789))))
+```
+
+In LISP code, we don't have many ideal options to deal with this, all options are syntactically correct, but in terms of conventional readability they are bad because when reading LISP code you want to use the indentation as a guide (which is why new-line parens are considered ugly by basically all LISP style guides)
+
+The other reason which is equally relevant is it is hard, it means the formatter would have to try to find an optimal solution by exploring different solutions until it finds the one that minimizes lines.
+
+But with an opinionated formatting style you can avoid this complexity, if you want your code to look nice don't write code like the decompiler sometimes outputs when it encounters something truly cursed:
+
+![](./docs/triangle-of-death.png)
+> `game-save.gc` about 200+ nested `let`s!
@@ -0,0 +1,362 @@
+#include "formatter.h"
+
+#include "formatter_tree.h"
+
+#include "common/log/log.h"
+#include "common/util/FileUtil.h"
+#include "common/util/string_util.h"
+
+#include "tree_sitter/api.h"
+
+#include "third-party/fmt/core.h"
+
+// Declare the `tree_sitter_opengoal` function, which is
+// implemented by the `tree-sitter-opengoal` library.
+extern "C" {
+extern const TSLanguage* tree_sitter_opengoal();
+}
+
+int hang_indentation_width(const FormatterTreeNode& curr_node) {
+  if (curr_node.token || curr_node.refs.empty()) {
+    return 0;
+  }
+  // Get the first element of the form
+  const auto& first_elt = curr_node.refs.at(0);
+  if (first_elt.token) {
+    return first_elt.token->length() +
+           2;  // +2 because the opening paren and then the following space
+  }
+  // Otherwise, continue nesting
+  return 1 + hang_indentation_width(first_elt);
+}
+
+// TODO - compute length of each node and store it
+void apply_formatting_config(
+    FormatterTreeNode& curr_node,
+    std::optional<std::shared_ptr<formatter_rules::config::FormFormattingConfig>>
+        config_from_parent = {}) {
+  using namespace formatter_rules;
+  // node is empty, base-case
+  if (curr_node.token || curr_node.refs.empty()) {
+    return;
+  }
+  // first, check to see if this form already has a predefined formatting configuration
+  // if it does, that simplifies things because there is only 1 way of formatting the form
+  std::optional<formatter_rules::config::FormFormattingConfig> predefined_config;
+  if (!config_from_parent && !curr_node.refs.empty() && curr_node.refs.at(0).token) {
+    const auto& form_head = curr_node.refs.at(0).token;
+    if (form_head && config::opengoal_form_config.find(form_head.value()) !=
+                         config::opengoal_form_config.end()) {
+      predefined_config = config::opengoal_form_config.at(form_head.value());
+      curr_node.formatting_config = predefined_config.value();
+    }
+  } else if (config_from_parent) {
+    predefined_config = *config_from_parent.value();
+    curr_node.formatting_config = predefined_config.value();
+  }
+  // In order to keep things simple, as well as because its ineffectual in lisp code (you can only
+  // enforce it so much without making things unreadable), line width will not matter for deciding
+  // whether or not to hang or flow the form
+  //
+  // This means that a hang would ALWAYS win, because it's 1 less line break.  Therefore this
+  // simplifies our approach there is no need to explore both braches to see which one would be
+  // preferred.
+  //
+  // Instead, we either use the predefined configuration (obviously) or we do some checks for some
+  // outlier conditions to see if things should be formatted differently
+  //
+  // Otherwise, we always default to a hang.
+  //
+  // NOTE - any modifications here to child elements could be superseeded later in the recursion
+  // in order to maintain your sanity, only modify things here that _arent_ touched by default
+  // configurations.  These are explicitly prepended with `parent_mutable_`
+  if (!predefined_config) {
+    if (curr_node.metadata.is_top_level) {
+      curr_node.formatting_config.indentation_width = 0;
+      curr_node.formatting_config.hang_forms = false;
+    } else if (constant_list::is_constant_list(curr_node)) {
+      // - Check if the form is a constant list (ie. a list of numbers)
+      curr_node.formatting_config.indentation_width = 1;
+      curr_node.formatting_config.hang_forms = false;
+      curr_node.formatting_config.has_constant_pairs =
+          constant_pairs::form_should_be_constant_paired(curr_node);
+      // If applicable, iterate through the constant pairs, since we can potentially pair up
+      // non-constant second elements in a pair (like a function call), there is the potential that
+      // they need to spill to the next line and get indented in extra.  This is an exceptional
+      // circumstance, we do NOT do this sort of thing when formatting normal forms (cond/case pairs
+      // are another similar situation)
+      if (curr_node.formatting_config.has_constant_pairs) {
+        for (int i = 0; i < (int)curr_node.refs.size(); i++) {
+          auto& child_ref = curr_node.refs.at(i);
+          const auto type = child_ref.metadata.node_type;
+          if (constant_types.find(type) == constant_types.end() &&
+              constant_pairs::is_element_second_in_constant_pair(curr_node, child_ref, i)) {
+            child_ref.formatting_config.parent_mutable_extra_indent = 2;
+          }
+        }
+      }
+
+    } else if (curr_node.formatting_config.hang_forms && curr_node.refs.size() > 1 &&
+               curr_node.refs.at(1).metadata.is_comment) {
+      // - Check if the second argument is a comment, it looks better if we flow instead
+      curr_node.formatting_config.hang_forms = false;
+    }
+  }
+  // If we are hanging, lets determine the indentation width since it is based on the form itself
+  if (curr_node.formatting_config.hang_forms) {
+    curr_node.formatting_config.indentation_width = hang_indentation_width(curr_node);
+  }
+  // iterate through the refs
+  for (int i = 0; i < (int)curr_node.refs.size(); i++) {
+    auto& ref = curr_node.refs.at(i);
+    if (!ref.token) {
+      // If the child has a pre-defined configuration at that index, we pass it along
+      if (predefined_config &&
+          predefined_config->index_configs.find(i) != predefined_config->index_configs.end()) {
+        apply_formatting_config(ref, predefined_config->index_configs.at(i));
+      } else if (predefined_config && predefined_config->default_index_config) {
+        apply_formatting_config(ref, predefined_config->default_index_config);
+      } else {
+        apply_formatting_config(ref);
+      }
+    }
+  }
+}
+
+int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
+  if (curr_node.token) {
+    return curr_node.token->length();
+  }
+  int width = 1;
+  for (const auto& ref : curr_node.refs) {
+    width += get_total_form_inlined_width(ref);
+  }
+  return width + 1;
+}
+
+bool form_contains_comment(const FormatterTreeNode& curr_node) {
+  if (curr_node.metadata.is_comment) {
+    return true;
+  }
+  for (const auto& ref : curr_node.refs) {
+    const auto contains_comment = form_contains_comment(ref);
+    if (contains_comment) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool form_contains_node_that_prevents_inlining(const FormatterTreeNode& curr_node) {
+  if (curr_node.formatting_config.should_prevent_inlining(curr_node.formatting_config,
+                                                          curr_node.refs.size())) {
+    return true;
+  }
+  for (const auto& ref : curr_node.refs) {
+    const auto prevents_inlining = form_contains_node_that_prevents_inlining(ref);
+    if (prevents_inlining) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool can_node_be_inlined(const FormatterTreeNode& curr_node, int cursor_pos) {
+  using namespace formatter_rules;
+  // First off, we cannot inline the top level
+  if (curr_node.metadata.is_top_level) {
+    return false;
+  }
+  // If the config explicitly prevents inlining, or it contains a sub-node that prevents inlining
+  if (curr_node.formatting_config.prevent_inlining ||
+      form_contains_node_that_prevents_inlining(curr_node)) {
+    return false;
+  }
+  // nor can we inline something that contains a comment in the middle
+  if (form_contains_comment(curr_node)) {
+    return false;
+  }
+  // constant pairs are not inlined!
+  if (curr_node.formatting_config.has_constant_pairs) {
+    return false;
+  }
+  // If this is set in the config, then the form is intended to be partially inlined
+  if (curr_node.formatting_config.inline_until_index({})) {
+    return false;
+  }
+  // let's see if we can inline the form all on one line to do that, we recursively explore
+  // the form to find the total width
+  int line_width = cursor_pos + get_total_form_inlined_width(curr_node);
+  return line_width <= indent::line_width_target;  // TODO - comments
+}
+
+std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
+                                          std::vector<std::string> output = {},
+                                          int cursor_pos = 0) {
+  using namespace formatter_rules;
+  if (!curr_node.token && curr_node.refs.empty()) {
+    // special case to handle an empty list
+    return {"()"};
+  }
+
+  // If its a token, just print the token and move on
+  if (curr_node.token) {
+    return {curr_node.token.value()};
+  }
+
+  bool inline_form = can_node_be_inlined(curr_node, cursor_pos);
+  // TODO - also if the form is inlinable, we can skip all the complication below and just...inline
+  // it!
+  // TODO - should figure out the inlining here as well, instead of the bool above
+
+  // Iterate the form, building up a list of the final lines but don't worry about indentation
+  // at this stage.  Once the lines are finalized, it's easy to add the indentation later
+  //
+  // This means we may combine elements onto the same line in this step.
+  std::vector<std::string> form_lines = {};
+
+  for (int i = 0; i < (int)curr_node.refs.size(); i++) {
+    const auto& ref = curr_node.refs.at(i);
+    // Add new line entry
+    if (ref.token) {
+      // Cleanup block-comments
+      std::string val = ref.token.value();
+      if (ref.metadata.node_type == "block_comment") {
+        // TODO - change this sanitization to return a list of lines instead of a single new-lined
+        // line
+        val = comments::format_block_comment(ref.token.value());
+      }
+      form_lines.push_back(val);
+    } else {
+      // If it's not a token, we have to recursively build up the form
+      // TODO - add the cursor_pos here
+      const auto& lines = apply_formatting(ref, {}, cursor_pos);
+      for (int i = 0; i < (int)lines.size(); i++) {
+        const auto& line = lines.at(i);
+        form_lines.push_back(fmt::format(
+            "{}{}", str_util::repeat(ref.formatting_config.parent_mutable_extra_indent, " "),
+            line));
+      }
+    }
+    // If we are hanging forms, combine the first two forms onto the same line
+    if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
+        (curr_node.formatting_config.hang_forms ||
+         curr_node.formatting_config.combine_first_two_lines)) {
+      form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+      form_lines.erase(form_lines.begin() + 1);
+    } else if ((i + 1) < (int)curr_node.refs.size()) {
+      const auto& next_ref = curr_node.refs.at(i + 1);
+      // combine the next inline comment or constant pair
+      if ((next_ref.metadata.node_type == "comment" && next_ref.metadata.is_inline) ||
+          (curr_node.formatting_config.has_constant_pairs &&
+           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1))) {
+        if (next_ref.token) {
+          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token.value());
+          i++;
+        } else if (can_node_be_inlined(next_ref, cursor_pos)) {
+          const auto& lines = apply_formatting(next_ref, {}, cursor_pos);  // TODO - cursor pos
+          for (const auto& line : lines) {
+            form_lines.at(form_lines.size() - 1) += fmt::format(" {}", line);
+          }
+          i++;
+        }
+      }
+    }
+    // If we are at the top level, potential separate with a new line
+    if (blank_lines::should_insert_blank_line(curr_node, ref, i)) {
+      form_lines.at(form_lines.size() - 1) += "\n";
+    }
+  }
+
+  // Consolidate any lines if the configuration requires it
+  if (curr_node.formatting_config.inline_until_index(form_lines)) {
+    std::vector<std::string> new_form_lines = {};
+    for (int i = 0; i < (int)form_lines.size(); i++) {
+      if (i < curr_node.formatting_config.inline_until_index(form_lines)) {
+        if (new_form_lines.empty()) {
+          new_form_lines.push_back(form_lines.at(i));
+        } else {
+          new_form_lines.at(0) += fmt::format(" {}", form_lines.at(i));
+        }
+      } else {
+        new_form_lines.push_back(form_lines.at(i));
+      }
+    }
+    form_lines = new_form_lines;
+  }
+
+  // Apply necessary indentation to each line and add parens
+  if (!curr_node.metadata.is_top_level) {
+    std::string form_surround_start = "(";
+    std::string form_surround_end = ")";
+    form_lines[0] = fmt::format("{}{}", form_surround_start, form_lines[0]);
+    form_lines[form_lines.size() - 1] =
+        fmt::format("{}{}", form_lines[form_lines.size() - 1], form_surround_end);
+  }
+  std::string curr_form = "";
+  if (curr_node.formatting_config.parent_mutable_extra_indent > 0) {
+    curr_form += str_util::repeat(curr_node.formatting_config.parent_mutable_extra_indent, " ");
+  }
+  if (inline_form) {
+    form_lines = {fmt::format("{}", fmt::join(form_lines, " "))};
+  } else {
+    for (int i = 0; i < (int)form_lines.size(); i++) {
+      if (i > 0) {
+        auto& line = form_lines.at(i);
+        line = fmt::format("{}{}",
+                           str_util::repeat(curr_node.formatting_config.indentation_width_for_index(
+                                                curr_node.formatting_config, i),
+                                            " "),
+                           line);
+      }
+    }
+  }
+  return form_lines;
+}
+
+std::string join_formatted_lines(const std::vector<std::string> lines) {
+  // TODO - respect original file line endings
+  return fmt::format("{}", fmt::join(lines, "\n"));
+}
+
+std::optional<std::string> formatter::format_code(const std::string& source) {
+  // Create a parser.
+  std::shared_ptr<TSParser> parser(ts_parser_new(), TreeSitterParserDeleter());
+
+  // Set the parser's language (JSON in this case).
+  ts_parser_set_language(parser.get(), tree_sitter_opengoal());
+
+  // Build a syntax tree based on source code stored in a string.
+  std::shared_ptr<TSTree> tree(
+      ts_parser_parse_string(parser.get(), NULL, source.c_str(), source.length()),
+      TreeSitterTreeDeleter());
+
+  // Get the root node of the syntax tree.
+  TSNode root_node = ts_tree_root_node(tree.get());
+  if (ts_node_is_null(root_node) || ts_node_has_error(root_node)) {
+    return std::nullopt;
+  }
+
+  try {
+    // There are three phases of formatting
+    // 1. Simplify the AST down to something that is easier to work on from a formatting perspective
+    // this also gathers basic metadata that can be done at this stage, like if the token is a
+    // comment or if the form is on the top-level
+    auto formatting_tree = FormatterTree(source, root_node);
+    // 2. Recursively iterate through this simplified FormatterTree and figure out what rules
+    // need to be applied to produce an optimal result
+    apply_formatting_config(formatting_tree.root);
+    // 3. Use this updated FormatterTree to print out the final source-code, while doing so
+    // we may deviate from the optimal result to produce something even more optimal by inlining
+    // forms that can fit within the line width.
+    const auto formatted_lines = apply_formatting(formatting_tree.root);
+    // 4. Now we joint he lines together, it's easier when formatting to leave all lines independent
+    // so adding indentation is easier
+    const auto formatted_source = join_formatted_lines(formatted_lines);
+    return formatted_source;
+  } catch (std::exception& e) {
+    lg::error("Unable to format code - {}", e.what());
+  }
+
+  return std::nullopt;
+}
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <optional>
+#include <string>
+
+#include "common/formatter/rules/formatting_rules.h"
+#include "common/formatter/rules/rule_config.h"
+
+#include "tree_sitter/api.h"
+
+// TODO:
+// - Considering _eventually_ adding line-length heuristics
+namespace formatter {
+
+struct TreeSitterParserDeleter {
+  void operator()(TSParser* ptr) const { ts_parser_delete(ptr); }
+};
+
+struct TreeSitterTreeDeleter {
+  void operator()(TSTree* ptr) const { ts_tree_delete(ptr); }
+};
+
+std::optional<std::string> format_code(const std::string& source);
+}  // namespace formatter
@@ -0,0 +1,121 @@
+#include "formatter_tree.h"
+
+#include "common/util/string_util.h"
+
+#include "third-party/fmt/core.h"
+
+std::string get_source_code(const std::string& source, const TSNode& node) {
+  uint32_t start = ts_node_start_byte(node);
+  uint32_t end = ts_node_end_byte(node);
+  return source.substr(start, end - start);
+}
+
+int num_blank_lines_following_node(const std::string& source, const TSNode& node) {
+  int num_lines = -1;  // The first new-line encountered is not a blank line
+  uint32_t cursor = ts_node_end_byte(node);
+  // TODO - this breaks on lines with whitespace as well, should probably seek past that!
+  while (cursor < source.length() && source.at(cursor) == '\n') {
+    num_lines++;
+    cursor++;
+  }
+  return num_lines;
+}
+
+// Check if the original source only has whitespace up to a new-line before it's token
+bool node_preceeded_by_only_whitespace(const std::string& source, const TSNode& node) {
+  // NOTE - this returns incorrectly because we skip brackets in lists, we'll see if that matters
+  int32_t pos = ts_node_start_byte(node) - 1;
+  while (pos > 0) {
+    const auto& c = source.at(pos);
+    if (c == '\n') {
+      return true;
+    } else if (c == ' ' || c == '\t') {
+      pos--;
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+FormatterTreeNode::FormatterTreeNode(const std::string& source, const TSNode& node)
+    : token(get_source_code(source, node)) {
+  metadata.node_type = ts_node_type(node);
+  metadata.is_comment = str_util::starts_with(str_util::ltrim(token.value()), ";") ||
+                        str_util::starts_with(str_util::ltrim(token.value()), "#|");
+  // Set any metadata based on the value of the token
+  metadata.num_blank_lines_following = num_blank_lines_following_node(source, node);
+  metadata.is_inline = !node_preceeded_by_only_whitespace(source, node);
+};
+
+// Check if the original source only has whitespace up to a new-line after it's token
+bool node_followed_by_only_whitespace(const std::string& source, const TSNode& node) {
+  uint32_t pos = ts_node_end_byte(node);
+  while (pos < source.length()) {
+    const auto& c = source.at(pos);
+    if (c == '\n') {
+      return true;
+    } else if (c == ' ' || c == '\t') {
+      pos++;
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+bool nodes_on_same_line(const std::string& source, const TSNode& n1, const TSNode& n2) {
+  // Get the source between the two lines, if there are any new-lines, the answer is NO
+  uint32_t start = ts_node_start_byte(n1);
+  uint32_t end = ts_node_end_byte(n2);
+  const auto code_between = source.substr(start, end - start);
+  return !str_util::contains(code_between, "\n");
+}
+
+FormatterTree::FormatterTree(const std::string& source, const TSNode& root_node) {
+  root = FormatterTreeNode();
+  root.metadata.is_top_level = true;
+  construct_formatter_tree_recursive(source, root_node, root);
+}
+
+// TODO make an imperative version eventually
+void FormatterTree::construct_formatter_tree_recursive(const std::string& source,
+                                                       TSNode curr_node,
+                                                       FormatterTreeNode& tree_node) {
+  if (ts_node_child_count(curr_node) == 0) {
+    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
+    return;
+  }
+  const std::string curr_node_type = ts_node_type(curr_node);
+  FormatterTreeNode list_node;
+  if (curr_node_type == "list_lit") {
+    list_node = FormatterTreeNode();
+  } else if (curr_node_type == "str_lit") {
+    // Strings are a special case, they are literals and essentially tokens but the grammar can
+    // detect formatter identifiers, this is useful for semantic highlighting but doesn't matter for
+    // formatting So for strings, we treat them as if they should be a single token
+    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
+    return;
+  } else if (curr_node_type == "quoting_lit") {
+    // same story for quoted symbols
+    // TODO - expect to have to add more here
+    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
+    return;
+  }
+  for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
+    const auto child_node = ts_node_child(curr_node, i);
+    // We skip parens
+    const auto contents = get_source_code(source, child_node);
+    if (contents == "(" || contents == ")") {
+      continue;
+    }
+    if (curr_node_type == "list_lit") {
+      construct_formatter_tree_recursive(source, child_node, list_node);
+    } else {
+      construct_formatter_tree_recursive(source, child_node, tree_node);
+    }
+  }
+  if (curr_node_type == "list_lit") {
+    tree_node.refs.push_back(list_node);
+  }
+}
@@ -0,0 +1,66 @@
+#pragma once
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "rules/rule_config.h"
+#include "tree_sitter/api.h"
+
+// Treesitter is fantastic for validating and parsing our code into a structured tree format without
+// whitespace so we can do that ourselves (formatting) However, the treesitter AST is a bit too
+// detailed for purposes of formatting.
+//
+// When formatting there is no need to know things like are we
+// in a function, or a symbol, etc. This extra information is fantastic for parsing or manipulating
+// the language, but becomes burdensome when just trying to write elegant formatting code when all
+// we really care about is:
+// - getting all the text tokens for the source code
+// - having them in a proper, nested format
+// The treesitter format is complicated and highly nested, leading to some very hard to understand
+// code. So my solution is a 2-pass format.
+//
+// Pass 1 - convert the AST into a simplified FormatterTree
+// Pass 2 - use the simplified tree to output the final code
+
+class FormatterTreeNode {
+ public:
+  struct Metadata {
+    std::string node_type;
+    bool is_top_level = false;
+    bool is_comment = false;
+    bool is_inline = false;
+    int num_blank_lines_following = 0;
+    bool is_binding_list = false;  // TODO set this
+  };
+  std::vector<FormatterTreeNode> refs;
+  Metadata metadata;
+  // The token is optional because list nodes do not contain a token, they just contain a bunch of
+  // eventually token node refs
+  std::optional<std::string> token;
+
+  formatter_rules::config::FormFormattingConfig formatting_config;
+
+  FormatterTreeNode() = default;
+  FormatterTreeNode(const std::string& source, const TSNode& node);
+  FormatterTreeNode(const Metadata& _metadata) : metadata(_metadata){};
+
+  bool is_list() const { return !token.has_value(); }
+};
+
+// A FormatterTree has a very simple and crude tree structure where:
+// - Nodes are essentially forms, which contain in-order tokens or references to nested forms
+// - Nodes can have associated metadata, often related to their context in the original code
+// - Nodes can also have multiple formatting rules associated with them.  Often this is the default
+// rule or based on pre-configured overrides due to the head of the form, ex. 'defun'
+class FormatterTree {
+ public:
+  FormatterTree(const std::string& source, const TSNode& root_node);
+  FormatterTreeNode root;
+
+ private:
+  void construct_formatter_tree_recursive(const std::string& source,
+                                          TSNode curr_node,
+                                          FormatterTreeNode& tree_node);
+};
@@ -0,0 +1,132 @@
+#include "formatting_rules.h"
+
+#include <set>
+
+#include "rule_config.h"
+
+#include "common/util/string_util.h"
+
+#include "third-party/fmt/core.h"
+
+namespace formatter_rules {
+
+// TODO - probably need to include quoted literals as well, though the grammar currently does not
+// differentiate between a quoted symbol and a quoted form
+const std::set<std::string> constant_types = {"kwd_lit",  "num_lit",  "str_lit",
+                                              "char_lit", "null_lit", "bool_lit"};
+
+namespace constant_list {
+bool is_constant_list(const FormatterTreeNode& node) {
+  if (!node.is_list() || node.refs.empty()) {
+    return false;
+  }
+  const auto& type = node.refs.at(0).metadata.node_type;
+  return constant_types.find(type) != constant_types.end();
+}
+}  // namespace constant_list
+
+namespace blank_lines {
+
+bool should_insert_blank_line(const FormatterTreeNode& containing_node,
+                              const FormatterTreeNode& node,
+                              const int index) {
+  // We only do this at the top level and don't leave a trailing new-line
+  if (!containing_node.metadata.is_top_level || index >= (int)containing_node.refs.size() - 1) {
+    return false;
+  }
+  // If it's a comment, but has no following blank lines, dont insert a blank line
+  if (node.metadata.is_comment && node.metadata.num_blank_lines_following == 0) {
+    return false;
+  }
+  // If the next form is a comment and is inline, don't insert a comment
+  if ((index + 1) < (int)containing_node.refs.size() &&
+      containing_node.refs.at(index + 1).metadata.is_comment &&
+      containing_node.refs.at(index + 1).metadata.is_inline) {
+    return false;
+  }
+
+  // TODO - only if the form doesn't fit on a single line
+  return true;
+}
+
+}  // namespace blank_lines
+
+namespace comments {
+std::string format_block_comment(const std::string& comment) {
+  // Normalize block comments, remove any trailing or leading whitespace
+  // Only allow annotations on the first line, like #|@file
+  // Don't mess with internal indentation as the user might intend it to be a certain way.
+  std::string new_comment = "";
+  std::string comment_contents = "";
+  bool seek_until_whitespace = str_util::starts_with(comment, "#|@");
+  int chars_seeked = 0;
+  for (const auto& c : comment) {
+    if (c == '\n' || (seek_until_whitespace && (c == ' ' || c == '\t')) ||
+        (!seek_until_whitespace && (c != '#' && c != '|'))) {
+      break;
+    }
+    chars_seeked++;
+    new_comment += c;
+  }
+  // Remove the first line content and any leading whitespace
+  comment_contents = str_util::ltrim_newlines(comment.substr(chars_seeked));
+  // Remove trailing whitespace
+  comment_contents = str_util::rtrim(comment_contents);
+  // remove |#
+  // TODO - check suffix
+  comment_contents.pop_back();
+  comment_contents.pop_back();
+  comment_contents = str_util::rtrim(comment_contents);
+  new_comment += fmt::format("\n{}\n|#", comment_contents);
+  return new_comment;
+}
+}  // namespace comments
+
+namespace constant_pairs {
+
+// TODO - remove index, not needed, could just pass in the previous node
+bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node,
+                                        const FormatterTreeNode& node,
+                                        const int index) {
+  if (containing_node.refs.empty() || index == 0) {
+    return false;
+  }
+  // Ensure that a keyword came before hand
+  if (containing_node.refs.at(index - 1).metadata.node_type != "kwd_lit") {
+    return false;
+  } else if (node.metadata.node_type == "kwd_lit") {
+    // NOTE - there is ambiugity here which cannot be totally solved (i think?)
+    // if the element itself is also a keyword, assume this is two adjacent keywords and they should
+    // not be paired
+    return false;
+  }
+  return true;
+}
+
+bool form_should_be_constant_paired(const FormatterTreeNode& node) {
+  // Criteria for a list to be constant paired:
+  // - needs to start with a non-symbol
+  // - needs atleast the minimum amount of pairs, so 2 pairs can still be inlined
+  if (node.refs.empty()) {
+    return false;
+  }
+  int num_pairs = 0;
+  for (int i = 0; i < (int)node.refs.size() - 1; i++) {
+    const auto& ref = node.refs.at(i);
+    const auto& next_ref = node.refs.at(i + 1);
+    if (ref.token && next_ref.token) {
+      // If the first element a keyword and the following item is a constant, it's a pair
+      // move forward one extra index
+      if (ref.metadata.node_type == "kwd_lit" &&
+          constant_types.find(next_ref.metadata.node_type) != constant_types.end()) {
+        num_pairs++;
+        i++;
+      }
+    }
+  }
+  return num_pairs >= min_pair_amount;
+}
+
+}  // namespace constant_pairs
+
+}  // namespace formatter_rules
@@ -0,0 +1,119 @@
+#pragma once
+
+#include <set>
+#include <string>
+
+#include "common/formatter/formatter_tree.h"
+
+namespace formatter_rules {
+
+extern const std::set<std::string> constant_types;
+namespace constant_list {
+bool is_constant_list(const FormatterTreeNode& node);
+}
+
+// The formatter will try to collapse as much space as possible in the top-level, this means
+// separating forms by a single empty blank line
+//
+// The exception are comments, top level comments will retain their following blank lines from the
+// original source
+// - this could be none, in the case where a comment is directly next to a form (like this one!)
+//   - you don't want them to be separated!
+// - or this could be top level comments / comment blocks documenting things but not being
+// associated with a form
+//   - in this case, you want them to remain separated
+//
+// Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/blank.md
+namespace blank_lines {
+bool should_insert_blank_line(const FormatterTreeNode& containing_node,
+                              const FormatterTreeNode& node,
+                              const int index);
+}  // namespace blank_lines
+
+// TODO:
+// - align consecutive comment lines
+// - if/when the formatter is concerned with line length, there are implications here
+//
+// Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/comments.md
+namespace comments {
+std::string format_block_comment(const std::string& comment);
+}
+
+// Paired elements in a list will be kept in-line rather than the default new-line indentation
+// For example:
+// (:msg "hello world" :delay 100 :fn (lambda () (+ 1 1)))
+// Would typically become:
+// (:msg
+//  "hello world"
+//  :delay
+//  100
+//  :fn
+//    (lambda ()
+//      (+ 1 1)))
+// But with constant pairs:
+// (:msg "hello world"
+//  :delay 100
+//  :fn
+//    (lambda ()
+//      (+ 1 1)))
+//
+// Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/constantpairs.md
+namespace constant_pairs {
+const static int min_pair_amount = 4;
+
+// Determines if the given element is the second element in a constant pair, if it is then we would
+// usually want to elide the new-line in whatever code that applies it
+//
+// This is true if:
+// - the element is in a list
+// - the element is preceeded by a keyword
+// - the element is a:
+//   - keyword, symbol, string, number, or boolean
+bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node,
+                                        const FormatterTreeNode& node,
+                                        const int index);
+bool form_should_be_constant_paired(const FormatterTreeNode& node);
+}  // namespace constant_pairs
+
+// There are two main types of indentations "flow"s and "hang"s
+// (this is
+//       a
+//       hang)
+// (this
+//   is
+//   a
+//   flow)
+//
+// `flow` is the default indentation mode and right now the determination between the two is down to
+// manual configuration based on the head of the form. In general, we only `flow` or `hang` if the
+// form cannot fit on the current line.
+//
+// Additionally, if the head of the form is a constant we `flow` with an indent of `1` instead of
+// `2`
+//
+// By default, we always hang unless:
+// - the head-form overrides the configuration
+// - TODO it will use more lines than the flow approach
+//
+// Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/indent.md
+namespace indent {
+const static int line_width_target = 120;
+
+}  // namespace indent
+
+// Let forms fall into two main categories
+// - Ones that can be entirely inlined
+// - Ones that cannot
+//
+// Single line let forms:
+// - If there is a single binding, let it continue to be one line
+// - otherwise, flow it just as a multi-line let would be done
+//
+// For multi-line let forms, the difficulty is how to format the binding:
+// - align the bindings to within the binding list (single indent space)
+// - always format the binding value on the same line as the binding, no new lines
+// - forms inside the let binding are flowed
+//
+// Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/let.md
+namespace let {}  // namespace let
+}  // namespace formatter_rules
@@ -0,0 +1,88 @@
+#include "rule_config.h"
+
+#include "common/formatter/formatter_tree.h"
+
+namespace formatter_rules {
+namespace config {
+
+// TODO - this could be greatly simplified with C++20's designated initialization
+FormFormattingConfig new_flow_rule(int start_index) {
+  FormFormattingConfig cfg;
+  cfg.hang_forms = false;
+  cfg.inline_until_index = [start_index](const std::vector<std::string>& curr_lines) {
+    return start_index;
+  };
+  return cfg;
+}
+
+FormFormattingConfig new_flow_rule_prevent_inlining_indexes(
+    int start_index,
+    const std::vector<int>& inlining_preventation_indices) {
+  FormFormattingConfig cfg;
+  cfg.hang_forms = false;
+  cfg.inline_until_index = [start_index](std::vector<std::string> curr_lines) {
+    if (curr_lines.size() >= 4 && curr_lines.at(3) == "()") {
+      return 4;
+    }
+    return start_index;
+  };
+  for (const auto& index : inlining_preventation_indices) {
+    auto temp_config = std::make_shared<FormFormattingConfig>();
+    temp_config->prevent_inlining = true;
+    temp_config->hang_forms = false;
+    temp_config->indentation_width = 1;
+    cfg.index_configs.emplace(index, temp_config);
+  }
+  return cfg;
+}
+
+FormFormattingConfig new_binding_rule() {
+  FormFormattingConfig cfg;
+  cfg.hang_forms = false;
+  cfg.combine_first_two_lines = true;
+  auto binding_list_config = std::make_shared<FormFormattingConfig>();
+  binding_list_config->hang_forms = false;
+  binding_list_config->indentation_width = 1;
+  binding_list_config->indentation_width_for_index = [](FormFormattingConfig /*cfg*/, int index) {
+    if (index == 0) {
+      return 0;
+    }
+    return 4;
+  };
+  binding_list_config->should_prevent_inlining = [](FormFormattingConfig /*config*/, int num_refs) {
+    // Only prevent inlining a binding list, if there are more than 1 bindings
+    if (num_refs > 1) {
+      return true;
+    }
+    return false;
+  };
+  binding_list_config->prevent_inlining =
+      true;  // TODO - we only want to prevent inlining if there are more than 2 elements
+  cfg.index_configs.emplace(1, binding_list_config);
+  return cfg;
+}
+
+FormFormattingConfig new_pair_rule(bool combine_first_two_expr) {
+  FormFormattingConfig cfg;
+  cfg.hang_forms = false;
+  cfg.prevent_inlining = true;
+  cfg.combine_first_two_lines = combine_first_two_expr;
+  auto pair_config = std::make_shared<FormFormattingConfig>();
+  pair_config->hang_forms = false;
+  pair_config->indentation_width = 1;
+  cfg.default_index_config = pair_config;
+  return cfg;
+}
+
+const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config = {
+    {"case", new_pair_rule(true)},
+    {"cond", new_pair_rule(false)},
+    {"defmethod", new_flow_rule(3)},
+    {"deftype", new_flow_rule_prevent_inlining_indexes(3, {3, 4, 5})},
+    {"defun", new_flow_rule(3)},
+    {"dotimes", new_flow_rule(2)},
+    {"let", new_binding_rule()},
+    {"when", new_flow_rule(2)},
+    {"with-dma-buffer-add-bucket", new_flow_rule(2)}};
+}  // namespace config
+}  // namespace formatter_rules
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace formatter_rules {
+namespace config {
+struct FormFormattingConfig {
+  // new
+  bool hang_forms = true;  // TODO - remove this eventually, it's only involved in setting the
+                           // indentation width, which we can do via the indentation_width function
+  int indentation_width =
+      2;  // 2 for a flow // TODO - also remove this, prefer storing the first node's width in the
+          // metadata on the first pass, that's basically all this does
+  std::function<int(FormFormattingConfig, int)> indentation_width_for_index =
+      [](FormFormattingConfig config, int /*index*/) { return config.indentation_width; };
+  bool combine_first_two_lines =
+      false;  // NOTE - basically hang, but will probably stick around after hang is gone, may be
+              // redundant (inline_until_index!)
+  std::function<std::optional<int>(const std::vector<std::string>& /*curr_lines*/)>
+      inline_until_index = [](std::vector<std::string> /*curr_lines*/) { return std::nullopt; };
+  bool has_constant_pairs = false;
+  bool prevent_inlining = false;  // TODO - duplicate of below
+  std::function<bool(FormFormattingConfig, int num_refs)> should_prevent_inlining =
+      [](FormFormattingConfig config, int /*num_refs*/) { return config.prevent_inlining; };
+  int parent_mutable_extra_indent = 0;
+  std::optional<std::shared_ptr<FormFormattingConfig>> default_index_config;
+  std::unordered_map<int, std::shared_ptr<FormFormattingConfig>> index_configs = {};
+};
+
+extern const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config;
+}  // namespace config
+}  // namespace formatter_rules
@@ -7,21 +7,22 @@

 #include "common/util/Assert.h"
 #include "common/util/FileUtil.h"
+#include "common/common_types.h"

 #include "third-party/fmt/core.h"
 #include "third-party/json.hpp"

-#ifdef __linux__
-u32 get_current_tid() {
-  return (u32)pthread_self();
+#ifdef OS_POSIX
+u64 get_current_tid() {
+  return (u64)pthread_self();
 }
 #else
 #define NOMINMAX
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 #include "Processthreadsapi.h"
-u32 get_current_tid() {
-  return (u32)GetCurrentThreadId();
+u64 get_current_tid() {
+  return (u64)GetCurrentThreadId();
 }
 #endif
 #include "common/log/log.h"
@@ -58,6 +59,10 @@ void GlobalProfiler::instant_event(const char* name) {
  event(name, ProfNode::INSTANT);
 }

+void GlobalProfiler::root_event() {
+  instant_event("ROOT");
+}
+
 void GlobalProfiler::begin_event(const char* name) {
  event(name, ProfNode::BEGIN);
 }
@@ -8,9 +8,9 @@

 struct ProfNode {
  u64 ts;
-  char name[32];
+  u64 tid;
+  char name[128];
  enum Kind : u8 { BEGIN, END, INSTANT, UNUSED } kind = UNUSED;
-  u32 tid;
 };

 class GlobalProfiler {
@@ -24,6 +24,7 @@ class GlobalProfiler {
  void clear();
  void set_enable(bool en);
  void dump_to_json(const std::string& path);
+  void root_event();

 private:
  std::atomic_bool m_enabled = false;
@@ -1,7 +1,7 @@
 #pragma once

 #include "common/common_types.h"
-#include "common/versions.h"
+#include "common/versions/versions.h"

 constexpr s32 BINTEGER_OFFSET = 0;
 constexpr s32 PAIR_OFFSET = 2;
@@ -41,6 +41,11 @@ constexpr int bits_for_sym() {
  return b + 1;
 }
 static_assert(bits_for_sym() != -1, "symbol table invalid length");
+
+// amount of levels in level heap
+constexpr int LEVEL_MAX = 2;
+// total amount of levels, including ones outside level heap (default-level)
+constexpr int LEVEL_TOTAL = LEVEL_MAX + 1;
 }  // namespace jak1

 namespace jak2 {
@@ -50,18 +55,40 @@ constexpr s32 SYM_TABLE_MEM_SIZE = 0x30000;
 // from the "off-by-one" symbol pointer
 constexpr int SYM_TO_STRING_OFFSET = 0xff37;
 constexpr int SYM_TO_HASH_OFFSET = 0x1fe6f;
+
+// amount of levels in level heap
+constexpr int LEVEL_MAX = 6;
+// total amount of levels, including ones outside level heap (default-level)
+constexpr int LEVEL_TOTAL = LEVEL_MAX + 1;
 }  // namespace jak2

+// TODO copypaste from jak 2 for now
+namespace jak3 {
+// for now, we don't have the ability to extend the size of the symbol table
+constexpr s32 GOAL_MAX_SYMBOLS = 0x4000;
+constexpr s32 SYM_TABLE_MEM_SIZE = 0x30000;
+// from the "off-by-one" symbol pointer
+constexpr int SYM_TO_STRING_OFFSET = 0xff37;
+constexpr int SYM_TO_HASH_OFFSET = 0x1fe6f;
+
+// amount of levels in level heap
+constexpr int LEVEL_MAX = 10;
+// total amount of levels, including ones outside level heap (default-level)
+constexpr int LEVEL_TOTAL = LEVEL_MAX + 1;
+}  // namespace jak3
+
 constexpr s32 max_symbols(GameVersion version) {
  switch (version) {
    case GameVersion::Jak1:
      return jak1::GOAL_MAX_SYMBOLS;
    case GameVersion::Jak2:
      return jak2::GOAL_MAX_SYMBOLS;
+    case GameVersion::Jak3:
+      return jak2::GOAL_MAX_SYMBOLS;
  }
 }

-enum class RegClass { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID };
+enum class RegClass : u8 { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID };

 constexpr u32 GOAL_NEW_METHOD = 0;       // method ID of GOAL new
 constexpr u32 GOAL_DEL_METHOD = 1;       // method ID of GOAL delete
@@ -16,21 +16,23 @@ class Interpreter {
 public:
  Interpreter(const std::string& user_profile = "#f");
  ~Interpreter();
-  void execute_repl(ReplWrapper& repl);
+  void execute_repl(REPL::Wrapper& repl);
  void throw_eval_error(const Object& o, const std::string& err);
  Object eval_with_rewind(const Object& obj, const std::shared_ptr<EnvironmentObject>& env);
  bool get_global_variable_by_name(const std::string& name, Object* dest);
  void set_global_variable_by_name(const std::string& name, const Object& value);
  void set_global_variable_to_symbol(const std::string& name, const std::string& value);
+  void set_global_variable_to_int(const std::string& name, int value);
  Object eval(Object obj, const std::shared_ptr<EnvironmentObject>& env);
  Object intern(const std::string& name);
-  HeapObject* intern_ptr(const std::string& name);
+  InternedSymbolPtr intern_ptr(const std::string& name);
  void disable_printfs();
  Object eval_symbol(const Object& sym, const std::shared_ptr<EnvironmentObject>& env);
  bool eval_symbol(const Object& sym,
                   const std::shared_ptr<EnvironmentObject>& env,
                   Object* result);
  Arguments get_args(const Object& form, const Object& rest, const ArgumentSpec& spec);
+  Arguments get_args_no_named(const Object& form, const Object& rest, const ArgumentSpec& spec);
  void set_args_in_env(const Object& form,
                       const Arguments& args,
                       const ArgumentSpec& arg_spec,
@@ -53,7 +55,7 @@ class Interpreter {
 private:
  friend class Goal;
  void load_goos_library();
-  void define_var_in_env(Object& env, Object& var, const std::string& name);
+  void define_var_in_env(Object& env, const Object& var, const std::string& name);
  void expect_env(const Object& form, const Object& o);
  void vararg_check(
      const Object& form,
@@ -62,8 +64,11 @@ class Interpreter {
      const std::unordered_map<std::string, std::pair<bool, std::optional<ObjectType>>>& named);

  Object eval_pair(const Object& o, const std::shared_ptr<EnvironmentObject>& env);
+
+ public:
  ArgumentSpec parse_arg_spec(const Object& form, Object& rest);

+ private:
  Object quasiquote_helper(const Object& form, const std::shared_ptr<EnvironmentObject>& env);

  IntType number_to_integer(const Object& obj);
@@ -116,6 +121,9 @@ class Interpreter {
  Object eval_read(const Object& form,
                   Arguments& args,
                   const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_read_data_file(const Object& form,
+                             Arguments& args,
+                             const std::shared_ptr<EnvironmentObject>& env);
  Object eval_read_file(const Object& form,
                        Arguments& args,
                        const std::shared_ptr<EnvironmentObject>& env);
@@ -197,6 +205,18 @@ class Interpreter {
  Object eval_string_append(const Object& form,
                            Arguments& args,
                            const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_string_starts_with(const Object& form,
+                                 Arguments& args,
+                                 const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_string_ends_with(const Object& form,
+                               Arguments& args,
+                               const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_string_split(const Object& form,
+                           Arguments& args,
+                           const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_string_substr(const Object& form,
+                            Arguments& args,
+                            const std::shared_ptr<EnvironmentObject>& env);
  Object eval_ash(const Object& form,
                  Arguments& args,
                  const std::shared_ptr<EnvironmentObject>& env);
@@ -206,6 +226,9 @@ class Interpreter {
  Object eval_string_to_symbol(const Object& form,
                               Arguments& args,
                               const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_int_to_string(const Object& form,
+                            Arguments& args,
+                            const std::shared_ptr<EnvironmentObject>& env);
  Object eval_get_env(const Object& form,
                      Arguments& args,
                      const std::shared_ptr<EnvironmentObject>& env);
@@ -241,6 +264,16 @@ class Interpreter {
  Object eval_while(const Object& form,
                    const Object& rest,
                    const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_let(const Object& form,
+                  const Object& rest,
+                  const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_let_star(const Object& form,
+                       const Object& rest,
+                       const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_let_common(const Object& form,
+                         const Object& rest,
+                         const std::shared_ptr<EnvironmentObject>& env,
+                         bool star);

  Object eval_make_string_hash_table(const Object& form,
                                     Arguments& args,
@@ -252,26 +285,46 @@ class Interpreter {
                             Arguments& args,
                             const std::shared_ptr<EnvironmentObject>& env);

+  const Object& true_or_false(bool val);
+
+  void init_special_forms(
+      const std::unordered_map<std::string,
+                               Object (Interpreter::*)(const Object&,
+                                                       const Object&,
+                                                       const std::shared_ptr<EnvironmentObject>&)>&
+          forms);
+
+  void init_builtin_forms(
+      const std::unordered_map<std::string,
+                               Object (Interpreter::*)(const Object&,
+                                                       Arguments&,
+                                                       const std::shared_ptr<EnvironmentObject>&)>&
+          forms);
+
  bool want_exit = false;
  bool disable_printing = false;

-  std::unordered_map<std::string,
-                     Object (Interpreter::*)(const Object& form,
-                                             Arguments& args,
-                                             const std::shared_ptr<EnvironmentObject>& env)>
+  std::unordered_map<
+      void*,
+      Object (Interpreter::*)(const Object&, Arguments&, const std::shared_ptr<EnvironmentObject>&)>
      builtin_forms;

-  std::unordered_map<
-      std::string,
-      std::function<Object(const Object&, Arguments&, const std::shared_ptr<EnvironmentObject>&)>>
+  std::vector<std::pair<
+      void*,
+      std::function<Object(const Object&, Arguments&, const std::shared_ptr<EnvironmentObject>&)>>>
      m_custom_forms;
-  std::unordered_map<std::string,
-                     Object (Interpreter::*)(const Object& form,
-                                             const Object& rest,
-                                             const std::shared_ptr<EnvironmentObject>& env)>
+
+  std::vector<std::pair<void*,
+                        Object (Interpreter::*)(const Object& form,
+                                                const Object& rest,
+                                                const std::shared_ptr<EnvironmentObject>& env)>>
      special_forms;
  int64_t gensym_id = 0;

  std::unordered_map<std::string, ObjectType> string_to_type;
+
+  const char* m_false_sym = nullptr;
+  const char* m_true_sym = nullptr;
+  Object m_false_object, m_true_object;
 };
 }  // namespace goos
@@ -41,14 +41,92 @@
 #include "Object.h"

 #include <cinttypes>
+#include <cstring>

 #include "common/util/FileUtil.h"
+#include "common/util/crc32.h"
 #include "common/util/print_float.h"

 #include "third-party/fmt/core.h"

 namespace goos {

+SymbolTable::SymbolTable() {
+  m_power_of_two_size = 1;  // 2 ^ 1 = 2
+  m_entries.resize(2);
+  m_used_entries = 0;
+  m_next_resize = (m_entries.size() * kMaxUsed);
+  m_mask = 0b1;
+}
+
+SymbolTable::~SymbolTable() {
+  for (auto& e : m_entries) {
+    delete[] e.name;
+  }
+}
+
+InternedSymbolPtr SymbolTable::intern(const char* str) {
+  size_t string_len = strlen(str);
+  u32 hash = crc32((const u8*)str, string_len);
+
+  // probe
+  for (u32 i = 0; i < m_entries.size(); i++) {
+    u32 slot_addr = (hash + i) & m_mask;
+    auto& slot = m_entries[slot_addr];
+    if (!slot.name) {
+      // not found, insert!
+      slot.hash = hash;
+      auto* name = new char[string_len + 1];
+      memcpy(name, str, string_len + 1);
+      slot.name = name;
+      m_used_entries++;
+
+      if (m_used_entries >= m_next_resize) {
+        resize();
+        return intern(str);
+      }
+      return {name};
+    } else {
+      if (slot.hash != hash) {
+        continue;  // bad hash
+      }
+      if (strcmp(slot.name, str) != 0) {
+        continue;  // bad name
+      }
+      return {slot.name};
+    }
+  }
+
+  // should be impossible to reach.
+  ASSERT_NOT_REACHED();
+}
+
+void SymbolTable::resize() {
+  m_power_of_two_size++;
+  m_mask = (1U << m_power_of_two_size) - 1;
+
+  std::vector<Entry> new_entries(m_entries.size() * 2);
+  for (const auto& old_entry : m_entries) {
+    if (old_entry.name) {
+      bool done = false;
+      for (u32 i = 0; i < new_entries.size(); i++) {
+        u32 slot_addr = (old_entry.hash + i) & m_mask;
+        auto& slot = new_entries[slot_addr];
+        if (!slot.name) {
+          slot.name = old_entry.name;
+          slot.hash = old_entry.hash;
+          done = true;
+          break;
+        }
+      }
+      ASSERT(done);
+    }
+  }
+
+  m_entries = std::move(new_entries);
+  m_next_resize = kMaxUsed * m_entries.size();
+}
+
 /*!
 * Convert type to string (name in brackets)
 */
@@ -136,14 +214,9 @@ std::string fixed_to_string(char x) {
  return {buff};
 }

-/*!
- * Create a new symbol object by interning
- */
-Object SymbolObject::make_new(SymbolTable& st, const std::string& name) {
-  Object obj;
-  obj.type = ObjectType::SYMBOL;
-  obj.heap_obj = st.intern(name);
-  return obj;
+template <>
+std::string fixed_to_string(InternedSymbolPtr x) {
+  return x.name_ptr;
 }

 /*!
@@ -203,10 +276,23 @@ Object build_list(std::vector<Object>&& objects) {

  Object result;
  result.type = ObjectType::PAIR;
-  result.heap_obj = head;
+  result.heap_obj = std::move(head);
  return result;
 }

+/*!
+ * Is this a float object that's a power of two?
+ * NOTE: assumes 64-bit float.
+ */
+bool Object::is_power_of_2_float() const {
+  FloatType val = as_float();
+  u64 val_i = -1;
+  memcpy(&val_i, &val, sizeof(val));
+  u64 mantissa = val_i & ((1LL << 52) - 1);
+  u64 exponent = (val_i >> 52) & ((1LL << 11) - 1);
+  return mantissa == 0 && exponent != 0 && exponent != ((1LL << 11) - 1);
+}
+
 /*!
 * Compare two objects for equality.
 * Does "expensive" checking.
@@ -224,8 +310,9 @@ bool Object::operator==(const Object& other) const {
      return float_obj == other.float_obj;
    case ObjectType::CHAR:
      return char_obj == other.char_obj;
-
    case ObjectType::SYMBOL:
+      return symbol_obj == other.symbol_obj;
+
    case ObjectType::ENVIRONMENT:
    case ObjectType::LAMBDA:
    case ObjectType::MACRO:
@@ -257,7 +344,11 @@ bool Object::operator==(const Object& other) const {
 }

 bool Object::is_symbol(const std::string& name) const {
-  return is_symbol() && as_symbol()->name == name;
+  return is_symbol() && name == as_symbol().name_ptr;
+}
+
+bool Object::is_string(const std::string& val) const {
+  return is_string() && as_string()->data == val;
 }

 template <>
@@ -43,6 +43,7 @@
 *
 */

+#include <cstring>
 #include <map>
 #include <memory>
 #include <stdexcept>
@@ -57,9 +58,6 @@

 namespace goos {

-using FloatType = double;
-using IntType = s64;
-
 /*!
 * All objects have one of these kinds
 */
@@ -71,9 +69,9 @@ enum class ObjectType : u8 {
  INTEGER,
  FLOAT,
  CHAR,
+  SYMBOL,

  // allocated
-  SYMBOL,
  STRING,
  PAIR,
  ARRAY,
@@ -87,6 +85,29 @@ enum class ObjectType : u8 {
 std::string object_type_to_string(ObjectType kind);

 // Some objects are "fixed", meaning they are stored inline in the Object, and not heap allocated.
+// These use value semantics and are copied on object assignment.
+
+// For int/float, these are simply double/int64_t
+using FloatType = double;
+using IntType = s64;
+
+// For symbols, we simply store a pointer to the symbol's name string.
+// this is wrapped in a type just to avoid confusion with normal const char*.
+struct InternedSymbolPtr {
+  const char* name_ptr;
+
+  struct hash {
+    auto operator()(const InternedSymbolPtr& x) const {
+      return std::hash<const void*>()((const void*)x.name_ptr);
+    }
+  };
+  bool operator==(const char* msg) const { return strcmp(msg, name_ptr) == 0; }
+  bool operator!=(const char* msg) const { return strcmp(msg, name_ptr) != 0; }
+  bool operator==(const std::string& str) const { return str == name_ptr; }
+  bool operator!=(const std::string& str) const { return str != name_ptr; }
+  bool operator==(const InternedSymbolPtr& other) const { return other.name_ptr == name_ptr; }
+  bool operator!=(const InternedSymbolPtr& other) const { return other.name_ptr != name_ptr; }
+};

 /*!
 * By default, convert a fixed object data to string with std::to_string.
@@ -115,6 +136,31 @@ std::string fixed_to_string<char>(char);
 template <>
 std::string fixed_to_string<IntType>(IntType);

+template <>
+std::string fixed_to_string<InternedSymbolPtr>(InternedSymbolPtr);
+
+class SymbolTable {
+ public:
+  SymbolTable(const SymbolTable&) = delete;
+  SymbolTable& operator=(const SymbolTable&) = delete;
+  SymbolTable();
+  ~SymbolTable();
+  InternedSymbolPtr intern(const char* str);
+
+ private:
+  void resize();
+  int m_power_of_two_size = 0;
+  struct Entry {
+    u32 hash = 0;
+    const char* name = nullptr;
+  };
+  std::vector<Entry> m_entries;
+  int m_used_entries = 0;
+  int m_next_resize = 0;
+  u32 m_mask = 0;
+  static constexpr float kMaxUsed = 0.7;
+};
+
 /*!
 * Common implementation for a fixed object
 */
@@ -139,6 +185,8 @@ class FixedObject {
      return object_type_to_string(ObjectType::INTEGER);
    if (std::is_same<T, char>())
      return object_type_to_string(ObjectType::CHAR);
+    if (std::is_same<T, InternedSymbolPtr>())
+      return object_type_to_string(ObjectType::SYMBOL);
    throw std::runtime_error("Unsupported FixedObject type");
  }
 };
@@ -149,6 +197,7 @@ class FixedObject {
 using IntegerObject = FixedObject<int64_t>;
 using FloatObject = FixedObject<double>;
 using CharObject = FixedObject<char>;
+using SymbolObject = FixedObject<InternedSymbolPtr>;

 // Other objects are separate allocated on the heap. These objects should be HeapObjects.

@@ -162,7 +211,6 @@ class HeapObject {
 // forward declare all HeapObjects
 class PairObject;
 class EnvironmentObject;
-class SymbolObject;
 class StringObject;
 class LambdaObject;
 class MacroObject;
@@ -180,6 +228,7 @@ class Object {
    IntegerObject integer_obj;
    FloatObject float_obj;
    CharObject char_obj;
+    SymbolObject symbol_obj;
  };

  ObjectType type = ObjectType::INVALID;
@@ -192,6 +241,8 @@ class Object {
        return float_obj.print();
      case ObjectType::CHAR:
        return char_obj.print();
+      case ObjectType::SYMBOL:
+        return symbol_obj.print();
      case ObjectType::EMPTY_LIST:
        return "()";
      default:
@@ -207,6 +258,8 @@ class Object {
        return float_obj.inspect();
      case ObjectType::CHAR:
        return char_obj.inspect();
+      case ObjectType::SYMBOL:
+        return symbol_obj.inspect();
      case ObjectType::EMPTY_LIST:
        return "[empty list] ()\n";
      default:
@@ -244,16 +297,30 @@ class Object {
    return o;
  }

+  static Object make_symbol(SymbolTable* table, const char* name) {
+    Object o;
+    o.type = ObjectType::SYMBOL;
+    o.symbol_obj.value = table->intern(name);
+    return o;
+  }
+
  PairObject* as_pair() const;
  EnvironmentObject* as_env() const;
  std::shared_ptr<EnvironmentObject> as_env_ptr() const;
-  SymbolObject* as_symbol() const;
  StringObject* as_string() const;
  LambdaObject* as_lambda() const;
  MacroObject* as_macro() const;
  ArrayObject* as_array() const;
  StringHashTableObject* as_string_hash_table() const;

+  const InternedSymbolPtr& as_symbol() const {
+    if (type != ObjectType::SYMBOL) {
+      throw std::runtime_error("as_symbol called on a " + object_type_to_string(type) + " " +
+                               print());
+    }
+    return symbol_obj.value;
+  }
+
  IntType& as_int() {
    if (type != ObjectType::INTEGER) {
      throw std::runtime_error("as_int called on a " + object_type_to_string(type) + " " + print());
@@ -302,66 +369,19 @@ class Object {
  bool is_symbol() const { return type == ObjectType::SYMBOL; }
  bool is_symbol(const std::string& name) const;
  bool is_string() const { return type == ObjectType::STRING; }
+  bool is_string(const std::string& val) const;
  bool is_pair() const { return type == ObjectType::PAIR; }
  bool is_array() const { return type == ObjectType::ARRAY; }
  bool is_env() const { return type == ObjectType::ENVIRONMENT; }
  bool is_macro() const { return type == ObjectType::MACRO; }
  bool is_string_hash_table() const { return type == ObjectType::STRING_HASH_TABLE; }

+  bool is_power_of_2_float() const;
+
  bool operator==(const Object& other) const;
  bool operator!=(const Object& other) const { return !((*this) == other); }
 };

-class SymbolTable;
-
-/*!
- * A Symbol Object, which is just a wrapper around a string.
- * The make_new function will correctly
- */
-class SymbolObject : public HeapObject {
- public:
-  std::string name;
-  explicit SymbolObject(std::string _name) : name(std::move(_name)) {}
-  static Object make_new(SymbolTable& st, const std::string& name);
-
-  std::string print() const override { return name; }
-
-  std::string inspect() const override { return "[symbol] " + name + "\n"; }
-
-  ~SymbolObject() = default;
-};
-
-/*!
- * A Symbol Table, which holds all symbols.
- */
-class SymbolTable {
- public:
-  std::shared_ptr<HeapObject> intern(const std::string& name) {
-    const auto& kv = table.find(name);
-    if (kv == table.end()) {
-      auto iter = table.insert({name, std::make_shared<SymbolObject>(name)});
-      return (*iter.first).second;
-    } else {
-      return kv->second;
-    }
-  }
-
-  HeapObject* intern_ptr(const std::string& name) {
-    const auto& kv = table.find(name);
-    if (kv == table.end()) {
-      auto iter = table.insert({name, std::make_shared<SymbolObject>(name)});
-      return (*iter.first).second.get();
-    } else {
-      return kv->second.get();
-    }
-  }
-
-  ~SymbolTable() = default;
-
- private:
-  std::unordered_map<std::string, std::shared_ptr<HeapObject>> table;
-};
-
 class StringObject : public HeapObject {
 public:
  std::string data;
@@ -436,14 +456,41 @@ class PairObject : public HeapObject {
  ~PairObject() = default;
 };

+class EnvironmentMap {
+ public:
+  EnvironmentMap(const EnvironmentMap&) = delete;
+  EnvironmentMap& operator=(const EnvironmentMap&) = delete;
+  EnvironmentMap();
+  Object* lookup(InternedSymbolPtr ptr);
+  void set(InternedSymbolPtr ptr, const Object& obj);
+  void clear();
+
+ private:
+  struct Entry {
+    const char* key = nullptr;
+    Object value;
+  };
+  std::vector<Entry> m_entries;
+
+  void resize();
+  int m_power_of_two_size = 0;
+  int m_used_entries = 0;
+  int m_next_resize = 0;
+  u32 m_mask = 0;
+  static constexpr float kMaxUsed = 0.7;
+};
+
 class EnvironmentObject : public HeapObject {
 public:
  std::string name;
  std::shared_ptr<EnvironmentObject> parent_env;

-  // the symbols will be stored in the symbol table and never removed, so we don't need shared
-  // pointers here.
-  std::unordered_map<HeapObject*, Object> vars;
+  // note: this is keyed on the address in the symbol table.
+  EnvironmentMap vars;
+
+  // this find work by any name string
+  Object* find(const char* n, SymbolTable* st) { return vars.lookup(st->intern(n)); }
+  Object* find(InternedSymbolPtr ptr) { return vars.lookup(ptr); }

  EnvironmentObject() = default;

@@ -475,11 +522,8 @@ class EnvironmentObject : public HeapObject {

  std::string inspect() const override {
    std::string result = "[environment]\n  name: " + name +
-                         "\n  parent: " + (parent_env ? parent_env->print() : "NONE") +
-                         "\n  vars:\n";
-    for (const auto& kv : vars) {
-      result += "    " + kv.first->print() + ": " + kv.second.print() + "\n";
-    }
+                         "\n  parent: " + (parent_env ? parent_env->print() : "NONE") + "\n";
+
    return result;
  }
 };
@@ -659,14 +703,6 @@ inline std::shared_ptr<EnvironmentObject> Object::as_env_ptr() const {
  return std::dynamic_pointer_cast<EnvironmentObject>(heap_obj);
 }

-inline SymbolObject* Object::as_symbol() const {
-  if (type != ObjectType::SYMBOL) {
-    throw std::runtime_error("as_symbol called on a " + object_type_to_string(type) + " " +
-                             print());
-  }
-  return static_cast<SymbolObject*>(heap_obj.get());
-}
-
 inline StringObject* Object::as_string() const {
  if (type != ObjectType::STRING) {
    throw std::runtime_error("as_string called on a " + object_type_to_string(type) + " " +
@@ -12,12 +12,12 @@ bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments*
    auto arg = current.as_pair()->car;

    // did we get a ":keyword"
-    if (arg.is_symbol() && arg.as_symbol()->name.at(0) == ':') {
-      auto key_name = arg.as_symbol()->name.substr(1);
+    if (arg.is_symbol() && arg.as_symbol().name_ptr[0] == ':') {
+      auto key_name = arg.as_symbol().name_ptr + 1;

      // check for multiple definition of key
      if (args.named.find(key_name) != args.named.end()) {
-        *err_string = "Key argument " + key_name + " multiply defined";
+        *err_string = fmt::format("Key argument {} multiply defined", key_name);
        return false;
      }

@@ -39,6 +39,17 @@ bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments*
  return true;
 }

+void get_va_no_named(const goos::Object& rest, goos::Arguments* result) {
+  goos::Arguments args;
+  // loop over forms in list
+  goos::Object current = rest;
+  while (!current.is_empty_list()) {
+    args.unnamed.push_back(current.as_pair()->car);
+    current = current.as_pair()->cdr;
+  }
+  *result = args;
+}
+
 bool va_check(
    const goos::Arguments& args,
    const std::vector<std::optional<goos::ObjectType>>& unnamed,
@@ -53,6 +64,7 @@ bool va_check(

  for (size_t i = 0; i < unnamed.size(); i++) {
    if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
+      // special case -- an empty list is a valid pair
      *err_string = fmt::format("Argument {} has type {} but {} was expected\nArgument is: {}", i,
                                object_type_to_string(args.unnamed[i].type),
                                object_type_to_string(unnamed[i].value()), args.unnamed[i].print());
@@ -9,6 +9,7 @@

 namespace goos {
 bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments* result);
+void get_va_no_named(const goos::Object& rest, goos::Arguments* result);
 bool va_check(
    const goos::Arguments& args,
    const std::vector<std::optional<goos::ObjectType>>& unnamed,
@@ -28,4 +29,4 @@ void for_each_in_list(const goos::Object& list, T f) {
 }

 int list_length(const goos::Object& list);
-}  // namespace goos
+}  // namespace goos
@@ -98,7 +98,7 @@ void add_to_token_list(const goos::Object& obj, std::vector<FormToken>* tokens)
      // look for a (quote x) to print as 'x

      auto& first = obj.as_pair()->car;
-      if (first.is_symbol() && first.as_symbol()->name == "quote") {
+      if (first.is_symbol() && first.as_symbol() == "quote") {
        auto& second = obj.as_pair()->cdr;
        if (second.is_pair() && second.as_pair()->cdr.is_empty_list()) {
          tokens->emplace_back(FormToken::TokenKind::QUOTE);
@@ -4,6 +4,7 @@
 #include "common/util/Assert.h"

 #include "third-party/fmt/core.h"
+#include "third-party/fmt/format.h"

 namespace pretty_print {

@@ -100,7 +101,7 @@ inline const std::string quote_symbol(Node::QuoteKind kind) {
    case Node::QuoteKind::UNQUOTE_SPLICING:
      return ",@";
    default:
-      ASSERT_MSG(false, fmt::format("invalid quote kind {}", kind));
+      ASSERT_MSG(false, fmt::format("invalid quote kind {}", fmt::underlying(kind)));
      return "[invalid]";
  }
 }
@@ -236,9 +237,34 @@ void break_list(Node* node) {
  node->top_line_count = 1;

  const std::unordered_set<std::string> sameline_splitters = {
-      "if",     "<",  ">",  "<=",       ">=",  "set!",  "=",       "!=",     "+",
-      "-",      "*",  "/",  "the",      "->",  "and",   "or",      "logand", "logior",
-      "logxor", "+!", "*!", "logtest?", "not", "zero?", "nonzero?"};
+      "if",
+      "<",
+      ">",
+      "<=",
+      ">=",
+      "set!",
+      "=",
+      "!=",
+      "+",
+      "-",
+      "*",
+      "/",
+      "the",
+      "->",
+      "and",
+      "or",
+      "logand",
+      "logior",
+      "logxor",
+      "+!",
+      "*!",
+      "logtest?",
+      "not",
+      "zero?",
+      "nonzero?",
+      "dma-buffer-add-gs-set",
+      "dma-buffer-add-gs-set-flusha",
+  };

  if (node->child_nodes.at(0).kind == Node::Kind::LIST) {
    // ((foo
@@ -262,7 +288,11 @@ void break_list(Node* node) {
      node->sub_elt_indent += name.size();
    } else if (name == "defmethod") {
      // things with 4 things in the top line: (defmethod <method> <type> <args>
-      node->top_line_count = 4;
+      // or just 3 things in the top line: (defmethod <method> <args>
+      node->top_line_count = 3;
+      if (node->child_nodes.size() >= 4 && node->child_nodes[2].kind == Node::Kind::ATOM) {
+        node->top_line_count = 4;
+      }
    } else if (name == "until" || name == "while" || name == "dotimes" || name == "countdown" ||
               name == "when" || name == "behavior" || name == "lambda" || name == "defpart" ||
               name == "define") {
@@ -1,7 +1,11 @@
 #include "Printer.h"

+#include <cmath>
 #include <mutex>

+#include "common/goos/Object.h"
+#include "common/util/print_float.h"
+
 #include "third-party/fmt/core.h"

 namespace pretty_print {
@@ -21,9 +25,7 @@ const std::unordered_map<u32, std::string> const_floats = {{0x40490fda, "PI"},
 goos::Object float_representation(float value) {
  u32 int_value;
  memcpy(&int_value, &value, 4);
-  u8 exp = (int_value >> 23) & 0xff;
-  u32 mant = int_value & 0x7fffff;
-  if ((exp == 0 && mant != 0) || exp == 0xff) {
+  if (!proper_float(value)) {
    // lg::warn("PS2-incompatible float (0x{:08X}) detected! Writing as the-as cast.", int_value);
    return pretty_print::build_list("the-as", "float", fmt::format("#x{:x}", int_value));
  } else if (const_floats.find(int_value) != const_floats.end()) {
@@ -47,7 +49,7 @@ goos::Reader& get_pretty_printer_reader() {

 goos::Object to_symbol(const std::string& str) {
  std::lock_guard<std::mutex> guard(pretty_printer_reader_mutex);
-  return goos::SymbolObject::make_new(get_pretty_printer_reader().symbolTable, str);
+  return goos::Object::make_symbol(&get_pretty_printer_reader().symbolTable, str.c_str());
 }

 goos::Object new_string(const std::string& str) {
@@ -73,14 +75,12 @@ goos::Object build_list(const std::vector<goos::Object>& objects) {
 // build a list out of an array of forms
 goos::Object build_list(const goos::Object* objects, int count) {
  ASSERT(count);
-  auto car = objects[0];
-  goos::Object cdr;
-  if (count - 1) {
-    cdr = build_list(objects + 1, count - 1);
-  } else {
-    cdr = goos::Object::make_empty_list();
+  goos::Object result = goos::Object::make_empty_list();
+  for (int i = count; i-- > 0;) {
+    result = goos::PairObject::make_new(objects[i], result);
  }
-  return goos::PairObject::make_new(car, cdr);
+
+  return result;
 }

 // build a list out of a vector of strings that are converted to symbols
@@ -11,9 +11,8 @@

 #include "Reader.h"

-#include "ReplUtils.h"
-
 #include "common/log/log.h"
+#include "common/repl/util.h"
 #include "common/util/FileUtil.h"
 #include "common/util/FontUtils.h"

@@ -195,7 +194,7 @@ bool Reader::is_valid_source_char(char c) const {
 /*!
 * Prompt the user and read the result.
 */
-std::optional<Object> Reader::read_from_stdin(const std::string& prompt, ReplWrapper& repl) {
+std::optional<Object> Reader::read_from_stdin(const std::string& prompt, REPL::Wrapper& repl) {
  // escape code will make sure that we remove any color
  std::string prompt_full = "\033[0m" + prompt;

@@ -239,17 +238,9 @@ Object Reader::read_from_string(const std::string& str,
 * Read a file
 */
 Object Reader::read_from_file(const std::vector<std::string>& file_path, bool check_encoding) {
-  std::string joined_name;
+  std::string joined_path = fmt::format("{}", fmt::join(file_path, "/"));

-  for (const auto& thing : file_path) {
-    if (!joined_name.empty()) {
-      joined_name += '/';
-    }
-
-    joined_name += thing;
-  }
-
-  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_name);
+  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_path);
  db.insert(textFrag);

  auto result = internal_read(textFrag, check_encoding);
@@ -295,7 +286,7 @@ Object Reader::internal_read(std::shared_ptr<SourceText> text,
  try {
    auto objs = read_list(ts, false);
    if (add_top_level) {
-      return PairObject::make_new(SymbolObject::make_new(symbolTable, "top-level"), objs);
+      return PairObject::make_new(Object::make_symbol(&symbolTable, "top-level"), objs);
    } else {
      return objs;
    }
@@ -468,12 +459,51 @@ bool Reader::read_array(TextStream& stream, Object& o) {
  return true;
 }

+struct ListBuilder {
+  Object head;
+  std::shared_ptr<PairObject> prev_tail;
+  std::shared_ptr<PairObject> tail;
+  int size = 0;
+
+  ListBuilder() { head = Object::make_empty_list(); }
+
+  void push_back(Object&& o) {
+    size++;
+    if (!tail) {
+      tail = std::make_shared<PairObject>(o, Object{});
+      head.type = ObjectType::PAIR;
+      head.heap_obj = tail;
+    } else {
+      auto next = std::make_shared<PairObject>(o, Object{});
+      tail->cdr.type = ObjectType::PAIR;
+      tail->cdr.heap_obj = next;
+      prev_tail = std::move(tail);
+      tail = std::move(next);
+    }
+  }
+
+  Object pop_back() {
+    auto obj = tail->car;
+    tail = std::move(prev_tail);
+    return obj;
+  }
+
+  void finalize() {
+    if (tail) {
+      tail->cdr = Object::make_empty_list();
+    } else {
+      head = Object::make_empty_list();
+    }
+  }
+};
+
 /*!
 * Call this on the character after the open paren.
 */
 Object Reader::read_list(TextStream& ts, bool expect_close_paren) {
  ts.seek_past_whitespace_and_comments();
-  std::vector<Object> objects;
+  // std::vector<Object> objects;
+  ListBuilder list_builder;

  bool got_close_paren = false;      // does this list end?
  bool got_dot = false;              // did we get a . ?
@@ -515,22 +545,23 @@ Object Reader::read_list(TextStream& ts, bool expect_close_paren) {
    }

    // inserter function, used to properly insert a next object
-    auto insert_object = [&](const Object& o) {
+    auto insert_object = [&](Object&& o) {
      if (got_thing_after_dot) {
        throw_reader_error(ts, "A list cannot have multiple entries after the dot", -1);
      }

      if (reader_macro_string_stack.empty()) {
-        objects.push_back(o);
+        list_builder.push_back(std::move(o));
+        // objects.push_back(o);
      } else {
-        Object to_push_back = o;
+        Object to_push_back = std::move(o);
        while (!reader_macro_string_stack.empty()) {
-          to_push_back =
-              build_list({SymbolObject::make_new(symbolTable, reader_macro_string_stack.back()),
-                          to_push_back});
+          to_push_back = build_list(
+              {Object::make_symbol(&symbolTable, reader_macro_string_stack.back().c_str()),
+               to_push_back});
          reader_macro_string_stack.pop_back();
        }
-        objects.push_back(to_push_back);
+        list_builder.push_back(std::move(to_push_back));
      }

      // remember if we got an object after the dot
@@ -560,7 +591,7 @@ Object Reader::read_list(TextStream& ts, bool expect_close_paren) {

      if (read_object(tok, ts, obj)) {
        ts.seek_past_whitespace_and_comments();
-        insert_object(obj);
+        insert_object(std::move(obj));
      } else {
        throw_reader_error(ts, "invalid token encountered in reader: " + tok.text,
                           -int(tok.text.size()));
@@ -583,12 +614,12 @@ Object Reader::read_list(TextStream& ts, bool expect_close_paren) {

  // build up list or improper list, link it, and return!
  if (got_thing_after_dot) {
-    if (objects.size() < 2) {
+    if (list_builder.size < 2) {
      throw_reader_error(ts, "A list with a dot must have at least one thing before the dot", -1);
    }
-    auto back = objects.back();
-    objects.pop_back();
-    auto rv = build_list(objects);
+    auto back = list_builder.pop_back();
+    list_builder.finalize();
+    auto rv = list_builder.head;

    auto lst = rv;
    while (true) {
@@ -602,7 +633,8 @@ Object Reader::read_list(TextStream& ts, bool expect_close_paren) {
    db.link(rv, ts.text, start_offset);
    return rv;
  } else {
-    auto rv = build_list(std::move(objects));
+    list_builder.finalize();
+    auto rv = list_builder.head;
    db.link(rv, ts.text, start_offset);
    return rv;
  }
@@ -616,7 +648,7 @@ bool Reader::try_token_as_symbol(const Token& tok, Object& obj) {
  ASSERT(!tok.text.empty());
  char start = tok.text[0];
  if (m_valid_symbols_chars[(int)start]) {
-    obj = SymbolObject::make_new(symbolTable, tok.text);
+    obj = Object::make_symbol(&symbolTable, tok.text.c_str());
    return true;
  } else {
    return false;
@@ -16,10 +16,9 @@
 #include <unordered_map>
 #include <utility>

-#include "ReplUtils.h"
-
 #include "common/goos/Object.h"
 #include "common/goos/TextDB.h"
+#include "common/repl/util.h"
 #include "common/util/Assert.h"

 namespace goos {
@@ -74,7 +73,7 @@ class Reader {
  Object read_from_string(const std::string& str,
                          bool add_top_level = true,
                          const std::optional<std::string>& string_name = {});
-  std::optional<Object> read_from_stdin(const std::string& prompt, ReplWrapper& repl);
+  std::optional<Object> read_from_stdin(const std::string& prompt, REPL::Wrapper& repl);
  Object read_from_file(const std::vector<std::string>& file_path, bool check_encoding = false);
  bool check_string_is_valid(const std::string& str) const;

@@ -1,135 +0,0 @@
-#include "ReplUtils.h"
-
-#include "common/util/FileUtil.h"
-#include "common/versions.h"
-
-#include "third-party/fmt/color.h"
-#include "third-party/fmt/core.h"
-#include "third-party/replxx/include/replxx.hxx"
-
-// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
-// or perhaps, show the docstring for the current function being used?)
-
-using Replxx = replxx::Replxx;
-
-void ReplWrapper::clear_screen() {
-  repl.clear_screen();
-}
-
-void ReplWrapper::print_welcome_message() {
-  // TODO - dont print on std-out
-  // Welcome message / brief intro for documentation
-  std::string ascii;
-  ascii += " _____             _____ _____ _____ __    \n";
-  ascii += "|     |___ ___ ___|   __|     |  _  |  |   \n";
-  ascii += "|  |  | . | -_|   |  |  |  |  |     |  |__ \n";
-  ascii += "|_____|  _|___|_|_|_____|_____|__|__|_____|\n";
-  ascii += "      |_|                                  \n";
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
-
-  fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
-             versions::GOAL_VERSION_MINOR);
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(repl-help)");
-  fmt::print(" for help with common commands and REPL usage.\n");
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
-  fmt::print(" to connect to the local target.\n");
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
-  fmt::print(" to rebuild the entire game.\n\n");
-}
-
-void ReplWrapper::print_to_repl(const std::string_view& str) {
-  repl.print(str.data());
-}
-
-void ReplWrapper::set_history_max_size(size_t len) {
-  repl.set_max_history_size(len);
-}
-
-const char* ReplWrapper::readline(const std::string& prompt) {
-  return repl.input(prompt);
-}
-
-void ReplWrapper::add_to_history(const std::string& line) {
-  repl.history_add(line);
-}
-
-void ReplWrapper::save_history() {
-  fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
-  file_util::create_dir_if_needed_for_file(path.string());
-  repl.history_save(path.string());
-}
-
-void ReplWrapper::load_history() {
-  fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
-  if (fs::exists(path)) {
-    repl.history_load(path.string());
-  } else {
-    fmt::print("Couldn't locate REPL history file at '{}'\n", path.string());
-  }
-}
-
-std::pair<std::string, bool> ReplWrapper::get_current_repl_token(std::string const& context) {
-  // Find the current token
-  std::string token = "";
-  for (auto c = context.crbegin(); c != context.crend(); c++) {
-    if (std::isspace(*c)) {
-      break;
-    } else {
-      token = *c + token;
-    }
-  }
-
-  // If there is a preceeding '(' remove it
-  if (!token.empty() && token.at(0) == '(') {
-    token.erase(0, 1);
-    return {token, true};
-  }
-  return {token, false};
-}
-
-void ReplWrapper::print_help_message() {
-  fmt::print(fmt::emphasis::bold, "\nREPL Controls:\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:clear)\n");
-  fmt::print(" - Clear the current screen\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(e)\n");
-  fmt::print(" - Exit the compiler\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt [ip-address] [port-number])\n");
-  fmt::print(
-      " - Connect the listener to a running target. The IP address defaults to `127.0.0.1` and the "
-      "port to `8112`\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(r [ip-address] [port-number])\n");
-  fmt::print(
-      " - Attempt to reset the target and reconnect. After this, the target will have nothing "
-      "loaded.\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:status)\n");
-  fmt::print(" - Send a ping-like message to the target. Requires the target to be connected\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(shutdown-target)\n");
-  fmt::print(" - If the target is connected, make it exit\n");
-
-  fmt::print(fmt::emphasis::bold, "\nCompiling & Building:\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mi)\n");
-  fmt::print(" - Build entire game\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mng)\n");
-  fmt::print(" - Build game engine\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(m \"filename\")\n");
-  fmt::print(" - Compile an OpenGOAL source file\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(ml \"filename\")\n");
-  fmt::print(" - Compile and Load (or reload) an OpenGOAL source file\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(build-kernel)\n");
-  fmt::print(" - Build the GOAL kernel\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(make \"file-name\")\n");
-  fmt::print(
-      " - Build a file and any out-of-date dependencies. This file must be a target in the make "
-      "system.\n");
-
-  fmt::print(fmt::emphasis::bold, "\nOther:\n");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::magenta), "(gs)\n");
-  fmt::print(" - Enter a GOOS REPL\n");
-}
-
-void ReplWrapper::init_default_settings() {
-  repl.set_word_break_characters(" \t");
-}
@@ -1,34 +0,0 @@
-#pragma once
-
-#include <functional>
-#include <string>
-#include <vector>
-
-#include "third-party/replxx/include/replxx.hxx"
-
-using Replxx = replxx::Replxx;
-
-class ReplWrapper {
-  Replxx repl;
-
- public:
-  ReplWrapper() {}
-  Replxx& get_repl() { return repl; }
-  void init_default_settings();
-
-  // Functionality / Commands
-  void clear_screen();
-  void print_to_repl(const std::string_view& str);
-  void print_welcome_message();
-  void set_history_max_size(size_t len);
-  const char* readline(const std::string& prompt);
-  void add_to_history(const std::string& line);
-  void save_history();
-  void load_history();
-  void print_help_message();
-  std::pair<std::string, bool> get_current_repl_token(std::string const& context);
-
-  std::vector<std::string> examples{};
-  using cl = Replxx::Color;
-  std::vector<std::pair<std::string, cl>> regex_colors{};
-};
@@ -175,39 +175,45 @@ std::optional<TextDb::ShortInfo> TextDb::get_short_info_for(const std::shared_pt
  return std::make_optional(info_result);
 }

+std::optional<TextDb::ShortInfo> TextDb::try_get_short_info(
+    const std::shared_ptr<goos::HeapObject>& heap_obj) const {
+  auto it = m_map.find(heap_obj);
+  if (it != m_map.end()) {
+    auto& frag = it->second.frag;
+    // shorten the string
+    std::string name = frag->get_description();
+    size_t start = 0;
+    for (size_t i = 0; i < name.size(); i++) {
+      if (name[i] == '/' || name[i] == '\\') {
+        start = i + 1;
+      }
+    }
+    if (start < name.size()) {
+      name = name.substr(start);
+    }
+
+    ShortInfo result;
+    result.filename = name;
+
+    int line_idx = frag->get_line_idx(it->second.offset);
+    result.line_idx_to_display = line_idx + 1;
+
+    int offset_of_line = frag->get_offset_of_line(line_idx);
+    int offset_of_next_line = frag->get_offset_of_line(line_idx + 1);
+
+    int line_length = offset_of_next_line - offset_of_line;
+
+    int start_offset_in_line = it->second.offset - offset_of_line - 1;
+    result.pos_in_line = std::max(start_offset_in_line, 0);
+    result.line_text = std::string(frag->get_text() + offset_of_line + 1, line_length - 1);
+    return result;
+  }
+  return {};
+}
+
 std::optional<TextDb::ShortInfo> TextDb::try_get_short_info(const Object& o) const {
  if (o.is_pair()) {
-    auto it = m_map.find(o.heap_obj);
-    if (it != m_map.end()) {
-      auto& frag = it->second.frag;
-      // shorten the string
-      std::string name = frag->get_description();
-      size_t start = 0;
-      for (size_t i = 0; i < name.size(); i++) {
-        if (name[i] == '/' || name[i] == '\\') {
-          start = i + 1;
-        }
-      }
-      if (start < name.size()) {
-        name = name.substr(start);
-      }
-
-      ShortInfo result;
-      result.filename = name;
-
-      int line_idx = frag->get_line_idx(it->second.offset);
-      result.line_idx_to_display = line_idx + 1;
-
-      int offset_of_line = frag->get_offset_of_line(line_idx);
-      int offset_of_next_line = frag->get_offset_of_line(line_idx + 1);
-
-      int line_length = offset_of_next_line - offset_of_line;
-
-      int start_offset_in_line = it->second.offset - offset_of_line - 1;
-      result.pos_in_line = std::max(start_offset_in_line, 0);
-      result.line_text = std::string(frag->get_text() + offset_of_line + 1, line_length - 1);
-      return result;
-    }
+    return try_get_short_info(o.heap_obj);
  }

  return {};
@@ -245,4 +251,9 @@ void TextDb::inherit_info(const Object& parent, const Object& child) {
    }
  }
 }
+
+void TextDb::clear_info() {
+  m_map.clear();
+  m_fragments.clear();
+}
 }  // namespace goos
@@ -112,8 +112,11 @@ class TextDb {
  std::optional<ShortInfo> get_short_info_for(const std::shared_ptr<SourceText>& frag,
                                              int offset) const;
  std::optional<ShortInfo> try_get_short_info(const Object& o) const;
+  std::optional<ShortInfo> try_get_short_info(const std::shared_ptr<goos::HeapObject>& o) const;
+
  bool has_info(const Object& o) const;
  void inherit_info(const Object& parent, const Object& child);
+  void clear_info();

 private:
  std::vector<std::shared_ptr<SourceText>> m_fragments;
@@ -34,7 +34,7 @@ enum class ListenerMessageKind : u16 {
 */
 enum ListenerToTargetMsgKind : u16 {
  LTT_MSG_POKE = 1,           //! "Poke" the game and have it flush buffers
-  LTT_MSG_INSEPCT = 5,        //! Inspect an object
+  LTT_MSG_INSPECT = 5,        //! Inspect an object
  LTT_MSG_PRINT = 6,          //! Print an object
  LTT_MSG_PRINT_SYMBOLS = 7,  //! Print all symbols
  LTT_MSG_RESET = 8,          //! Reset the game
@@ -12,6 +12,7 @@
 #endif
 #include "common/util/Assert.h"
 #include "common/util/FileUtil.h"
+#include "common/util/string_util.h"

 namespace lg {
 struct Logger {
@@ -23,6 +24,7 @@ struct Logger {
  level file_log_level = level::trace;
  level flush_level = level::trace;
  std::mutex mutex;
+  bool disable_colors = false;

  ~Logger() {
    // will run when program exits.
@@ -35,9 +37,10 @@ struct Logger {
 Logger gLogger;

 namespace internal {
-const char* log_level_names[] = {"trace", "debug", "info", "warn", "error", "die"};
-const fmt::color log_colors[] = {fmt::color::gray,   fmt::color::turquoise, fmt::color::light_green,
-                                 fmt::color::yellow, fmt::color::red,       fmt::color::hot_pink};
+const char* log_level_names[] = {"trace", "debug", "info", "warn", "error", "die", "die"};
+const fmt::color log_colors[] = {
+    fmt::color::gray, fmt::color::turquoise, fmt::color::light_green, fmt::color::yellow,
+    fmt::color::red,  fmt::color::hot_pink,  fmt::color::hot_pink};

 void log_message(level log_level, LogTime& now, const char* message) {
 #ifdef __linux__
@@ -64,9 +67,14 @@ void log_message(level log_level, LogTime& now, const char* message) {
      }
    }

-    if (log_level >= gLogger.stdout_log_level) {
+    if (log_level >= gLogger.stdout_log_level ||
+        (log_level == level::die && gLogger.stdout_log_level == level::off_unless_die)) {
      fmt::print("{} [", time_string);
-      fmt::print(fg(log_colors[int(log_level)]), "{}", log_level_names[int(log_level)]);
+      if (gLogger.disable_colors) {
+        fmt::print("{}", log_level_names[int(log_level)]);
+      } else {
+        fmt::print(fg(log_colors[int(log_level)]), "{}", log_level_names[int(log_level)]);
+      }
      fmt::print("] {}\n", message);
      if (log_level >= gLogger.flush_level) {
        fflush(stdout);
@@ -97,7 +105,7 @@ void log_print(const char* message) {
      fflush(gLogger.fp);
    }

-    if (gLogger.stdout_log_level < lg::level::off) {
+    if (gLogger.stdout_log_level < lg::level::off_unless_die) {
      fmt::print(message);
      fflush(stdout);
      fflush(stderr);
@@ -106,10 +114,55 @@ void log_print(const char* message) {
 }
 }  // namespace internal

-void set_file(const std::string& filename) {
+// how many extra log files for a single program should be kept?
+constexpr int LOG_ROTATE_MAX = 10;
+
+void set_file(const std::string& filename,
+              const bool should_rotate,
+              const bool append,
+              const std::string& dir) {
  ASSERT(!gLogger.fp);
-  file_util::create_dir_if_needed_for_file(filename);
-  gLogger.fp = file_util::open_file(filename.c_str(), "w");
+  std::string file_path;
+  if (!dir.empty()) {
+    file_path = file_util::combine_path(dir, filename);
+  } else {
+    file_path = file_util::get_file_path({"log", filename});
+  }
+  file_util::create_dir_if_needed_for_file(file_path);
+  std::string complete_filename = file_path;
+  if (should_rotate) {
+    complete_filename += "." + str_util::current_local_timestamp_no_colons() + ".log";
+    // remove any log files with the old format
+    auto old_log_files =
+        file_util::find_files_in_dir(fs::path(complete_filename).parent_path(),
+                                     std::regex(fmt::format("{}\\.(\\d\\.)?log", filename)));
+    for (const auto& file : old_log_files) {
+      lg::info("removing {}", file.string());
+      fs::remove(file);
+    }
+    // remove the oldest log file if there are more than LOG_ROTATE_MAX
+    auto existing_log_files = file_util::find_files_in_dir(
+        fs::path(complete_filename).parent_path(), std::regex(fmt::format("{}.*\\.log", filename)));
+    // sort the names and remove them
+    existing_log_files = file_util::sort_filepaths(existing_log_files, true);
+    if (existing_log_files.size() > (LOG_ROTATE_MAX - 1)) {
+      lg::info("removing {} log files", existing_log_files.size() - (LOG_ROTATE_MAX - 1));
+      for (int i = 0; i < (int)existing_log_files.size() - (LOG_ROTATE_MAX - 1); i++) {
+        lg::info("removing {}", existing_log_files.at(i).string());
+        fs::remove(existing_log_files.at(i));
+      }
+    }
+  } else {
+    if (!str_util::ends_with(complete_filename, ".log")) {
+      complete_filename += ".log";
+    }
+  }
+
+  if (append) {
+    gLogger.fp = file_util::open_file(complete_filename.c_str(), "a");
+  } else {
+    gLogger.fp = file_util::open_file(complete_filename.c_str(), "w");
+  }
  ASSERT(gLogger.fp);
 }

@@ -131,6 +184,10 @@ void set_max_debug_levels() {
  gLogger.file_log_level = level::trace;
 }

+void disable_ansi_colors() {
+  gLogger.disable_colors = true;
+}
+
 void initialize() {
  ASSERT(!gLogger.initialized);

@@ -23,7 +23,16 @@ struct LogTime {
 #endif

 // Logging API
-enum class level { trace = 0, debug = 1, info = 2, warn = 3, error = 4, die = 5, off = 6 };
+enum class level {
+  trace = 0,
+  debug = 1,
+  info = 2,
+  warn = 3,
+  error = 4,
+  die = 5,
+  off_unless_die = 6,
+  off = 7
+};

 namespace internal {
 // log implementation stuff, not to be called by the user
@@ -31,11 +40,15 @@ void log_message(level log_level, LogTime& now, const char* message);
 void log_print(const char* message);
 }  // namespace internal

-void set_file(const std::string& filename);
+void set_file(const std::string& filename,
+              const bool should_rotate = true,
+              const bool append = false,
+              const std::string& dir = "");
 void set_flush_level(level log_level);
 void set_file_level(level log_level);
 void set_stdout_level(level log_level);
 void set_max_debug_levels();
+void disable_ansi_colors();
 void initialize();
 void finish();

@@ -18,8 +18,16 @@ class Vector {
    return result;
  }

+  static Vector<T, Size> unit(int idx) {
+    Vector<T, Size> result = Vector<T, Size>::zero();
+    result[idx] = T(1);
+    return result;
+  }
+
  template <typename... Args>
-  explicit Vector(Args... args) : m_data{T(args)...} {}
+  constexpr Vector(Args... args) : m_data{T(args)...} {
+    static_assert(sizeof...(args) == Size, "Incorrect number of args");
+  }

  T* begin() { return &m_data[0]; }
  T* end() { return &m_data[Size]; }
@@ -128,6 +136,13 @@ class Vector {
    return *this;
  }

+  Vector<T, Size>& operator+=(const T& other) {
+    for (int i = 0; i < Size; i++) {
+      m_data[i] += other;
+    }
+    return *this;
+  }
+
  Vector<T, Size> elementwise_multiply(const Vector<T, Size>& other) const {
    Vector<T, Size> result;
    for (int i = 0; i < Size; i++) {
@@ -217,6 +232,22 @@ class Vector {
    }
  }

+  Vector<T, Size> min(const Vector<T, Size>& other) const {
+    Vector<T, Size> result;
+    for (int i = 0; i < Size; i++) {
+      result[i] = std::min(m_data[i], other[i]);
+    }
+    return result;
+  }
+
+  Vector<T, Size> max(const Vector<T, Size>& other) const {
+    Vector<T, Size> result;
+    for (int i = 0; i < Size; i++) {
+      result[i] = std::max(m_data[i], other[i]);
+    }
+    return result;
+  }
+
  std::string to_string_aligned() const {
    std::string result = "[";
    for (auto x : m_data) {
@@ -50,4 +50,78 @@ math::Vector4f bsphere_of_triangle(const Vector3f* vertices);
 inline bool point_in_bsphere(const Vector4f& sphere, const Vector3f& pt) {
  return (sphere.xyz() - pt).squared_length() <= (sphere.w() * sphere.w());
 }
+
+template <typename T>
+math::Matrix<T, 4, 4> affine_inverse(const math::Matrix<T, 4, 4>& in) {
+  math::Matrix<T, 4, 4> result;
+
+  // transpose rotation
+  for (int i = 0; i < 3; i++) {
+    for (int j = 0; j < 3; j++) {
+      result(i, j) = in(j, i);
+    }
+  }
+
+  result(3, 0) = 0;
+  result(3, 1) = 0;
+  result(3, 2) = 0;
+  result(3, 3) = 1;
+  result(0, 3) = 0;
+  result(1, 3) = 0;
+  result(2, 3) = 0;
+  for (int rx = 0; rx < 3; rx++) {
+    for (int cx = 0; cx < 3; cx++) {
+      result(rx, 3) -= result(rx, cx) * in(cx, 3);
+    }
+  }
+
+  return result;
+}
+
+template <typename T>
+math::Matrix<T, 4, 4> inverse(const math::Matrix<T, 4, 4>& m) {
+  math::Matrix<T, 4, 4> im;
+  T A2323 = m(2, 2) * m(3, 3) - m(2, 3) * m(3, 2);
+  T A1323 = m(2, 1) * m(3, 3) - m(2, 3) * m(3, 1);
+  T A1223 = m(2, 1) * m(3, 2) - m(2, 2) * m(3, 1);
+  T A0323 = m(2, 0) * m(3, 3) - m(2, 3) * m(3, 0);
+  T A0223 = m(2, 0) * m(3, 2) - m(2, 2) * m(3, 0);
+  T A0123 = m(2, 0) * m(3, 1) - m(2, 1) * m(3, 0);
+  T A2313 = m(1, 2) * m(3, 3) - m(1, 3) * m(3, 2);
+  T A1313 = m(1, 1) * m(3, 3) - m(1, 3) * m(3, 1);
+  T A1213 = m(1, 1) * m(3, 2) - m(1, 2) * m(3, 1);
+  T A2312 = m(1, 2) * m(2, 3) - m(1, 3) * m(2, 2);
+  T A1312 = m(1, 1) * m(2, 3) - m(1, 3) * m(2, 1);
+  T A1212 = m(1, 1) * m(2, 2) - m(1, 2) * m(2, 1);
+  T A0313 = m(1, 0) * m(3, 3) - m(1, 3) * m(3, 0);
+  T A0213 = m(1, 0) * m(3, 2) - m(1, 2) * m(3, 0);
+  T A0312 = m(1, 0) * m(2, 3) - m(1, 3) * m(2, 0);
+  T A0212 = m(1, 0) * m(2, 2) - m(1, 2) * m(2, 0);
+  T A0113 = m(1, 0) * m(3, 1) - m(1, 1) * m(3, 0);
+  T A0112 = m(1, 0) * m(2, 1) - m(1, 1) * m(2, 0);
+
+  T det = m(0, 0) * (m(1, 1) * A2323 - m(1, 2) * A1323 + m(1, 3) * A1223) -
+          m(0, 1) * (m(1, 0) * A2323 - m(1, 2) * A0323 + m(1, 3) * A0223) +
+          m(0, 2) * (m(1, 0) * A1323 - m(1, 1) * A0323 + m(1, 3) * A0123) -
+          m(0, 3) * (m(1, 0) * A1223 - m(1, 1) * A0223 + m(1, 2) * A0123);
+  det = 1 / det;
+
+  im(0, 0) = det * (m(1, 1) * A2323 - m(1, 2) * A1323 + m(1, 3) * A1223);
+  im(0, 1) = det * -(m(0, 1) * A2323 - m(0, 2) * A1323 + m(0, 3) * A1223);
+  im(0, 2) = det * (m(0, 1) * A2313 - m(0, 2) * A1313 + m(0, 3) * A1213);
+  im(0, 3) = det * -(m(0, 1) * A2312 - m(0, 2) * A1312 + m(0, 3) * A1212);
+  im(1, 0) = det * -(m(1, 0) * A2323 - m(1, 2) * A0323 + m(1, 3) * A0223);
+  im(1, 1) = det * (m(0, 0) * A2323 - m(0, 2) * A0323 + m(0, 3) * A0223);
+  im(1, 2) = det * -(m(0, 0) * A2313 - m(0, 2) * A0313 + m(0, 3) * A0213);
+  im(1, 3) = det * (m(0, 0) * A2312 - m(0, 2) * A0312 + m(0, 3) * A0212);
+  im(2, 0) = det * (m(1, 0) * A1323 - m(1, 1) * A0323 + m(1, 3) * A0123);
+  im(2, 1) = det * -(m(0, 0) * A1323 - m(0, 1) * A0323 + m(0, 3) * A0123);
+  im(2, 2) = det * (m(0, 0) * A1313 - m(0, 1) * A0313 + m(0, 3) * A0113);
+  im(2, 3) = det * -(m(0, 0) * A1312 - m(0, 1) * A0312 + m(0, 3) * A0112);
+  im(3, 0) = det * -(m(1, 0) * A1223 - m(1, 1) * A0223 + m(1, 2) * A0123);
+  im(3, 1) = det * (m(0, 0) * A1223 - m(0, 1) * A0223 + m(0, 2) * A0123);
+  im(3, 2) = det * -(m(0, 0) * A1213 - m(0, 1) * A0213 + m(0, 2) * A0113);
+  im(3, 3) = det * (m(0, 0) * A1212 - m(0, 1) * A0212 + m(0, 2) * A0112);
+  return im;
+}
 }  // namespace math
@@ -0,0 +1,103 @@
+#include "config.h"
+
+#include "common/versions/versions.h"
+
+#include "third-party/fmt/core.h"
+
+namespace REPL {
+void to_json(json& j, const Config& obj) {
+  j = json{
+      {"gameVersionFolder", obj.game_version_folder},
+      {"numConnectToTargetAttempts", obj.target_connect_attempts},
+      {"asmFileSearchDirs", obj.asm_file_search_dirs},
+      {"keybinds", obj.keybinds},
+      {"perGameHistory", obj.per_game_history},
+  };
+}
+
+void from_json(const json& j, Config& obj) {
+  if (j.contains("gameVersionFolder")) {
+    j.at("gameVersionFolder").get_to(obj.game_version_folder);
+  }
+  if (j.contains("numConnectToTargetAttempts")) {
+    j.at("numConnectToTargetAttempts").get_to(obj.target_connect_attempts);
+  }
+  if (j.contains("asmFileSearchDirs")) {
+    j.at("asmFileSearchDirs").get_to(obj.asm_file_search_dirs);
+  }
+  if (j.contains("appendKeybinds")) {
+    j.at("appendKeybinds").get_to(obj.append_keybinds);
+  }
+  if (j.contains("keybinds")) {
+    std::vector<KeyBind> keybinds = j.at("keybinds");
+    if (!obj.append_keybinds) {
+      obj.keybinds = keybinds;
+    } else {
+      // append the keybinds
+      // - start with the provided ones
+      // - skip ones from the default set if they have the same key + modifier combination
+      for (const auto& default_bind : obj.keybinds) {
+        // check if it's a duplicate bind
+        bool duplicate = false;
+        for (const auto& new_bind : keybinds) {
+          if (new_bind.key == default_bind.key && new_bind.modifier == default_bind.modifier) {
+            duplicate = true;
+            break;
+          }
+        }
+        if (!duplicate) {
+          keybinds.push_back(default_bind);
+        }
+      }
+      obj.keybinds = keybinds;
+    }
+  }
+  if (j.contains("perGameHistory")) {
+    j.at("perGameHistory").get_to(obj.per_game_history);
+  }
+  // if there is game specific configuration, override any values we just set
+  if (j.contains(version_to_game_name(obj.game_version))) {
+    from_json(j.at(version_to_game_name(obj.game_version)), obj);
+  }
+}
+
+std::string KeyBind::string() const {
+  switch (modifier) {
+    case KeyBind::Modifier::CTRL:
+      return fmt::format("CTRL-{}", key);
+    case KeyBind::Modifier::SHIFT:
+      return fmt::format("SHIFT-{}", key);
+    case KeyBind::Modifier::META:
+      return fmt::format("META-{}", key);
+  }
+}
+
+void to_json(json& j, const KeyBind& obj) {
+  j = json{{"description", obj.description}, {"command", obj.command}, {"key", obj.key}};
+  switch (obj.modifier) {
+    case KeyBind::Modifier::CTRL:
+      j["modifier"] = "ctrl";
+      break;
+    case KeyBind::Modifier::SHIFT:
+      j["modifier"] = "shift";
+      break;
+    case KeyBind::Modifier::META:
+      j["modifier"] = "meta";
+      break;
+  }
+}
+
+void from_json(const json& j, KeyBind& obj) {
+  j.at("description").get_to(obj.description);
+  j.at("command").get_to(obj.command);
+  j.at("key").get_to(obj.key);
+  auto modString = j.at("modifier").get<std::string>();
+  if (modString == "ctrl") {
+    obj.modifier = KeyBind::Modifier::CTRL;
+  } else if (modString == "shift") {
+    obj.modifier = KeyBind::Modifier::SHIFT;
+  } else if (modString == "meta") {
+    obj.modifier = KeyBind::Modifier::META;
+  }
+}
+}  // namespace REPL
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/versions/versions.h"
+
+#include "third-party/json.hpp"
+
+using json = nlohmann::json;
+
+namespace REPL {
+struct KeyBind {
+  // NOTE - in my experience, meta doesn't work on windows and shift is probably a bad idea when
+  // typing text! but I leave it up to the user.
+  enum class Modifier { CTRL, SHIFT, META };
+  Modifier modifier;
+  std::string key;
+  std::string description;
+  std::string command;
+
+  std::string string() const;
+};
+void to_json(json& j, const KeyBind& obj);
+void from_json(const json& j, KeyBind& obj);
+
+struct Config {
+  GameVersion game_version;
+  Config(GameVersion _game_version) : game_version(_game_version){};
+
+  // this is the default REPL configuration
+  std::string game_version_folder;
+  int target_connect_attempts = 30;
+  std::vector<std::string> asm_file_search_dirs = {};
+  bool append_keybinds = true;
+  std::vector<KeyBind> keybinds = {
+      {KeyBind::Modifier::CTRL, "T", "Starts up the game runtime", "(test-play)"},
+      {KeyBind::Modifier::CTRL, "Q", "Exit the REPL", "(e)"},
+      {KeyBind::Modifier::CTRL, "L", "Listen to an available game process", "(lt)"},
+      {KeyBind::Modifier::CTRL, "W",
+       "Halt the attached process so you can re-launch a crashed game", "(:stop)"},
+      {KeyBind::Modifier::CTRL, "G", "Attach the debugger to the process", "(dbgc)"},
+      {KeyBind::Modifier::CTRL, "B", "Displays the most recently caught backtrace", "(:di)"},
+      {KeyBind::Modifier::CTRL, "N", "Full build of the game", "(mi)"}};
+  bool per_game_history = true;
+};
+void to_json(json& j, const Config& obj);
+void from_json(const json& j, Config& obj);
+
+}  // namespace REPL
@@ -2,7 +2,7 @@
 #include "ReplClient.h"

 #include "common/cross_sockets/XSocket.h"
-#include "common/versions.h"
+#include "common/versions/versions.h"

 #include "third-party/fmt/core.h"

@@ -2,7 +2,7 @@
 #include "ReplServer.h"

 #include "common/cross_sockets/XSocket.h"
-#include <common/versions.h>
+#include "common/versions/versions.h"

 #include "third-party/fmt/core.h"

@@ -13,6 +13,7 @@
 #include <WinSock2.h>
 #include <WS2tcpip.h>
 #endif
+#include "common/log/log.h"
 // clang-format on

 // TODO - basically REPL to listen and inject commands into a running REPL
@@ -30,7 +31,7 @@ ReplServer::~ReplServer() {

 void ReplServer::post_init() {
  // Add the listening socket to our set of sockets
-  fmt::print("[nREPL:{}:{}] awaiting connections\n", tcp_port, listening_socket);
+  lg::info("[nREPL:{}:{}] awaiting connections", tcp_port, listening_socket);
 }

 void ReplServer::ping_response(int socket) {
@@ -38,8 +39,8 @@ void ReplServer::ping_response(int socket) {
                                 versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
  auto resp = write_to_socket(socket, ping.c_str(), ping.size());
  if (resp == -1) {
-    fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
-               ntohs(addr.sin_port), socket);
+    lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
+             ntohs(addr.sin_port), socket);
    close_socket(socket);
    client_sockets.erase(socket);
  }
@@ -79,8 +80,8 @@ std::optional<std::string> ReplServer::get_msg() {
    if (new_socket < 0) {
      // TODO - handle error
    } else {
-      fmt::print("[nREPL:{}]: New socket connection: {}:{}:{}\n", tcp_port,
-                 inet_ntoa(addr.sin_addr), ntohs(addr.sin_port), new_socket);
+      lg::info("[nREPL:{}]: New socket connection: {}:{}:{}", tcp_port, address_to_string(addr),
+               ntohs(addr.sin_port), new_socket);

      // Say hello
      ping_response(new_socket);
@@ -107,10 +108,10 @@ std::optional<std::string> ReplServer::get_msg() {
      auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
      if (req_bytes == 0) {
        // Socket disconnected
-        // TODO - add a queue of messages in the ReplWrapper so we can print _BEFORE_ the prompt is
-        // output
-        fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
-                   ntohs(addr.sin_port), sock);
+        // TODO - add a queue of messages in the REPL::Wrapper so we can print _BEFORE_ the prompt
+        // is output
+        lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
+                 ntohs(addr.sin_port), sock);

        // Cleanup the socket and remove it from our set
        close_socket(sock);
@@ -121,12 +122,17 @@ std::optional<std::string> ReplServer::get_msg() {
        // get the body of the message
        int expected_size = header->length;
        int got = 0;
+        int tries = 0;
        while (got < expected_size) {
+          tries++;
+          if (tries > 100) {
+            break;
+          }
          if (got + expected_size > (int)buffer.size()) {
-            fmt::print(stderr,
-                       "[nREPL:{}]: Bad message, aborting the read.  Got :{}, Expected: {}, Buffer "
-                       "Size: {}",
-                       tcp_port, got, expected_size, buffer.size());
+            lg::error(
+                "[nREPL:{}]: Bad message, aborting the read.  Got :{}, Expected: {}, Buffer "
+                "Size: {}",
+                tcp_port, got, expected_size, buffer.size());
            return std::nullopt;
          }
          auto x = read_from_socket(sock, buffer.data() + got, expected_size - got);
@@ -142,6 +148,7 @@ std::optional<std::string> ReplServer::get_msg() {
            return std::nullopt;
          case ReplServerMessageType::EVAL:
            std::string msg(buffer.data(), header->length);
+            lg::debug("[nREPL:{}] Received Message: {}", tcp_port, msg);
            return std::make_optional(msg);
        }
      }
@@ -0,0 +1,261 @@
+#include "util.h"
+
+#include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
+#include "common/util/string_util.h"
+#include "common/versions/versions.h"
+
+#include "third-party/fmt/color.h"
+#include "third-party/fmt/core.h"
+#include "third-party/replxx/include/replxx.hxx"
+// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
+// or perhaps, show the docstring for the current function being used?)
+
+namespace REPL {
+void Wrapper::clear_screen() {
+  repl.clear_screen();
+}
+
+void Wrapper::print_welcome_message() {
+  // TODO - dont print on std-out
+  // Welcome message / brief intro for documentation
+  std::string ascii;
+  ascii += " _____             _____ _____ _____ __    \n";
+  ascii += "|     |___ ___ ___|   __|     |  _  |  |   \n";
+  ascii += "|  |  | . | -_|   |  |  |  |  |     |  |__ \n";
+  ascii += "|_____|  _|___|_|_|_____|_____|__|__|_____|\n";
+  ascii += "      |_|                                  \n";
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
+
+  fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
+             versions::GOAL_VERSION_MINOR);
+  fmt::print("Run {} or {} for help with common commands and REPL usage.\n",
+             fmt::styled("(repl-help)", fmt::emphasis::bold | fg(fmt::color::cyan)),
+             fmt::styled("(repl-keybinds)", fmt::emphasis::bold | fg(fmt::color::cyan)));
+  fmt::print("Run ");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
+  fmt::print(" to connect to the local target.\n");
+  fmt::print("Run ");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
+  fmt::print(" to rebuild the entire game.\n\n");
+}
+
+void Wrapper::print_to_repl(const std::string& str) {
+  repl.print(str.data());
+}
+
+void Wrapper::set_history_max_size(size_t len) {
+  repl.set_max_history_size(len);
+}
+
+const char* Wrapper::readline(const std::string& prompt) {
+  return repl.input(prompt);
+}
+
+void Wrapper::add_to_history(const std::string& line) {
+  repl.history_add(line);
+}
+
+void Wrapper::save_history() {
+  fs::path path;
+  if (repl_config.per_game_history) {
+    path = file_util::get_user_config_dir() / game_version_names[repl_config.game_version] /
+           ".opengoal.repl.history";
+  } else {
+    path = file_util::get_user_config_dir() / ".opengoal.repl.history";
+  }
+  file_util::create_dir_if_needed_for_file(path.string());
+  repl.history_save(path.string());
+}
+
+void Wrapper::load_history() {
+  fs::path path;
+  if (repl_config.per_game_history) {
+    path = file_util::get_user_config_dir() / game_version_names[repl_config.game_version] /
+           ".opengoal.repl.history";
+  } else {
+    path = file_util::get_user_config_dir() / ".opengoal.repl.history";
+  }
+  if (fs::exists(path)) {
+    repl.history_load(path.string());
+  } else {
+    fmt::print("Couldn't locate REPL history file at '{}'\n", path.string());
+  }
+}
+
+std::pair<std::string, bool> Wrapper::get_current_repl_token(std::string const& context) {
+  // Find the current token
+  std::string token = "";
+  for (auto c = context.crbegin(); c != context.crend(); c++) {
+    if (std::isspace(*c)) {
+      break;
+    } else {
+      token = *c + token;
+    }
+  }
+
+  // If there is a preceeding '(' remove it
+  if (!token.empty() && token.at(0) == '(') {
+    token.erase(0, 1);
+    return {token, true};
+  }
+  return {token, false};
+}
+
+void Wrapper::print_help_message() {
+  fmt::print(fmt::emphasis::bold, "\nREPL Controls:\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:clear)\n");
+  fmt::print(" - Clear the current screen\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(e)\n");
+  fmt::print(" - Exit the compiler\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt [ip-address] [port-number])\n");
+  fmt::print(
+      " - Connect the listener to a running target. The IP address defaults to `127.0.0.1` and the "
+      "port to `8112`\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(r [ip-address] [port-number])\n");
+  fmt::print(
+      " - Attempt to reset the target and reconnect. After this, the target will have nothing "
+      "loaded.\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:status)\n");
+  fmt::print(" - Send a ping-like message to the target. Requires the target to be connected\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(shutdown-target)\n");
+  fmt::print(" - If the target is connected, make it exit\n");
+
+  fmt::print(fmt::emphasis::bold, "\nCompiling & Building:\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mi)\n");
+  fmt::print(" - Build entire game\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mng)\n");
+  fmt::print(" - Build game engine\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(m \"filename\")\n");
+  fmt::print(" - Compile an OpenGOAL source file\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(ml \"filename\")\n");
+  fmt::print(" - Compile and Load (or reload) an OpenGOAL source file\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(build-kernel)\n");
+  fmt::print(" - Build the GOAL kernel\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(make \"file-name\")\n");
+  fmt::print(
+      " - Build a file and any out-of-date dependencies. This file must be a target in the make "
+      "system.\n");
+
+  fmt::print(fmt::emphasis::bold, "\nOther:\n");
+  fmt::print(fmt::emphasis::bold | fg(fmt::color::magenta), "(gs)\n");
+  fmt::print(" - Enter a GOOS REPL\n");
+}
+
+void Wrapper::print_keybind_help() {
+  fmt::print(fmt::emphasis::bold, "\nREPL KeyBinds:\n");
+  for (const auto& bind : repl_config.keybinds) {
+    fmt::print("{}\n", fmt::styled(bind.string(), fmt::fg(fmt::color::cyan)));
+    fmt::print("{}\n", fmt::styled(bind.description, fmt::fg(fmt::color::gray)));
+  }
+}
+
+replxx::Replxx::key_press_handler_t Wrapper::commit_text_action(std::string text_to_commit) {
+  return [this, text_to_commit](char32_t code) {
+    repl.set_state(
+        replxx::Replxx::State(text_to_commit.c_str(), static_cast<int>(text_to_commit.size())));
+    return repl.invoke(replxx::Replxx::ACTION::COMMIT_LINE, code);
+  };
+}
+
+void Wrapper::init_settings() {
+  // NOTE - a nice popular project that uses replxx
+  // - https://github.com/ClickHouse/ClickHouse/blob/master/base/base/ReplxxLineReader.cpp#L366
+  repl.set_word_break_characters(" \t");
+  repl.set_complete_on_empty(false);
+  repl.set_indent_multiline(false);
+  repl.enable_bracketed_paste();
+  // Setup default keybinds
+  for (const auto& bind : repl_config.keybinds) {
+    char32_t code;
+    switch (bind.modifier) {
+      case KeyBind::Modifier::CTRL:
+        code = replxx::Replxx::KEY::control(bind.key.at(0));
+        break;
+      case KeyBind::Modifier::SHIFT:
+        code = replxx::Replxx::KEY::shift(bind.key.at(0));
+        break;
+      case KeyBind::Modifier::META:
+        code = replxx::Replxx::KEY::meta(bind.key.at(0));
+        break;
+    }
+    repl.bind_key(code, commit_text_action(bind.command));
+  }
+}
+
+void Wrapper::reload_startup_file() {
+  startup_file = load_user_startup_file(username, repl_config.game_version);
+}
+
+std::string find_repl_username() {
+  // Two options - either:
+  // 1. look for the `user.txt` file, which should only contain the username
+  // 2. if this is absent AND there is a single folder inside the "user" folder, use that as the
+  // username
+  auto user_dir = file_util::get_jak_project_dir() / "goal_src" / "user";
+  auto dirs = file_util::find_directories_in_dir(user_dir);
+  if (dirs.size() == 1) {
+    return dirs.at(0).filename().string();
+  }
+
+  std::regex allowed_chars("(^[0-9a-zA-Z\\-\\.\\!\\?<>]*$)");
+  if (file_util::file_exists((user_dir / "user.txt").string())) {
+    auto text = file_util::read_text_file(user_dir / "user.txt");
+    text = str_util::trim(text);
+    if (!text.empty() && std::regex_match(text, allowed_chars)) {
+      return text;
+    }
+  }
+
+  return "unknown";
+}
+
+fs::path get_startup_file_path(const std::string& username, const GameVersion game_version) {
+  // - first check to see if there is a game version specific startup file to prefer
+  auto game_specific_path = file_util::get_jak_project_dir() / "goal_src" / "user" / username /
+                            fmt::format("startup-{}.gc", version_to_game_name(game_version));
+  if (file_util::file_exists(game_specific_path.string())) {
+    return game_specific_path;
+  }
+  return file_util::get_jak_project_dir() / "goal_src" / "user" / username / "startup.gc";
+}
+
+StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version) {
+  // Check for a `startup.gc` file, each line will be executed on the REPL on startup
+  auto startup_file_path = get_startup_file_path(username, game_version);
+  StartupFile startup_file;
+  if (file_util::file_exists(startup_file_path.string())) {
+    auto data = file_util::read_text_file(startup_file_path);
+    auto startup_cmds = str_util::split(data);
+    bool found_run_on_listen_line = false;
+    for (const auto& cmd : startup_cmds) {
+      if (found_run_on_listen_line) {
+        startup_file.run_after_listen.push_back(cmd);
+      } else {
+        startup_file.run_before_listen.push_back(cmd);
+      }
+      if (str_util::contains(cmd, "og:run-below-on-listen")) {
+        found_run_on_listen_line = true;
+      }
+    }
+  }
+  return startup_file;
+}
+
+REPL::Config load_repl_config(const std::string& username, const GameVersion game_version) {
+  auto repl_config_path =
+      file_util::get_jak_project_dir() / "goal_src" / "user" / username / "repl-config.json";
+  if (file_util::file_exists(repl_config_path.string())) {
+    try {
+      REPL::Config config(game_version);
+      auto repl_config_data =
+          parse_commented_json(file_util::read_text_file(repl_config_path), "repl-config.json");
+      from_json(repl_config_data, config);
+      return config;
+    } catch (std::exception& e) {
+      REPL::Config config(game_version);
+    }
+  }
+  return REPL::Config(game_version);
+}
+}  // namespace REPL
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <functional>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "config.h"
+
+#include "third-party/replxx/include/replxx.hxx"
+
+namespace REPL {
+
+struct StartupFile {
+  std::vector<std::string> run_before_listen = {};
+  std::vector<std::string> run_after_listen = {};
+};
+
+class Wrapper {
+  replxx::Replxx repl;
+
+ public:
+  std::string username;
+  Config repl_config;
+  StartupFile startup_file;
+  std::vector<std::string> examples{};
+  std::vector<std::pair<std::string, replxx::Replxx::Color>> regex_colors{};
+
+  Wrapper(GameVersion version) : repl_config(version) {}
+  Wrapper(const std::string& _username, const Config& config, const StartupFile& startup)
+      : username(_username), repl_config(config), startup_file(startup) {}
+  replxx::Replxx& get_repl() { return repl; }
+  void init_settings();
+  void reload_startup_file();
+
+  // Functionality / Commands
+  void clear_screen();
+  void print_to_repl(const std::string& str);
+  void print_welcome_message();
+  void set_history_max_size(size_t len);
+  const char* readline(const std::string& prompt);
+  void add_to_history(const std::string& line);
+  void save_history();
+  void load_history();
+  void print_help_message();
+  void print_keybind_help();
+  std::pair<std::string, bool> get_current_repl_token(std::string const& context);
+
+ private:
+  replxx::Replxx::key_press_handler_t commit_text_action(std::string text_to_commit);
+  std::vector<REPL::KeyBind> keybindings = {};
+};
+
+std::string find_repl_username();
+StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version);
+REPL::Config load_repl_config(const std::string& username, const GameVersion game_version);
+}  // namespace REPL
@@ -0,0 +1,89 @@
+#include "subtitles.h"
+
+#include "common/goos/ParseHelpers.h"
+#include "common/goos/Reader.h"
+#include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
+
+#include "third-party/fmt/core.h"
+
+void open_subtitle_project(const std::string& project_kind,
+                           const std::string& file_path,
+                           std::vector<GameSubtitleDefinitionFile>& subtitle_files) {
+  goos::Reader reader;
+  auto& proj = reader.read_from_file({file_path}).as_pair()->cdr.as_pair()->car;
+  if (!proj.is_pair() || !proj.as_pair()->car.is_symbol() ||
+      proj.as_pair()->car.as_symbol() != project_kind) {
+    throw std::runtime_error(fmt::format("invalid project '{}'", project_kind));
+  }
+
+  goos::for_each_in_list(proj.as_pair()->cdr, [&](const goos::Object& o) {
+    if (o.is_pair() && o.as_pair()->cdr.is_pair()) {
+      auto args = o.as_pair();
+      auto& action = args->car.as_symbol();
+      args = args->cdr.as_pair();
+
+      if (action == "file-json") {
+        auto new_file = GameSubtitleDefinitionFile();
+        while (true) {
+          const auto& kwarg = args->car.as_symbol();
+          args = args->cdr.as_pair();
+          if (kwarg == ":language-id") {
+            new_file.language_id = args->car.as_int();
+          } else if (kwarg == ":text-version") {
+            new_file.text_version = args->car.as_string()->data;
+          } else if (kwarg == ":lines") {
+            new_file.lines_path = args->car.as_string()->data;
+          } else if (kwarg == ":meta") {
+            new_file.meta_path = args->car.as_string()->data;
+          } else if (kwarg == ":lines-base") {
+            new_file.lines_base_path = args->car.as_string()->data;
+          } else if (kwarg == ":meta-base") {
+            new_file.meta_base_path = args->car.as_string()->data;
+          }
+          if (args->cdr.is_empty_list()) {
+            break;
+          }
+          args = args->cdr.as_pair();
+        }
+        subtitle_files.push_back(new_file);
+      } else {
+        throw std::runtime_error(
+            fmt::format("unknown action {} in {} project", action.name_ptr, project_kind));
+      }
+    } else {
+      throw std::runtime_error(fmt::format("invalid entry in {} project", project_kind));
+    }
+  });
+}
+
+const std::unordered_map<GameVersion, std::vector<std::string>> locale_lookup = {
+    {GameVersion::Jak1,
+     {"en-US", "fr-FR", "de-DE", "es-ES", "it-IT", "jp-JP", "en-GB", "pt-PT", "fi-FI", "sv-SE",
+      "da-DK", "no-NO", "nl-NL", "pt-BR", "hu-HU", "ca-ES", "is-IS"}},
+    {GameVersion::Jak2, {"en-US", "fr-FR", "de-DE", "es-ES", "it-IT", "jp-JP", "ko-KR", "en-GB"}}};
+
+std::string lookup_locale_code(const GameVersion game_version, const int language_id) {
+  if (locale_lookup.find(game_version) == locale_lookup.end() ||
+      (int)locale_lookup.at(game_version).size() < language_id) {
+    return "";
+  }
+  return locale_lookup.at(game_version).at(language_id);
+}
+
+const std::unordered_map<GameVersion, std::vector<int>> language_ids_with_audio = {
+    {GameVersion::Jak1, {0, 1, 2, 3, 4, 5, 6}},
+    {GameVersion::Jak2, {0, 1, 2, 3, 4, 5, 6, 7}}};
+
+bool dump_language_with_duplicates_from_base(const GameVersion game_version,
+                                             const int language_id) {
+  if (language_ids_with_audio.find(game_version) == language_ids_with_audio.end()) {
+    return true;
+  }
+  if (std::find(language_ids_with_audio.at(game_version).begin(),
+                language_ids_with_audio.at(game_version).end(),
+                language_id) == language_ids_with_audio.at(game_version).end()) {
+    return true;
+  }
+  return false;
+}
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "common/versions/versions.h"
+
+struct GameSubtitleDefinitionFile {
+  int language_id = -1;
+  std::string text_version = "jak1-v2";
+  std::string lines_path = "";
+  std::optional<std::string> lines_base_path = std::nullopt;
+  std::string meta_path = "";
+  std::optional<std::string> meta_base_path = std::nullopt;
+};
+
+void open_subtitle_project(const std::string& project_kind,
+                           const std::string& file_path,
+                           std::vector<GameSubtitleDefinitionFile>& inputs);
+std::string lookup_locale_code(const GameVersion game_version, const int language_id);
+// Languages that have audio tracks are not translated via Crowdin, therefore they must have
+// a copy of the scenes from their base english counterpart no matter what (so it can be
+// translated!)
+//
+// In contrast, languages like english-UK vs english-GB are almost entirely identical and cannot be
+// translated via Crowdin (it has an audio track) so it makes sense to minimize the noise in the
+// file by eliding the duplicated scenes that have no changes.
+bool dump_language_with_duplicates_from_base(const GameVersion game_version, const int language_id);
@@ -1,133 +0,0 @@
-#include "subtitles_deser.h"
-
-#include <algorithm>
-#include <regex>
-#include <vector>
-
-#include "common/util/FileUtil.h"
-
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/ranges.h"
-#include "third-party/json.hpp"
-
-bool write_subtitle_db_to_files(const GameSubtitleDB& db) {
-  // Write the subtitles out
-  std::vector<int> completed_banks = {};
-  for (const auto& [id, bank] : db.m_banks) {
-    // If we've done the bank before, skip it
-    auto it = find(completed_banks.begin(), completed_banks.end(), bank->m_lang_id);
-    if (it != completed_banks.end()) {
-      continue;
-    }
-    // Check to see if this bank is shared by any other, if so do it at the same time
-    // and skip it
-    // This is basically just to deal with US/UK english in a not so hacky way
-    std::vector<int> banks = {};
-    for (const auto& [_id, _bank] : db.m_banks) {
-      if (_bank->file_path == bank->file_path) {
-        banks.push_back(_bank->m_lang_id);
-        completed_banks.push_back(_bank->m_lang_id);
-      }
-    }
-
-    std::string file_contents = "";
-    file_contents += fmt::format("(language-id {})\n", fmt::join(banks, " "));
-    auto file_ver = parse_text_only_version(bank->file_path);
-    auto font = get_font_bank(file_ver);
-    file_contents += fmt::format("(text-version {})\n", get_text_version_name(file_ver));
-
-    for (const auto& group_name : db.m_subtitle_groups->m_group_order) {
-      bool last_was_single = false;
-      file_contents +=
-          fmt::format("\n;; -----------------\n;; {}\n;; -----------------\n", group_name);
-      std::vector<GameSubtitleSceneInfo> all_scenes;
-      for (const auto& [scene_name, scene] : bank->scenes()) {
-        all_scenes.push_back(scene);
-      }
-      std::sort(all_scenes.begin(), all_scenes.end(),
-                [](const GameSubtitleSceneInfo& a, const GameSubtitleSceneInfo& b) {
-                  if (a.kind() != b.kind()) {
-                    return a.kind() < b.kind();
-                  }
-                  if (a.kind() == SubtitleSceneKind::Movie) {
-                    return a.name() < b.name();
-                  } else if (a.kind() == SubtitleSceneKind::HintNamed) {
-                    if (a.id() == b.id()) {
-                      return a.name() < b.name();
-                    } else {
-                      return a.id() < b.id();
-                    }
-                  } else if (a.kind() == SubtitleSceneKind::Hint) {
-                    return a.id() < b.id();
-                  }
-                  return false;
-                });
-      for (const auto& scene : all_scenes) {
-        if (scene.m_sorting_group != group_name) {
-          continue;
-        }
-
-        if (last_was_single && scene.lines().size() == 1) {
-          file_contents += fmt::format("(\"{}\"", scene.name());
-        } else {
-          file_contents += fmt::format("\n(\"{}\"", scene.name());
-        }
-        if (scene.kind() == SubtitleSceneKind::Hint) {
-          file_contents += " :hint #x0";
-        } else if (scene.kind() == SubtitleSceneKind::HintNamed) {
-          file_contents += fmt::format(" :hint #x{0:x}", scene.id());
-        }
-        // more compact formatting for single-line entries
-        if (scene.lines().size() == 1) {
-          const auto& line = scene.lines().at(0);
-          if (line.line.empty()) {
-            file_contents += fmt::format(" ({})", line.frame);
-          } else {
-            file_contents += fmt::format(" ({}", line.frame);
-            if (line.offscreen && scene.kind() == SubtitleSceneKind::Movie) {
-              file_contents += " :offscreen";
-            }
-            file_contents +=
-                fmt::format(" \"{}\"", font->convert_game_to_utf8(line.speaker.c_str()));
-            file_contents += fmt::format(" \"{}\")", font->convert_game_to_utf8(line.line.c_str()));
-          }
-          file_contents += ")\n";
-          last_was_single = true;
-        } else {
-          file_contents += "\n";
-          for (auto& line : scene.lines()) {
-            // Clear screen entries
-            if (line.line.empty()) {
-              file_contents += fmt::format("  ({})\n", line.frame);
-            } else {
-              file_contents += fmt::format("  ({}", line.frame);
-              if (line.offscreen && scene.kind() == SubtitleSceneKind::Movie) {
-                file_contents += " :offscreen";
-              }
-              file_contents +=
-                  fmt::format(" \"{}\"", font->convert_game_to_utf8(line.speaker.c_str()));
-              file_contents +=
-                  fmt::format(" \"{}\")\n", font->convert_game_to_utf8(line.line.c_str()));
-            }
-          }
-          file_contents += "  )\n";
-          last_was_single = false;
-        }
-      }
-    }
-
-    // Commit it to the file
-    std::string full_path = (file_util::get_jak_project_dir() / fs::path(bank->file_path)).string();
-    file_util::write_text_file(full_path, file_contents);
-  }
-
-  // Write the subtitle group info out
-  nlohmann::json json(db.m_subtitle_groups->m_groups);
-  json[db.m_subtitle_groups->group_order_key] = nlohmann::json(db.m_subtitle_groups->m_group_order);
-  std::string file_path = (file_util::get_jak_project_dir() / "game" / "assets" / "jak1" /
-                           "subtitle" / "subtitle-groups.json")
-                              .string();
-  file_util::write_text_file(file_path, json.dump(2));
-
-  return true;
-}
@@ -1,5 +0,0 @@
-#pragma once
-
-#include "common/serialization/subtitles/subtitles_ser.h"
-
-bool write_subtitle_db_to_files(const GameSubtitleDB& db);
@@ -1,492 +0,0 @@
-#include "subtitles_ser.h"
-
-#include "common/goos/ParseHelpers.h"
-#include "common/goos/Reader.h"
-#include "common/util/FileUtil.h"
-#include "common/util/json_util.h"
-
-#include "third-party/fmt/core.h"
-
-int64_t get_int(const goos::Object& obj) {
-  if (obj.is_int()) {
-    return obj.integer_obj.value;
-  }
-  throw std::runtime_error(obj.print() + " was supposed to be an integer, but isn't");
-}
-
-const goos::Object& car(const goos::Object& x) {
-  if (!x.is_pair()) {
-    throw std::runtime_error("car: invalid pair");
-  }
-
-  return x.as_pair()->car;
-}
-
-const goos::Object& cdr(const goos::Object& x) {
-  if (!x.is_pair()) {
-    throw std::runtime_error("cdr: invalid pair");
-  }
-
-  return x.as_pair()->cdr;
-}
-
-std::string get_string(const goos::Object& x) {
-  if (x.is_string()) {
-    return x.as_string()->data;
-  }
-  throw std::runtime_error(x.print() + " was supposed to be a string, but isn't");
-}
-
-/*!
- * Parse a game text file.
- * Information is added to the game text database.
- *
- * The file should begin with (language-id x y z...) with the given language IDs.
- * Each entry should be (id "line for 1st language" "line for 2nd language" ...)
- * This adds the text line to each of the specified languages.
- */
-void parse_text(const goos::Object& data, GameTextDB& db) {
-  const GameTextFontBank* font = nullptr;
-  std::vector<std::shared_ptr<GameTextBank>> banks;
-  std::string possible_group_name;
-
-  for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
-    if (obj.is_pair()) {
-      auto& head = car(obj);
-      if (head.is_symbol("language-id")) {
-        if (banks.size() != 0) {
-          throw std::runtime_error("Languages have been set multiple times.");
-        }
-
-        if (cdr(obj).is_empty_list()) {
-          throw std::runtime_error("At least one language must be set.");
-        }
-
-        if (possible_group_name.empty()) {
-          throw std::runtime_error("Text group must be set before languages.");
-        }
-
-        for_each_in_list(cdr(obj), [&](const goos::Object& obj) {
-          auto lang = get_int(obj);
-          if (!db.bank_exists(possible_group_name, lang)) {
-            // database has no lang in this group yet
-            banks.push_back(db.add_bank(possible_group_name, std::make_shared<GameTextBank>(lang)));
-          } else {
-            banks.push_back(db.bank_by_id(possible_group_name, lang));
-          }
-        });
-      } else if (head.is_symbol("group-name")) {
-        if (!possible_group_name.empty()) {
-          throw std::runtime_error("group-name has been set multiple times.");
-        }
-
-        possible_group_name = get_string(car(cdr(obj)));
-
-        if (possible_group_name.empty()) {
-          throw std::runtime_error("invalid group-name.");
-        }
-
-        if (!cdr(cdr(obj)).is_empty_list()) {
-          throw std::runtime_error("group-name has too many arguments");
-        }
-      } else if (head.is_symbol("credits")) {
-        // parse a "credits" object. it's a list of lines where the ID automatically increments, and
-        // empty lines are skipped
-        if (banks.size() == 0) {
-          throw std::runtime_error("At least one language must be set before defining entries.");
-        }
-
-        if (cdr(obj).is_empty_list() || cdr(cdr(obj)).is_empty_list() ||
-            !car(cdr(obj)).is_symbol(":begin") || !car(cdr(cdr(obj))).is_int()) {
-          throw std::runtime_error("Invalid credits begin param");
-        }
-
-        const auto& it = cdr(cdr(obj));
-        int begin_id = car(it).as_int();
-        int id = begin_id - 1;
-        for_each_in_list(cdr(it), [&](const goos::Object& entry) {
-          ++id;
-          if (entry.is_string()) {
-            if (entry.as_string()->data.empty()) {
-              // empty string! just advance
-              return;
-            }
-
-            auto line = font->convert_utf8_to_game(entry.as_string()->data);
-            // add to all langs
-            for (auto& bank : banks) {
-              bank->set_line(id, line);
-            }
-          } else if (entry.is_pair()) {
-            int b_i = 0;
-            for_each_in_list(entry, [&](const goos::Object& entry) {
-              if (entry.is_string()) {
-                if (b_i >= int(banks.size())) {
-                  throw std::runtime_error(fmt::format("Too many strings in text id #x{:x}", id));
-                }
-
-                auto line = font->convert_utf8_to_game(entry.as_string()->data);
-                banks[b_i++]->set_line(id, line);
-              } else {
-                throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
-              }
-            });
-            if (b_i != int(banks.size())) {
-              throw std::runtime_error(
-                  fmt::format("Not enough strings specified in text id #x{:x}", id));
-            }
-          } else {
-            throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
-          }
-        });
-      } else if (head.is_symbol("text-version")) {
-        if (font) {
-          throw std::runtime_error("text version is already set");
-        }
-
-        const auto& ver_name = car(cdr(obj));
-        if (!ver_name.is_symbol()) {
-          throw std::runtime_error("invalid text version entry");
-        }
-
-        font = get_font_bank(ver_name.as_symbol()->name);
-      }
-
-      else if (head.is_int()) {
-        if (!font) {
-          throw std::runtime_error("Text version must be set before defining entries.");
-        }
-        if (banks.size() == 0) {
-          throw std::runtime_error("At least one language must be set before defining entries.");
-        }
-        int i = 0;
-        int id = head.as_int();
-        for_each_in_list(cdr(obj), [&](const goos::Object& entry) {
-          if (entry.is_string()) {
-            if (i >= int(banks.size())) {
-              throw std::runtime_error(fmt::format("Too many strings in text id #x{:x}", id));
-            }
-
-            auto line = font->convert_utf8_to_game(entry.as_string()->data);
-            banks[i++]->set_line(id, line);
-          } else {
-            throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
-          }
-        });
-        if (i != int(banks.size())) {
-          throw std::runtime_error(
-              fmt::format("Not enough strings specified in text id #x{:x}", id));
-        }
-      } else {
-        throw std::runtime_error("Invalid game text file entry: " + head.print());
-      }
-    } else {
-      throw std::runtime_error("Invalid game text file");
-    }
-  });
-  if (banks.size() == 0) {
-    throw std::runtime_error("At least one language must be set.");
-  }
-}
-
-/*!
- * Parse a game subtitle file.
- * Information is added to the game subtitles database.
- *
- * The file should begin with (language-id x y z...) for the given language IDs.
- * Each scene should be (scene-name <entry 1> <entry 2> ... )
- * This adds the subtitle to each of the specified languages.
- */
-void parse_subtitle(const goos::Object& data, GameSubtitleDB& db, const std::string& file_path) {
-  const GameTextFontBank* font = nullptr;
-  std::map<int, std::shared_ptr<GameSubtitleBank>> banks;
-
-  for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
-    if (obj.is_pair()) {
-      auto& head = car(obj);
-      if (head.is_symbol() && head.as_symbol()->name == "language-id") {
-        if (banks.size() != 0) {
-          throw std::runtime_error("Languages have been set multiple times.");
-        }
-
-        if (cdr(obj).is_empty_list()) {
-          throw std::runtime_error("At least one language must be set.");
-        }
-
-        for_each_in_list(cdr(obj), [&](const goos::Object& obj) {
-          auto lang = get_int(obj);
-          if (!db.bank_exists(lang)) {
-            // database has no lang yet
-            banks[lang] = db.add_bank(std::make_shared<GameSubtitleBank>(lang));
-            banks[lang]->file_path = file_path;
-          } else {
-            banks[lang] = db.bank_by_id(lang);
-            banks[lang]->file_path = file_path;
-          }
-        });
-      } else if (head.is_symbol("text-version")) {
-        if (font) {
-          throw std::runtime_error("text version is already set");
-        }
-
-        const auto& ver_name = car(cdr(obj));
-        if (!ver_name.is_symbol()) {
-          throw std::runtime_error("invalid text version entry");
-        }
-
-        font = get_font_bank(ver_name.as_symbol()->name);
-      }
-
-      else if (head.is_string() || head.is_int()) {
-        if (!font) {
-          throw std::runtime_error("Text version must be set before defining entries.");
-        }
-        if (banks.size() == 0) {
-          throw std::runtime_error("At least one language must be set before defining scenes.");
-        }
-        auto kind = SubtitleSceneKind::Movie;
-        int id = 0;
-        auto entries = cdr(obj);
-        if (head.is_int()) {
-          kind = SubtitleSceneKind::Hint;
-        } else if (entries.is_pair() && car(entries).is_symbol()) {
-          const auto& parm = car(entries).as_symbol()->name;
-          if (parm == ":hint") {
-            entries = cdr(entries);
-            id = car(entries).as_int();
-            kind = SubtitleSceneKind::HintNamed;
-          } else {
-            throw std::runtime_error("Unknown parameter for subtitle scene");
-          }
-          entries = cdr(entries);
-        }
-
-        GameSubtitleSceneInfo scene(kind);
-        if (kind == SubtitleSceneKind::Movie || kind == SubtitleSceneKind::HintNamed) {
-          scene.set_name(head.as_string()->data);
-        } else if (kind == SubtitleSceneKind::Hint) {
-          id = head.as_int();
-        }
-        scene.set_id(id);
-        scene.m_sorting_group = db.m_subtitle_groups->find_group(scene.name());
-        scene.m_sorting_group_idx = db.m_subtitle_groups->find_group_index(scene.m_sorting_group);
-
-        for_each_in_list(entries, [&](const goos::Object& entry) {
-          if (entry.is_pair()) {
-            // expected formats:
-            // (time <args>)
-            // all arguments have default values. the arguments are:
-            // "speaker" "line" - two strings. one for the speaker's name and one for the actual
-            //                    line. speaker can be empty. default is just empty string.
-            // :offscreen - speaker is offscreen. default is not offscreen.
-
-            if (!car(entry).is_int()) {
-              throw std::runtime_error("Each entry must start with a timestamp (number)");
-            }
-
-            auto time = car(entry).as_int();
-            goos::StringObject *speaker = nullptr, *line = nullptr;
-            bool offscreen = false;
-            if (scene.kind() == SubtitleSceneKind::Hint ||
-                scene.kind() == SubtitleSceneKind::HintNamed) {
-              offscreen = true;
-            }
-            for_each_in_list(cdr(entry), [&](const goos::Object& arg) {
-              if (arg.is_string()) {
-                if (!speaker) {
-                  speaker = arg.as_string();
-                } else if (!line) {
-                  line = arg.as_string();
-                } else {
-                  throw std::runtime_error("Invalid string in subtitle entry");
-                }
-              } else if (speaker && !line) {
-                throw std::runtime_error(
-                    "Invalid object in subtitle entry, expecting actual line string after speaker");
-              } else if (arg.is_symbol()) {
-                if (scene.kind() == SubtitleSceneKind::Movie &&
-                    arg.as_symbol()->name == ":offscreen") {
-                  offscreen = true;
-                } else {
-                  throw std::runtime_error(
-                      fmt::format("Unknown parameter {} in subtitle", arg.as_symbol()->name));
-                }
-              }
-            });
-            auto line_str = font->convert_utf8_to_game(line ? line->data : "");
-            auto speaker_str = font->convert_utf8_to_game(speaker ? speaker->data : "");
-            scene.add_line(time, line_str, speaker_str, offscreen);
-          } else {
-            throw std::runtime_error(
-                fmt::format("{} | Each entry must be a non-empty list", scene.name()));
-          }
-        });
-        for (auto& [lang, bank] : banks) {
-          if (!bank->scene_exists(scene.name())) {
-            bank->add_scene(scene);
-          } else {
-            auto& old_scene = bank->scene_by_name(scene.name());
-            old_scene.from_other_scene(scene);
-          }
-        }
-      } else {
-        throw std::runtime_error("Invalid game subtitles file entry: " + head.print());
-      }
-    } else {
-      throw std::runtime_error("Invalid game subtitles file");
-    }
-  });
-  if (banks.size() == 0) {
-    throw std::runtime_error("At least one language must be set.");
-  }
-}
-
-GameTextVersion parse_text_only_version(const std::string& filename) {
-  goos::Reader reader;
-  return parse_text_only_version(reader.read_from_file({filename}));
-}
-
-GameTextVersion parse_text_only_version(const goos::Object& data) {
-  const GameTextFontBank* font = nullptr;
-
-  for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
-    if (obj.is_pair()) {
-      auto& head = car(obj);
-      if (head.is_symbol("text-version")) {
-        if (font) {
-          throw std::runtime_error("text version is already set");
-        }
-
-        const auto& ver_name = car(cdr(obj));
-        if (!ver_name.is_symbol()) {
-          throw std::runtime_error("invalid text version entry");
-        }
-
-        font = get_font_bank(ver_name.as_symbol()->name);
-      }
-    }
-  });
-
-  if (!font) {
-    throw std::runtime_error("text version not found");
-  }
-
-  return font->version();
-}
-
-void GameSubtitleGroups::hydrate_from_asset_file() {
-  std::string file_path = (file_util::get_jak_project_dir() / "game" / "assets" / "jak1" /
-                           "subtitle" / "subtitle-groups.json")
-                              .string();
-  auto config_str = file_util::read_text_file(file_path);
-  auto group_data = parse_commented_json(config_str, file_path);
-
-  for (const auto& [key, val] : group_data.items()) {
-    try {
-      if (key == group_order_key) {
-        m_group_order = val.get<std::vector<std::string>>();
-      } else {
-        m_groups[key] = val.get<std::vector<std::string>>();
-      }
-    } catch (std::exception& ex) {
-      lg::print("Bad subtitle group entry - {} - {}", key, ex.what());
-    }
-  }
-}
-
-std::string GameSubtitleGroups::find_group(const std::string& scene_name) {
-  for (auto const& [group, scenes] : m_groups) {
-    for (auto const& name : scenes) {
-      if (name == scene_name) {
-        return group;
-      }
-    }
-  }
-  // Add to the uncategorized group if it wasn't found
-  m_groups[uncategorized_group].push_back(scene_name);
-  return uncategorized_group;
-}
-
-int GameSubtitleGroups::find_group_index(const std::string& group_name) {
-  auto it = find(m_group_order.begin(), m_group_order.end(), group_name);
-  if (it != m_group_order.end()) {
-    return it - m_group_order.begin();
-  } else {
-    return m_group_order.size() - 1;
-  }
-}
-
-void GameSubtitleGroups::remove_scene(const std::string& group_name,
-                                      const std::string& scene_name) {
-  if (m_groups.count(group_name) == 0) {
-    lg::error("Subtitle group {} doesn't exist! Abort.", group_name);
-    return;
-  }
-  m_groups[group_name].erase(
-      std::remove(m_groups[group_name].begin(), m_groups[group_name].end(), scene_name),
-      m_groups[group_name].end());
-}
-
-void GameSubtitleGroups::add_scene(const std::string& group_name, const std::string& scene_name) {
-  std::string group = group_name;
-  if (m_groups.count(group_name) == 0) {
-    lg::error("Subtitle group {} doesn't exist! Add to uncategorized.", group_name);
-    group = uncategorized_group;
-  }
-  auto it = std::find(m_groups[group].begin(), m_groups[group].end(), scene_name);
-  if (it != m_groups[group].end()) {
-    lg::error("Scene {} already exists in group {}", scene_name, group);
-  } else {
-    m_groups[group].push_back(scene_name);
-  }
-}
-
-// TODO - why not just return the inputs instead of passing in an empty one?
-void open_text_project(const std::string& kind,
-                       const std::string& filename,
-                       std::vector<std::string>& inputs) {
-  goos::Reader reader;
-  auto& proj = reader.read_from_file({filename}).as_pair()->cdr.as_pair()->car;
-  if (!proj.is_pair() || !proj.as_pair()->car.is_symbol() ||
-      proj.as_pair()->car.as_symbol()->name != kind) {
-    throw std::runtime_error(fmt::format("invalid {} project", kind));
-  }
-
-  goos::for_each_in_list(proj.as_pair()->cdr, [&](const goos::Object& o) {
-    if (o.is_pair() && o.as_pair()->cdr.is_pair()) {
-      auto& action = o.as_pair()->car.as_symbol()->name;
-
-      if (action == "file") {
-        auto& in = o.as_pair()->cdr.as_pair()->car.as_string()->data;
-        inputs.push_back(in);
-      } else {
-        throw std::runtime_error(fmt::format("unknown action {} in {} project", action, kind));
-      }
-    } else {
-      throw std::runtime_error(fmt::format("invalid entry in {} project", kind));
-    }
-  });
-}
-
-GameSubtitleDB load_subtitle_project() {
-  // Load the subtitle files
-  GameSubtitleDB db;
-  db.m_subtitle_groups = std::make_unique<GameSubtitleGroups>();
-  db.m_subtitle_groups->hydrate_from_asset_file();
-  try {
-    goos::Reader reader;
-    std::vector<std::string> inputs;
-    std::string subtitle_project =
-        (file_util::get_jak_project_dir() / "game" / "assets" / "game_subtitle.gp").string();
-    open_text_project("subtitle", subtitle_project, inputs);
-    for (auto& filename : inputs) {
-      auto code = reader.read_from_file({filename});
-      parse_subtitle(code, db, filename);
-    }
-  } catch (std::runtime_error& e) {
-    lg::error("error loading subtitle project: {}", e.what());
-  }
-  return db;
-}
-
-// TODO - write a deserializer, the compiler still can do the compiling!
@@ -1,194 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <map>
-#include <memory>
-#include <string>
-#include <unordered_set>
-#include <utility>
-
-#include "common/goos/Object.h"
-#include "common/log/log.h"
-#include "common/util/Assert.h"
-#include "common/util/FontUtils.h"
-
-/*!
- * The text bank contains all lines (accessed with an ID) for a language.
- */
-class GameTextBank {
- public:
-  GameTextBank(int lang_id) : m_lang_id(lang_id) {}
-
-  int lang() const { return m_lang_id; }
-  const std::map<int, std::string>& lines() const { return m_lines; }
-
-  bool line_exists(int id) const { return m_lines.find(id) != m_lines.end(); }
-  std::string line(int id) { return m_lines.at(id); }
-  void set_line(int id, std::string line) { m_lines[id] = line; }
-
- private:
-  int m_lang_id;
-  std::map<int, std::string> m_lines;
-};
-
-/*!
- * The text database contains a text bank for each language for each text group.
- * Each text bank contains a list of text lines. Very simple.
- */
-class GameTextDB {
- public:
-  const std::unordered_map<std::string, std::map<int, std::shared_ptr<GameTextBank>>>& groups()
-      const {
-    return m_banks;
-  }
-  const std::map<int, std::shared_ptr<GameTextBank>>& banks(std::string group) const {
-    return m_banks.at(group);
-  }
-
-  bool bank_exists(std::string group, int id) const {
-    if (m_banks.find(group) == m_banks.end())
-      return false;
-    return m_banks.at(group).find(id) != m_banks.at(group).end();
-  }
-
-  std::shared_ptr<GameTextBank> add_bank(std::string group, std::shared_ptr<GameTextBank> bank) {
-    ASSERT(!bank_exists(group, bank->lang()));
-    m_banks[group][bank->lang()] = bank;
-    return bank;
-  }
-  std::shared_ptr<GameTextBank> bank_by_id(std::string group, int id) {
-    if (!bank_exists(group, id)) {
-      return nullptr;
-    }
-    return m_banks.at(group).at(id);
-  }
-
- private:
-  std::unordered_map<std::string, std::map<int, std::shared_ptr<GameTextBank>>> m_banks;
-};
-
-/*!
- * The subtitle scene info (accessed through the scene name) contains all lines and their timestamps
- * and other settings.
- */
-enum class SubtitleSceneKind { Invalid = -1, Movie = 0, Hint = 1, HintNamed = 2 };
-class GameSubtitleSceneInfo {
- public:
-  struct SubtitleLine {
-    SubtitleLine(int frame, std::string line, std::string speaker, bool offscreen)
-        : frame(frame), line(line), speaker(speaker), offscreen(offscreen) {}
-
-    int frame;
-    std::string line;
-    std::string speaker;
-    bool offscreen;
-
-    bool operator<(const SubtitleLine& other) const { return (frame < other.frame); }
-  };
-
-  GameSubtitleSceneInfo(SubtitleSceneKind kind) : m_kind(kind) {}
-
-  const std::string& name() const { return m_name; }
-  const std::vector<SubtitleLine>& lines() const { return m_lines; }
-  int id() const { return m_id; }
-  SubtitleSceneKind kind() const { return m_kind; }
-
-  void clear_lines() { m_lines.clear(); }
-  void set_name(const std::string& new_name) { m_name = new_name; }
-  void set_id(int new_id) { m_id = new_id; }
-  void from_other_scene(GameSubtitleSceneInfo& scene) {
-    m_name = scene.name();
-    m_lines = scene.lines();
-    m_kind = scene.kind();
-    m_id = scene.id();
-  }
-
-  void add_line(int frame, std::string line, std::string speaker, bool offscreen) {
-    m_lines.emplace_back(SubtitleLine(frame, line, speaker, offscreen));
-    std::sort(m_lines.begin(), m_lines.end());
-  }
-
-  std::string m_name;
-  int m_id;
-  std::vector<SubtitleLine> m_lines;
-  SubtitleSceneKind m_kind;
-  std::string m_sorting_group;
-  int m_sorting_group_idx;
-};
-
-/*!
- * The subtitle bank contains subtitles for all scenes in a language.
- */
-class GameSubtitleBank {
- public:
-  GameSubtitleBank(int lang_id) : m_lang_id(lang_id) {}
-
-  int lang() const { return m_lang_id; }
-  const std::map<std::string, GameSubtitleSceneInfo>& scenes() const { return m_scenes; }
-
-  bool scene_exists(const std::string& name) const { return m_scenes.find(name) != m_scenes.end(); }
-  GameSubtitleSceneInfo& scene_by_name(const std::string& name) { return m_scenes.at(name); }
-  void add_scene(GameSubtitleSceneInfo& scene) {
-    ASSERT(!scene_exists(scene.name()));
-    m_scenes.insert({scene.name(), scene});
-  }
-
-  int m_lang_id;
-  std::string file_path;
-
-  std::map<std::string, GameSubtitleSceneInfo> m_scenes;
-};
-
-class GameSubtitleGroups {
- public:
-  std::vector<std::string> m_group_order;
-  std::map<std::string, std::vector<std::string>> m_groups;
-
-  void hydrate_from_asset_file();
-  std::string find_group(const std::string& scene_name);
-  int find_group_index(const std::string& group_name);
-  void remove_scene(const std::string& group_name, const std::string& scene_name);
-  void add_scene(const std::string& group_name, const std::string& scene_name);
-
-  std::string group_order_key = "_groups";
-  std::string uncategorized_group = "uncategorized";
-};
-
-/*!
- * The subtitles database contains a subtitles bank for each language.
- * Each subtitles bank contains a series of subtitle scene infos.
- */
-class GameSubtitleDB {
- public:
-  const std::map<int, std::shared_ptr<GameSubtitleBank>>& banks() const { return m_banks; }
-
-  bool bank_exists(int id) const { return m_banks.find(id) != m_banks.end(); }
-
-  std::shared_ptr<GameSubtitleBank> add_bank(std::shared_ptr<GameSubtitleBank> bank) {
-    ASSERT(!bank_exists(bank->lang()));
-    m_banks[bank->lang()] = bank;
-    return bank;
-  }
-  std::shared_ptr<GameSubtitleBank> bank_by_id(int id) {
-    if (!bank_exists(id)) {
-      return nullptr;
-    }
-    return m_banks.at(id);
-  }
-
-  std::map<int, std::shared_ptr<GameSubtitleBank>> m_banks;
-  std::unique_ptr<GameSubtitleGroups> m_subtitle_groups;
-};
-
-// TODO add docstrings
-
-void parse_text(const goos::Object& data, GameTextDB& db);
-void parse_subtitle(const goos::Object& data, GameSubtitleDB& db, const std::string& file_path);
-
-GameTextVersion parse_text_only_version(const std::string& filename);
-GameTextVersion parse_text_only_version(const goos::Object& data);
-
-void open_text_project(const std::string& kind,
-                       const std::string& filename,
-                       std::vector<std::string>& inputs);
-GameSubtitleDB load_subtitle_project();
@@ -0,0 +1,302 @@
+#include "subtitles_v1.h"
+
+#include "common/log/log.h"
+#include "common/util/FileUtil.h"
+
+#include "subtitles_v2.h"
+
+#include "third-party/fmt/core.h"
+
+void to_json(json& j, const SubtitleCutsceneLineMetadataV1& obj) {
+  json_serialize(frame_start);
+  json_serialize(offscreen);
+  json_serialize(speaker);
+  json_serialize(clear);
+}
+void from_json(const json& j, SubtitleCutsceneLineMetadataV1& obj) {
+  json_deserialize_if_exists(frame_start);
+  json_deserialize_if_exists(offscreen);
+  json_deserialize_if_exists(speaker);
+  json_deserialize_if_exists(clear);
+}
+
+void to_json(json& j, const SubtitleHintLineMetadataV1& obj) {
+  json_serialize(frame_start);
+  json_serialize(speaker);
+  json_serialize(clear);
+}
+void from_json(const json& j, SubtitleHintLineMetadataV1& obj) {
+  json_deserialize_if_exists(frame_start);
+  json_deserialize_if_exists(speaker);
+  json_deserialize_if_exists(clear);
+}
+void to_json(json& j, const SubtitleHintMetadataV1& obj) {
+  json_serialize(id);
+  json_serialize(lines);
+}
+void from_json(const json& j, SubtitleHintMetadataV1& obj) {
+  json_deserialize_if_exists(id);
+  json_deserialize_if_exists(lines);
+}
+
+void to_json(json& j, const SubtitleMetadataFileV1& obj) {
+  json_serialize(cutscenes);
+  json_serialize(hints);
+}
+
+void from_json(const json& j, SubtitleMetadataFileV1& obj) {
+  json_deserialize_if_exists(cutscenes);
+  json_deserialize_if_exists(hints);
+}
+void to_json(json& j, const SubtitleFileV1& obj) {
+  json_serialize(speakers);
+  json_serialize(cutscenes);
+  json_serialize(hints);
+}
+void from_json(const json& j, SubtitleFileV1& obj) {
+  json_deserialize_if_exists(speakers);
+  json_deserialize_if_exists(cutscenes);
+  json_deserialize_if_exists(hints);
+}
+
+std::pair<SubtitleMetadataFile, SubtitleFile> convert_v1_to_v2(
+    const GameSubtitleDefinitionFile& file_info,
+    const SubtitleMetadataFileV1& v1_meta_file,
+    const SubtitleFileV1& v1_lines_file) {
+  // Convert the old format into the new
+  SubtitleMetadataFile meta_file;
+  SubtitleFile lines_file;
+  lines_file.speakers = v1_lines_file.speakers;
+  for (const auto& [cutscene_name, cutscene_lines] : v1_meta_file.cutscenes) {
+    std::vector<std::string> scene_lines;
+    SubtitleSceneMetadata new_scene_meta;
+    // Iterate the lines, grab the actual text from the lines file if it's not a clear screen entry
+    int line_idx = 0;
+    int lines_added = 0;
+    for (const auto& line_meta : cutscene_lines) {
+      SubtitleLineMetadata new_meta;
+      new_meta.frame_start = line_meta.frame_start;
+      new_meta.offscreen = line_meta.offscreen;
+      new_meta.speaker = line_meta.speaker;
+      new_meta.frame_end = 0;  // v1 doesn't use frame_end
+      new_meta.merge = false;  // or merge
+      new_scene_meta.lines.push_back(new_meta);
+      if (line_meta.clear) {
+        scene_lines.push_back("");
+        lines_added++;
+      } else {
+        if (v1_lines_file.speakers.find(line_meta.speaker) == v1_lines_file.speakers.end() ||
+            v1_lines_file.cutscenes.find(cutscene_name) == v1_lines_file.cutscenes.end() ||
+            int(v1_lines_file.cutscenes.at(cutscene_name).size()) < line_idx) {
+          lg::warn(
+              "{} Couldn't find {} in line file, or line list is too small, or speaker could not "
+              "be resolved {}!",
+              file_info.language_id, cutscene_name, line_meta.speaker);
+        } else {
+          scene_lines.push_back(v1_lines_file.cutscenes.at(cutscene_name).at(line_idx));
+          lines_added++;
+        }
+        line_idx++;
+      }
+    }
+    // Verify we added the amount of lines we expected to
+    if (lines_added != int(cutscene_lines.size())) {
+      throw std::runtime_error(
+          fmt::format("Cutscene: '{}' has a mismatch in metadata lines vs text lines. Expected {} "
+                      "only added {} lines",
+                      cutscene_name, cutscene_lines.size(), lines_added));
+    }
+    meta_file.cutscenes.emplace(cutscene_name, new_scene_meta);
+    lines_file.cutscenes.emplace(cutscene_name, scene_lines);
+  }
+  // Now hints
+  for (const auto& [hint_name, hint_info] : v1_meta_file.hints) {
+    std::vector<std::string> scene_lines;
+    SubtitleSceneMetadata new_scene_meta;
+    if (hint_info.id != "0") {
+      new_scene_meta.m_hint_id = std::stoi(hint_info.id, nullptr, 16);
+    }
+    // Iterate the lines, grab the actual text from the lines file if it's not a clear screen entry
+    int line_idx = 0;
+    int lines_added = 0;
+    for (const auto& line_meta : hint_info.lines) {
+      SubtitleLineMetadata new_meta;
+      new_meta.frame_start = line_meta.frame_start;
+      new_meta.frame_end = 0;  // unused by v1
+      new_meta.offscreen = true;
+      new_meta.speaker = line_meta.speaker;
+      new_scene_meta.lines.push_back(new_meta);
+      if (line_meta.clear) {
+        scene_lines.push_back("");
+        lines_added++;
+      } else {
+        if (v1_lines_file.speakers.find(line_meta.speaker) == v1_lines_file.speakers.end() ||
+            v1_lines_file.hints.find(hint_name) == v1_lines_file.hints.end() ||
+            line_idx >= (int)v1_lines_file.hints.at(hint_name).size()) {
+          lg::warn(
+              "{} Couldn't find {} in line file, or line list is too small, or speaker could not "
+              "be resolved {}!",
+              file_info.language_id, hint_name, line_meta.speaker);
+        } else {
+          scene_lines.push_back(v1_lines_file.hints.at(hint_name).at(line_idx));
+          lines_added++;
+        }
+        line_idx++;
+      }
+    }
+    // Verify we added the amount of lines we expected to
+    if (lines_added != (int)hint_info.lines.size()) {
+      throw std::runtime_error(
+          fmt::format("Hint: '{}' has a mismatch in metadata lines vs text lines. Expected {} "
+                      "only added {} lines",
+                      hint_name, hint_info.lines.size(), lines_added));
+    }
+    meta_file.other.emplace(hint_name, new_scene_meta);
+    lines_file.other.emplace(hint_name, scene_lines);
+  }
+  return {meta_file, lines_file};
+}
+
+GameSubtitlePackage read_json_files_v1(const GameSubtitleDefinitionFile& file_info) {
+  // Parse the files
+  SubtitleMetadataFileV1 v1_meta_base_file;
+  SubtitleMetadataFileV1 v1_meta_combined_file;
+  SubtitleFileV1 v1_lines_base_file;
+  SubtitleFileV1 v1_lines_lang_file;
+  SubtitleFileV1 v1_lines_combined_file;
+  try {
+    // If we have a base file defined, load that and merge it
+    if (file_info.meta_base_path) {
+      auto base_data =
+          parse_commented_json(file_util::read_text_file(file_util::get_jak_project_dir() /
+                                                         file_info.meta_base_path.value()),
+                               "subtitle_meta_base_path");
+      v1_meta_base_file = base_data;
+      auto data = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.meta_path),
+          "subtitle_meta_path");
+      base_data.at("cutscenes").update(data.at("cutscenes"));
+      base_data.at("hints").update(data.at("hints"));
+      v1_meta_combined_file = base_data;
+    } else {
+      v1_meta_combined_file = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.meta_path),
+          "subtitle_meta_path");
+    }
+    if (file_info.lines_base_path) {
+      auto base_data =
+          parse_commented_json(file_util::read_text_file(file_util::get_jak_project_dir() /
+                                                         file_info.lines_base_path.value()),
+                               "subtitle_line_base_path");
+      v1_lines_base_file = base_data;
+      auto data = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.lines_path),
+          "subtitle_line_path");
+      v1_lines_lang_file = data;
+      base_data.at("cutscenes").update(data.at("cutscenes"));
+      base_data.at("hints").update(data.at("hints"));
+      base_data.at("speakers").update(data.at("speakers"));
+      v1_lines_combined_file = base_data;
+    } else {
+      v1_lines_combined_file = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.lines_path),
+          "subtitle_line_path");
+      v1_lines_lang_file = v1_lines_combined_file;
+    }
+    GameSubtitlePackage package;
+    std::tie(package.base_meta, package.base_lines) =
+        convert_v1_to_v2(file_info, v1_meta_base_file, v1_lines_base_file);
+    std::tie(package.combined_meta, package.combined_lines) =
+        convert_v1_to_v2(file_info, v1_meta_combined_file, v1_lines_combined_file);
+    for (const auto& [scene_name, scene_info] : v1_lines_lang_file.cutscenes) {
+      package.scenes_defined_in_lang.insert(scene_name);
+    }
+    for (const auto& [scene_name, scene_info] : v1_lines_lang_file.hints) {
+      package.scenes_defined_in_lang.insert(scene_name);
+    }
+    return package;
+  } catch (std::exception& e) {
+    lg::error("Unable to parse subtitle json entry, couldn't successfully load files - {}",
+              e.what());
+    throw;
+  }
+}
+
+SubtitleMetadataFileV1 dump_bank_meta_v1(const GameVersion /*game_version*/,
+                                         std::shared_ptr<GameSubtitleBank> bank) {
+  auto meta_file = SubtitleMetadataFileV1();
+  for (const auto& [scene_name, scene_info] : bank->m_scenes) {
+    // Avoid dumping duplicates
+    if (bank->m_base_scenes.find(scene_name) != bank->m_base_scenes.end() &&
+        scene_info.same_metadata_as_other(bank->m_base_scenes.at(scene_name))) {
+      continue;
+    }
+    if (scene_info.is_cutscene) {
+      std::vector<SubtitleCutsceneLineMetadataV1> lines;
+      for (const auto& line : scene_info.m_lines) {
+        auto line_meta = SubtitleCutsceneLineMetadataV1();
+        line_meta.frame_start = line.metadata.frame_start;
+        if (line.text.empty()) {
+          line_meta.clear = true;
+        } else {
+          line_meta.offscreen = line.metadata.offscreen;
+          line_meta.speaker = line.metadata.speaker;
+        }
+        lines.push_back(line_meta);
+      }
+      meta_file.cutscenes[scene_name] = lines;
+    } else {
+      SubtitleHintMetadataV1 hint;
+      hint.id = fmt::format("{:x}", scene_info.m_hint_id);
+      std::vector<SubtitleHintLineMetadataV1> lines;
+      for (const auto& line : scene_info.m_lines) {
+        auto line_meta = SubtitleHintLineMetadataV1();
+        line_meta.frame_start = line.metadata.frame_start;
+        if (line.text.empty()) {
+          line_meta.clear = true;
+        } else {
+          line_meta.speaker = line.metadata.speaker;
+        }
+        lines.push_back(line_meta);
+      }
+      hint.lines = lines;
+      meta_file.hints[scene_name] = hint;
+    }
+  }
+  return meta_file;
+}
+
+SubtitleFileV1 dump_bank_lines_v1(const GameVersion game_version,
+                                  std::shared_ptr<GameSubtitleBank> bank) {
+  const auto dump_with_duplicates =
+      dump_language_with_duplicates_from_base(game_version, bank->m_lang_id);
+  SubtitleFileV1 file;
+  file.speakers = bank->m_speakers;
+  for (const auto& [scene_name, scene_info] : bank->m_scenes) {
+    // Avoid dumping duplicates if needed
+    if (!dump_with_duplicates &&
+        bank->m_base_scenes.find(scene_name) != bank->m_base_scenes.end() &&
+        scene_info.same_lines_as_other(bank->m_base_scenes.at(scene_name))) {
+      continue;
+    }
+    if (scene_info.is_cutscene) {
+      file.cutscenes[scene_name] = {};
+      for (const auto& scene_line : scene_info.m_lines) {
+        if (scene_line.text.empty()) {
+          continue;
+        }
+        file.cutscenes[scene_name].push_back(scene_line.text);
+      }
+    } else {
+      file.hints[scene_name] = {};
+      for (const auto& scene_line : scene_info.m_lines) {
+        if (scene_line.text.empty()) {
+          continue;
+        }
+        file.hints[scene_name].push_back(scene_line.text);
+      }
+    }
+  }
+
+  return file;
+}
@@ -0,0 +1,54 @@
+#pragma once
+
+#include "common/serialization/subtitles/subtitles.h"
+#include "common/serialization/subtitles/subtitles_v2.h"
+#include "common/util/json_util.h"
+
+struct SubtitleCutsceneLineMetadataV1 {
+  int frame_start;
+  bool offscreen;
+  std::string speaker;
+  bool clear;
+};
+void to_json(json& j, const SubtitleCutsceneLineMetadataV1& obj);
+void from_json(const json& j, SubtitleCutsceneLineMetadataV1& obj);
+
+struct SubtitleHintLineMetadataV1 {
+  int frame_start;
+  std::string speaker;
+  bool clear;
+};
+void to_json(json& j, const SubtitleHintLineMetadataV1& obj);
+void from_json(const json& j, SubtitleHintLineMetadataV1& obj);
+
+struct SubtitleHintMetadataV1 {
+  std::string id;  // hex
+  std::vector<SubtitleHintLineMetadataV1> lines;
+};
+void to_json(json& j, const SubtitleHintMetadataV1& obj);
+void from_json(const json& j, SubtitleHintMetadataV1& obj);
+
+struct SubtitleMetadataFileV1 {
+  std::unordered_map<std::string, std::vector<SubtitleCutsceneLineMetadataV1>> cutscenes;
+  std::unordered_map<std::string, SubtitleHintMetadataV1> hints;
+};
+void to_json(json& j, const SubtitleMetadataFileV1& obj);
+void from_json(const json& j, SubtitleMetadataFileV1& obj);
+
+struct SubtitleFileV1 {
+  std::unordered_map<std::string, std::string> speakers;
+  std::unordered_map<std::string, std::vector<std::string>> cutscenes;
+  std::unordered_map<std::string, std::vector<std::string>> hints;
+};
+void to_json(json& j, const SubtitleFileV1& obj);
+void from_json(const json& j, SubtitleFileV1& obj);
+
+// These functions essentially convert to and from the V1/V2 formats to either load from disk, or
+// persist to disk
+//
+// Returns the full file pair, then the base pair
+GameSubtitlePackage read_json_files_v1(const GameSubtitleDefinitionFile& file_info);
+SubtitleMetadataFileV1 dump_bank_meta_v1(const GameVersion game_version,
+                                         std::shared_ptr<GameSubtitleBank> bank);
+SubtitleFileV1 dump_bank_lines_v1(const GameVersion game_version,
+                                  std::shared_ptr<GameSubtitleBank> bank);
@@ -0,0 +1,406 @@
+#include "subtitles_v2.h"
+
+#include "common/log/log.h"
+#include "common/util/FileUtil.h"
+
+#include "subtitles_v1.h"
+
+#include "third-party/fmt/core.h"
+
+void to_json(json& j, const SubtitleLineMetadata& obj) {
+  json_serialize(frame_start);
+  json_serialize(frame_end);
+  json_serialize(offscreen);
+  json_serialize(speaker);
+  json_serialize(merge);
+}
+
+void from_json(const json& j, SubtitleLineMetadata& obj) {
+  json_deserialize_if_exists(frame_start);
+  json_deserialize_if_exists(frame_end);
+  json_deserialize_if_exists(offscreen);
+  json_deserialize_if_exists(speaker);
+  json_deserialize_if_exists(merge);
+}
+
+void to_json(json& j, const SubtitleSceneMetadata& obj) {
+  json_serialize(lines);
+}
+
+void from_json(const json& j, SubtitleSceneMetadata& obj) {
+  json_deserialize_if_exists(lines);
+}
+
+void to_json(json& j, const SubtitleMetadataFile& obj) {
+  json_serialize(cutscenes);
+  json_serialize(other);
+}
+
+void from_json(const json& j, SubtitleMetadataFile& obj) {
+  json_deserialize_if_exists(cutscenes);
+  json_deserialize_if_exists(other);
+}
+
+void to_json(json& j, const SubtitleFile& obj) {
+  json_serialize(speakers);
+  json_serialize(cutscenes);
+  json_serialize(other);
+}
+
+void from_json(const json& j, SubtitleFile& obj) {
+  json_deserialize_if_exists(speakers);
+  json_deserialize_if_exists(cutscenes);
+  json_deserialize_if_exists(other);
+}
+
+// matches enum in `subtitle2.gc` with "none" (first) and "max" (last and removed)
+const std::unordered_map<std::string, u16> jak2_speaker_name_to_enum_val = {
+    {"none", 0},
+    {"computer", 1},
+    {"jak", 2},
+    {"darkjak", 3},
+    {"daxter", 4},
+    {"samos", 5},
+    {"keira", 6},
+    {"keira-before-class-3", 7},
+    {"kid", 8},
+    {"kor", 9},
+    {"metalkor", 10},
+    {"baron", 11},
+    {"errol", 12},
+    {"torn", 13},
+    {"tess", 14},
+    {"guard", 15},
+    {"guard-a", 16},
+    {"guard-b", 17},
+    {"krew", 18},
+    {"sig", 19},
+    {"brutter", 20},
+    {"vin", 21},
+    {"youngsamos", 22},
+    {"youngsamos-before-rescue", 23},
+    {"pecker", 24},
+    {"onin", 25},
+    {"ashelin", 26},
+    {"jinx", 27},
+    {"mog", 28},
+    {"grim", 29},
+    {"agent", 30},
+    {"citizen-male", 31},
+    {"citizen-female", 32},
+    {"oracle", 33},
+    {"precursor", 34}};
+
+GameSubtitlePackage read_json_files_v2(const GameSubtitleDefinitionFile& file_info) {
+  GameSubtitlePackage package;
+  SubtitleFile lang_lines;
+  try {
+    // If we have a base file defined, load that and merge it
+    if (file_info.meta_base_path) {
+      auto base_data =
+          parse_commented_json(file_util::read_text_file(file_util::get_jak_project_dir() /
+                                                         file_info.meta_base_path.value()),
+                               "subtitle_meta_base_path");
+      package.base_meta = base_data;
+      auto data = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.meta_path),
+          "subtitle_meta_path");
+      base_data.at("cutscenes").update(data.at("cutscenes"));
+      base_data.at("other").update(data.at("other"));
+      package.combined_meta = base_data;
+    } else {
+      package.combined_meta = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.meta_path),
+          "subtitle_meta_path");
+    }
+    if (file_info.lines_base_path) {
+      auto base_data =
+          parse_commented_json(file_util::read_text_file(file_util::get_jak_project_dir() /
+                                                         file_info.lines_base_path.value()),
+                               "subtitle_line_base_path");
+      package.base_lines = base_data;
+      auto data = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.lines_path),
+          "subtitle_line_path");
+      lang_lines = data;
+      base_data.at("cutscenes").update(data.at("cutscenes"));
+      base_data.at("other").update(data.at("other"));
+      base_data.at("speakers").update(data.at("speakers"));
+      package.combined_lines = base_data;
+    } else {
+      package.combined_lines = parse_commented_json(
+          file_util::read_text_file(file_util::get_jak_project_dir() / file_info.lines_path),
+          "subtitle_line_path");
+      lang_lines = package.combined_lines;
+    }
+    for (const auto& [scene_name, scene_info] : lang_lines.cutscenes) {
+      package.scenes_defined_in_lang.insert(scene_name);
+    }
+    for (const auto& [scene_name, scene_info] : lang_lines.other) {
+      package.scenes_defined_in_lang.insert(scene_name);
+    }
+  } catch (std::exception& e) {
+    lg::error("Unable to parse subtitle json entry, couldn't successfully load files - {}",
+              e.what());
+    throw;
+  }
+  return package;
+}
+
+void GameSubtitleDB::init_banks_from_file(const GameSubtitleDefinitionFile& file_info) {
+  // Init Settings
+  std::shared_ptr<GameSubtitleBank> bank;
+  if (!bank_exists(file_info.language_id)) {
+    // database has no lang yet
+    bank = add_bank(std::make_shared<GameSubtitleBank>(file_info.language_id));
+  } else {
+    bank = bank_by_id(file_info.language_id);
+  }
+  bank->m_text_version = get_text_version_from_name(file_info.text_version);
+  bank->m_file_path = file_info.lines_path;
+  bank->m_file_base_path = file_info.lines_base_path;
+  try {
+    if (m_subtitle_version == SubtitleFormat::V1) {
+      const auto package = read_json_files_v1(file_info);
+      bank->m_speakers = package.combined_lines.speakers;
+      bank->add_scenes_from_files(package);
+    } else {
+      const auto package = read_json_files_v2(file_info);
+      bank->m_speakers = package.combined_lines.speakers;
+      bank->m_speakers.emplace("none", "none");
+      bank->add_scenes_from_files(package);
+    }
+  } catch (std::exception& e) {
+    throw;
+  }
+}
+
+GameSubtitleSceneInfo GameSubtitleBank::new_scene_from_meta(
+    const std::string& scene_name,
+    const SubtitleSceneMetadata& scene_meta,
+    const std::unordered_map<std::string, std::vector<std::string>>& relevant_lines) {
+  GameSubtitleSceneInfo new_scene;
+  new_scene.m_name = scene_name;
+  new_scene.m_hint_id = scene_meta.m_hint_id;
+  new_scene.only_defined_in_base = false;
+  new_scene.is_cutscene = false;
+  int line_idx = 0;
+  int lines_added = 0;
+  for (const auto& line_meta : scene_meta.lines) {
+    // In V1, there was a concept of a "clear" line, you don't have to specify these in the "lines"
+    // file as they are just blank lines.
+    //
+    // In V2, there are no longer "clear" lines, but there are lines that are "merged" which
+    // essentially inherit the text from the base game (since Jak 2+ actually has subtitles!)
+    //
+    // In either case, we acknowledge that there is a line, but there is no text to retrieve at that
+    // index.
+    if (line_meta.merge || (relevant_lines.find(scene_name) != relevant_lines.end() &&
+                            (int)relevant_lines.at(scene_name).size() > line_idx &&
+                            relevant_lines.at(scene_name).at(line_idx).empty())) {
+      new_scene.m_lines.push_back({"", line_meta});
+      lines_added++;
+    } else if (m_speakers.find(line_meta.speaker) == m_speakers.end() ||
+               relevant_lines.find(scene_name) == relevant_lines.end() ||
+               line_idx >= (int)relevant_lines.at(scene_name).size()) {
+      lg::warn(
+          "{} Couldn't find {} in line file, or line list is too small, or speaker could not "
+          "be resolved {}!",
+          m_lang_id, scene_name, line_meta.speaker);
+    } else {
+      new_scene.m_lines.push_back({relevant_lines.at(scene_name).at(line_idx), line_meta});
+      lines_added++;
+    }
+    line_idx++;
+  }
+  // Verify we added the amount of lines we expected to
+  if (lines_added != int(scene_meta.lines.size())) {
+    throw std::runtime_error(
+        fmt::format("Cutscene: '{}' has a mismatch in metadata lines vs text lines. Expected {} "
+                    "only added {} lines",
+                    scene_name, scene_meta.lines.size(), lines_added));
+  }
+  return new_scene;
+}
+
+void GameSubtitleBank::add_scenes_from_files(const GameSubtitlePackage& package) {
+  // Save the base and lang specific file info separately for later context
+  for (const auto& [scene_name, scene_meta] : package.base_meta.cutscenes) {
+    auto new_scene = new_scene_from_meta(scene_name, scene_meta, package.base_lines.cutscenes);
+    new_scene.is_cutscene = true;
+    m_base_scenes.emplace(scene_name, new_scene);
+  }
+  for (const auto& [scene_name, scene_meta] : package.base_meta.other) {
+    auto new_scene = new_scene_from_meta(scene_name, scene_meta, package.base_lines.other);
+    m_base_scenes.emplace(scene_name, new_scene);
+  }
+  // Iterate through the metadata file as blank lines are now omitted from the lines file now
+  for (const auto& [scene_name, scene_meta] : package.combined_meta.cutscenes) {
+    auto new_scene = new_scene_from_meta(scene_name, scene_meta, package.combined_lines.cutscenes);
+    new_scene.is_cutscene = true;
+    // Check if the only place lines were defined was in the base file
+    if (package.scenes_defined_in_lang.find(scene_name) == package.scenes_defined_in_lang.end()) {
+      new_scene.only_defined_in_base = true;
+    }
+    m_scenes.emplace(scene_name, new_scene);
+  }
+  for (const auto& [scene_name, scene_meta] : package.combined_meta.other) {
+    auto new_scene = new_scene_from_meta(scene_name, scene_meta, package.combined_lines.other);
+    // Check if the only place lines were defined was in the base file
+    if (package.scenes_defined_in_lang.find(scene_name) == package.scenes_defined_in_lang.end()) {
+      new_scene.only_defined_in_base = true;
+    }
+    m_scenes.emplace(scene_name, new_scene);
+  }
+}
+
+// TODO - for jak 3+, this needs some game version context info (could infer from text version)
+std::vector<std::string> GameSubtitleBank::speaker_names_ordered_by_enum_value() {
+  // Create a temporary vector of pairs (key, value)
+  std::vector<std::pair<std::string, u16>> temp_vec(jak2_speaker_name_to_enum_val.begin(),
+                                                    jak2_speaker_name_to_enum_val.end());
+  // Sort the temporary vector based on the enum value in ascending order
+  std::sort(temp_vec.begin(), temp_vec.end(),
+            [](const auto& a, const auto& b) { return a.second < b.second; });
+  // Extract the sorted keys into a new vector
+  std::vector<std::string> sorted_names;
+  sorted_names.reserve(temp_vec.size());
+  for (const auto& pair : temp_vec) {
+    if (pair.second == 0) {
+      // we write #f for invalid entries, including the "none" at the start
+      sorted_names.push_back("#f");
+    } else {
+      sorted_names.push_back(m_speakers.at(pair.first));
+    }
+  }
+  return sorted_names;
+}
+
+u16 GameSubtitleBank::speaker_enum_value_from_name(const std::string& speaker_id) {
+  if (jak2_speaker_name_to_enum_val.find(speaker_id) == jak2_speaker_name_to_enum_val.end()) {
+    throw std::runtime_error(
+        fmt::format("'{}' speaker could not be found in the enum value mapping, update it or fix "
+                    "the invalid speaker!",
+                    speaker_id));
+  }
+  return u16(jak2_speaker_name_to_enum_val.at(speaker_id));
+}
+
+SubtitleMetadataFile dump_bank_meta_v2(const GameVersion game_version,
+                                       std::shared_ptr<GameSubtitleBank> bank) {
+  const auto dump_with_duplicates =
+      dump_language_with_duplicates_from_base(game_version, bank->m_lang_id);
+  (void)dump_with_duplicates;
+  auto meta_file = SubtitleMetadataFile();
+  for (const auto& [scene_name, scene_info] : bank->m_scenes) {
+    // Avoid dumping duplicates
+    if (bank->m_base_scenes.find(scene_name) != bank->m_base_scenes.end() &&
+        scene_info.same_metadata_as_other(bank->m_base_scenes.at(scene_name))) {
+      continue;
+    }
+    SubtitleSceneMetadata scene_meta;
+    for (const auto& line : scene_info.m_lines) {
+      scene_meta.lines.push_back(line.metadata);
+    }
+    if (scene_info.is_cutscene) {
+      meta_file.cutscenes[scene_name] = scene_meta;
+    } else {
+      meta_file.other[scene_name] = scene_meta;
+    }
+  }
+  return meta_file;
+}
+
+SubtitleFile dump_bank_lines_v2(const GameVersion game_version,
+                                std::shared_ptr<GameSubtitleBank> bank) {
+  const auto dump_with_duplicates =
+      dump_language_with_duplicates_from_base(game_version, bank->m_lang_id);
+  SubtitleFile file;
+  file.speakers = bank->m_speakers;
+  if (file.speakers.find("none") != file.speakers.end()) {
+    file.speakers.erase("none");
+  }
+  for (const auto& [scene_name, scene_info] : bank->m_scenes) {
+    // Avoid dumping duplicates if needed
+    if (!dump_with_duplicates &&
+        bank->m_base_scenes.find(scene_name) != bank->m_base_scenes.end() &&
+        scene_info.same_lines_as_other(bank->m_base_scenes.at(scene_name))) {
+      continue;
+    }
+    for (const auto& scene_line : scene_info.m_lines) {
+      // Skip merged lines
+      if (scene_line.metadata.merge) {
+        continue;
+      }
+      if (scene_info.is_cutscene) {
+        file.cutscenes[scene_name].push_back(scene_line.text);
+      } else {
+        file.other[scene_name].push_back(scene_line.text);
+      }
+    }
+  }
+  return file;
+}
+
+bool GameSubtitleDB::write_subtitle_db_to_files(const GameVersion game_version) {
+  try {
+    for (const auto& [language_id, bank] : m_banks) {
+      json meta_file;
+      if (m_subtitle_version == SubtitleFormat::V1) {
+        meta_file = dump_bank_meta_v1(game_version, bank);
+
+      } else {
+        meta_file = dump_bank_meta_v2(game_version, bank);
+      }
+      std::string dump_path =
+          (file_util::get_jak_project_dir() / "game" / "assets" /
+           version_to_game_name(game_version) / "subtitle" /
+           fmt::format("subtitle_meta_{}.json", lookup_locale_code(game_version, language_id)))
+              .string();
+      file_util::write_text_file(dump_path, meta_file.dump(2));
+      // Now dump the actual subtitle lines
+      json lines_file;
+      if (m_subtitle_version == SubtitleFormat::V1) {
+        lines_file = dump_bank_lines_v1(game_version, bank);
+      } else {
+        lines_file = dump_bank_lines_v2(game_version, bank);
+      }
+      dump_path =
+          (file_util::get_jak_project_dir() / "game" / "assets" /
+           version_to_game_name(game_version) / "subtitle" /
+           fmt::format("subtitle_lines_{}.json", lookup_locale_code(game_version, language_id)))
+              .string();
+      file_util::write_text_file(dump_path, lines_file.dump(2));
+    }
+  } catch (std::exception& ex) {
+    lg::error(ex.what());
+    return false;
+  }
+  return true;
+}
+
+GameSubtitleDB load_subtitle_project(const GameSubtitleDB::SubtitleFormat format_version,
+                                     const GameVersion game_version) {
+  // Load the subtitle files
+  GameSubtitleDB db;
+  db.m_subtitle_version = format_version;
+  try {
+    std::vector<GameSubtitleDefinitionFile> files;
+    std::string subtitle_project = (file_util::get_jak_project_dir() / "game" / "assets" /
+                                    version_to_game_name(game_version) / "game_subtitle.gp")
+                                       .string();
+    if (format_version == GameSubtitleDB::SubtitleFormat::V1) {
+      open_subtitle_project("subtitle", subtitle_project, files);
+    } else {
+      open_subtitle_project("subtitle-v2", subtitle_project, files);
+    }
+    for (auto& file : files) {
+      db.init_banks_from_file(file);
+    }
+  } catch (std::runtime_error& e) {
+    // TODO - these run in gk, all exceptions must go...not reliable
+    lg::error("error loading subtitle project: {}", e.what());
+    db.m_load_error = e.what();
+  }
+  return db;
+}
@@ -0,0 +1,171 @@
+#pragma once
+
+#include "common/serialization/subtitles/subtitles.h"
+#include "common/util/Assert.h"
+#include "common/util/FontUtils.h"
+#include "common/util/json_util.h"
+
+struct SubtitleLineMetadata {
+  int frame_start;
+  int frame_end;
+  bool offscreen;
+  std::string speaker;
+  bool merge;
+
+  bool operator==(const SubtitleLineMetadata& other) const {
+    if (frame_start != other.frame_start || frame_end != other.frame_end ||
+        offscreen != other.offscreen || speaker != other.speaker || merge != other.merge) {
+      return false;
+    }
+    return true;
+  }
+  bool operator!=(const SubtitleLineMetadata& other) const { return !(*this == other); }
+};
+void to_json(json& j, const SubtitleLineMetadata& obj);
+void from_json(const json& j, SubtitleLineMetadata& obj);
+
+struct SubtitleSceneMetadata {
+  std::vector<SubtitleLineMetadata> lines;
+  int m_hint_id = 0;  // used only for jak1, intentionally ignored in serialization/deserialization
+};
+void to_json(json& j, const SubtitleSceneMetadata& obj);
+void from_json(const json& j, SubtitleSceneMetadata& obj);
+
+struct SubtitleMetadataFile {
+  std::unordered_map<std::string, SubtitleSceneMetadata> cutscenes;
+  std::unordered_map<std::string, SubtitleSceneMetadata> other;
+};
+void to_json(json& j, const SubtitleMetadataFile& obj);
+void from_json(const json& j, SubtitleMetadataFile& obj);
+
+struct SubtitleFile {
+  std::unordered_map<std::string, std::string> speakers;
+  std::unordered_map<std::string, std::vector<std::string>> cutscenes;
+  std::unordered_map<std::string, std::vector<std::string>> other;
+};
+void to_json(json& j, const SubtitleFile& obj);
+void from_json(const json& j, SubtitleFile& obj);
+
+struct SubtitleLine {
+  std::string text;
+  SubtitleLineMetadata metadata;
+
+  bool operator<(const SubtitleLine& other) const {
+    return (metadata.frame_start < other.metadata.frame_start);
+  }
+};
+
+// Returns the individual components of each subtitle "package":
+// - the base file
+// - the combination of the two
+// This allows for more context when dumping the files back out and you want
+// to skip duplicate info
+struct GameSubtitlePackage {
+  SubtitleMetadataFile base_meta;
+  SubtitleMetadataFile combined_meta;
+  SubtitleFile base_lines;
+  SubtitleFile combined_lines;
+  std::unordered_set<std::string> scenes_defined_in_lang;
+};
+
+struct GameSubtitleSceneInfo {
+  std::string m_name;
+  std::vector<SubtitleLine> m_lines;
+  bool is_cutscene;
+  bool only_defined_in_base = false;
+  int m_hint_id;  // used only for jak1
+
+  void add_line(const std::string& text,
+                const int frame_start,
+                const int frame_end,
+                const bool offscreen,
+                const std::string& speaker,
+                const bool merge) {
+    m_lines.push_back({text, {frame_start, frame_end, offscreen, speaker, merge}});
+    std::sort(m_lines.begin(), m_lines.end());
+  }
+  bool same_lines_as_other(const GameSubtitleSceneInfo& other) const {
+    if (m_lines.size() != other.m_lines.size()) {
+      return false;
+    }
+    // Check each line
+    for (size_t i = 0; i < m_lines.size(); i++) {
+      if (m_lines.at(i).text != other.m_lines.at(i).text) {
+        return false;
+      }
+    }
+    return true;
+  }
+  bool same_metadata_as_other(const GameSubtitleSceneInfo& other) const {
+    if (m_name != other.m_name || m_lines.size() != other.m_lines.size() ||
+        is_cutscene != other.is_cutscene || m_hint_id != other.m_hint_id) {
+      return false;
+    }
+    // Check each line's metadata
+    for (size_t i = 0; i < m_lines.size(); i++) {
+      if (m_lines.at(i).metadata != other.m_lines.at(i).metadata) {
+        return false;
+      }
+    }
+    return true;
+  }
+};
+
+/*!
+ * The subtitle bank contains subtitles for all scenes in a language.
+ */
+class GameSubtitleBank {
+ public:
+  GameSubtitleBank(const int lang_id) : m_lang_id(lang_id) {}
+
+  int m_lang_id;
+  GameTextVersion m_text_version;
+  std::string m_file_path;
+  std::optional<std::string> m_file_base_path;
+
+  std::map<std::string, GameSubtitleSceneInfo> m_base_scenes;
+  std::map<std::string, GameSubtitleSceneInfo> m_lang_scenes;
+  std::map<std::string, GameSubtitleSceneInfo> m_scenes;
+  bool scene_exists(const std::string& name) const { return m_scenes.find(name) != m_scenes.end(); }
+  GameSubtitleSceneInfo new_scene_from_meta(
+      const std::string& scene_name,
+      const SubtitleSceneMetadata& scene_meta,
+      const std::unordered_map<std::string, std::vector<std::string>>& relevant_lines);
+  GameSubtitleSceneInfo& scene_by_name(const std::string& name) { return m_scenes.at(name); }
+  void add_scenes_from_files(const GameSubtitlePackage& package);
+
+  std::unordered_map<std::string, std::string> m_speakers;
+  std::vector<std::string> speaker_names_ordered_by_enum_value();
+  u16 speaker_enum_value_from_name(const std::string& speaker_id);
+};
+
+/*!
+ * The subtitles database contains a subtitles bank for each language.
+ * Each subtitles bank contains a series of subtitle scene infos.
+ */
+class GameSubtitleDB {
+ public:
+  enum class SubtitleFormat { V1, V2 };
+  std::optional<std::string> m_load_error = {};
+  std::map<int, std::shared_ptr<GameSubtitleBank>> m_banks;
+  SubtitleFormat m_subtitle_version = SubtitleFormat::V2;
+
+  bool bank_exists(int id) const { return m_banks.find(id) != m_banks.end(); }
+  std::shared_ptr<GameSubtitleBank> add_bank(std::shared_ptr<GameSubtitleBank> bank) {
+    ASSERT(!bank_exists(bank->m_lang_id));
+    m_banks[bank->m_lang_id] = bank;
+    return bank;
+  }
+  std::shared_ptr<GameSubtitleBank> bank_by_id(int id) {
+    if (!bank_exists(id)) {
+      return nullptr;
+    }
+    return m_banks.at(id);
+  }
+
+  void init_banks_from_file(const GameSubtitleDefinitionFile& file_info);
+  bool write_subtitle_db_to_files(const GameVersion game_version);
+};
+
+GameSubtitleDB load_subtitle_project(const GameSubtitleDB::SubtitleFormat format_version,
+                                     GameVersion game_version);