i18n: add finnish to menus

In the original game, they had no choice but to use the memory card file
as their method of persisting settings. We are not limited by such
things.

It's inconvenient to have to load your save-file when launching the game
to initialize these settings to your liking, it's also confusing
behaviour to even some players that have played the game heavily for
over a decade. We can do better by globally saving these settings to the
`pc-settings` file instead.

Originally I only migrated the volume settings, then i figured it would
be nice to also have play-hints and subtitles settings persisted. More
could debatably be moved (language is a big one...) but these were the
low hanging fruit.

I also reduced the default volumes as that is something else that has
come up a few times.

.github/scripts/releases/extract_build_unix.sh

@@ -32,4 +32,4 @@ cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
 cp -r $SOURCE/goal_src $DEST/data
 cp -r $SOURCE/game/assets $DEST/data/game/
 cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
-cp -r $SOURCE/custom_levels $DEST/data
+cp -r $SOURCE/custom_assets $DEST/data

.github/scripts/releases/extract_build_windows.sh

@@ -24,4 +24,4 @@ cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
 cp -r $SOURCE/goal_src $DEST/data
 cp -r $SOURCE/game/assets $DEST/data/game/
 cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
-cp -r $SOURCE/custom_levels $DEST/data
+cp -r $SOURCE/custom_assets $DEST/data

.github/workflows/linux-build-clang.yaml

@@ -33,7 +33,7 @@ jobs:
             libxi-dev zip ninja-build libgl1-mesa-dev libssl-dev
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.github/workflows/linux-build-gcc.yaml

@@ -34,7 +34,7 @@ jobs:
             sudo update-alternatives --set g++ /usr/bin/g++-10
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.github/workflows/macos-build-arm.yaml

@@ -27,7 +27,7 @@ jobs:
         run: arch -arm64 brew install cmake ninja
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.github/workflows/macos-build.yaml

@@ -28,7 +28,7 @@ jobs:
         run: brew install cmake nasm ninja
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.github/workflows/windows-build-clang.yaml

@@ -36,7 +36,7 @@ jobs:
           }
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.github/workflows/windows-build-msvc.yaml

@@ -31,7 +31,7 @@ jobs:
           }
 
       - name: Setup sccache
-        uses: hendrikmuhs/[email protected]2
+        uses: hendrikmuhs/[email protected]3
         with:
           variant: sccache
           key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}-${{ github.sha }}

.gitignore

@@ -54,7 +54,14 @@ imgui.ini
 node_modules/
 
 # texture replacements
-texture_replacements/*
+custom_assets/jak1/texture_replacements/*
+custom_assets/jak2/texture_replacements/*
+custom_assets/jak3/texture_replacements/*
+
+# merc replacements
+custom_assets/jak1/merc_replacements/*
+custom_assets/jak2/merc_replacements/*
+custom_assets/jak3/merc_replacements/*
 
 # generated cmake files
 svnrev.h
@@ -76,3 +83,4 @@ __pycache__/
 /TODO.md
 unifont-15.0.03.ttf
 *.diff
+goalc-report.html

.vs/launch.vs.json

@@ -7,42 +7,58 @@
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Unit-Tests - Summary",
-      "args": ["--gtest_brief=1"]
+      "args": [
+        "--gtest_brief=1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Unit-Tests - Verbose",
-      "args": ["--gtest_brief=0"]
+      "args": [
+        "--gtest_brief=0"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Draft Tests - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*Draft*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*Draft*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - TypeConsistency - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*TypeConsistency*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*TypeConsistency*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - TypeConsistency - Jak 2 - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*Jak2TypeConsistency*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*Jak2TypeConsistency*\""
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - WithGameTests - Verbose",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*WithGameTests*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*WithGameTests*\""
+      ]
     },
     {
       "type": "default",
@@ -113,56 +129,109 @@
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 1 - Runtime",
-      "args": ["-v", "--game", "jak1", "--", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak1",
+        "--",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 1 - Runtime (boot)",
-      "args": ["-v", "--game", "jak1", "--", "-boot", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak1",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (no boot)",
-      "args": ["-v", "--game", "jak2", "--", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (boot)",
-      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 2 - Runtime (release)",
-      "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso"]
+      "args": [
+        "-v",
+        "--game",
+        "jak2",
+        "--",
+        "-boot",
+        "-fakeiso"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "gk.exe (bin\\gk.exe)",
       "name": "Game - Jak 3 - Runtime (boot)",
-      "args": ["-v", "--game", "jak3", "--", "-boot", "-fakeiso", "-debug"]
+      "args": [
+        "-v",
+        "--game",
+        "jak3",
+        "--",
+        "-boot",
+        "-fakeiso",
+        "-debug"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc.exe (bin\\goalc.exe)",
       "name": "REPL - Jak 1",
-      "args": ["--user-auto", "--game", "jak1"]
+      "args": [
+        "--user-auto",
+        "--game",
+        "jak1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc.exe (bin\\goalc.exe)",
       "name": "REPL - Jak 2",
-      "args": ["--user-auto", "--game", "jak2"]
+      "args": [
+        "--user-auto",
+        "--game",
+        "jak2"
+      ]
     },
     {
       "type": "default",
@@ -262,14 +331,26 @@
       "project": "CMakeLists.txt",
       "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
       "name": "Tools - EE Memory Analyze - Jak 1",
-      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak1"]
+      "args": [
+        "\"${workspaceRoot}/eeMemory.bin\"",
+        "--output-path",
+        "\"${workspaceRoot}\"",
+        "--game",
+        "jak1"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
       "name": "Tools - EE Memory Analyze - Jak 2",
-      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak2"]
+      "args": [
+        "\"${workspaceRoot}/eeMemory.bin\"",
+        "--output-path",
+        "\"${workspaceRoot}\"",
+        "--game",
+        "jak2"
+      ]
     },
     {
       "type": "default",
@@ -286,35 +367,53 @@
       "project": "CMakeLists.txt",
       "projectTarget": "extractor.exe (bin\\extractor.exe)",
       "name": "Tools - Extractor - Full",
-      "args": ["\"E:\\ISOs\\Jak\\Jak 1.iso\""]
+      "args": [
+        "\"E:\\ISOs\\Jak\\Jak 1.iso\""
+      ]
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "lsp.exe (bin\\lsp.exe)",
-      "name" : "Tools - LSP",
-      "args" : []
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "lsp.exe (bin\\lsp.exe)",
+      "name": "Tools - LSP",
+      "args": []
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
-      "name" : "Tools - Type Searcher",
-      "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--size", 255, "--fields", "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"]
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "type_searcher.exe (bin\\type_searcher.exe)",
+      "name": "Tools - Type Searcher",
+      "args": [
+        "--game",
+        "jak2",
+        "--output-path",
+        "./search-results.json",
+        "--size",
+        255,
+        "--fields",
+        "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"
+      ]
     },
     {
-      "type" : "default",
-      "project" : "CMakeLists.txt",
-      "projectTarget" : "formatter.exe (bin\\formatter.exe)",
-      "name" : "Tools - Formatter - Inplace",
-      "args" : ["--write", "--file", "C:\\Users\\xtvas\\Repos\\opengoal\\jak-project\\decompiler_out\\jak3\\mood-h_disasm.gc"]
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "formatter.exe (bin\\formatter.exe)",
+      "name": "Tools - Formatter - Inplace",
+      "args": [
+        "--write",
+        "--file",
+        "C:\\Users\\xtvas\\Repositories\\opengoal\\jak-project\\goal_src\\jak1\\engine\\camera\\cam-states.gc"
+      ]
     },
     {
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
       "name": "Tests - Formatter",
-      "args": ["--gtest_brief=0", "--gtest_filter=\"*FormatterTests*\""]
+      "args": [
+        "--gtest_brief=0",
+        "--gtest_filter=\"*FormatterTests*\""
+      ]
     }
   ]
-}
+}

.vscode/launch.json

@@ -8,10 +8,10 @@
       "name": "run python script",
       "type": "python",
       "request": "launch",
-      "program": "${workspaceFolder}/scripts/gsrc/copy-common-naming.py",
+      "program": "${workspaceFolder}/scripts/gsrc/compare-compilation-outputs.py",
       "console": "integratedTerminal",
       "cwd": "${workspaceFolder}",
-      "args": ["--update-names-from-refs", "--decompiler", "./out/build/Release/bin/decompiler"]
+      "args": []
     },
   ]
 }

CMakeLists.txt

@@ -101,7 +101,6 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
     message(STATUS "AppleClang detected - Setting Defaults")
     set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} \
-        -march=native \
         -Wall \
         -Winit-self \
         -ggdb \
@@ -117,6 +116,12 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
         -Wsign-promo \
         -fdiagnostics-color=always"
         )
+    
+    # pin to AVX for macOS, hopefully all macOS runners have atleast this architecture
+    # technically speaking, SSE4 is the cutoff for Apple Silicon so...only a matter of time!
+    if(NOT CMAKE_CXX_COMPILER_TARGET STREQUAL "arm64-apple-darwin")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
+    endif()
 
     # additional c++ flags for release mode for our projects
     if(CMAKE_BUILD_TYPE MATCHES "Release")

common/CMakeLists.txt

@@ -51,7 +51,7 @@ add_library(common
         repl/config.cpp
         repl/nrepl/ReplClient.cpp
         repl/nrepl/ReplServer.cpp
-        repl/util.cpp
+        repl/repl_wrapper.cpp
         serialization/subtitles/subtitles_v1.cpp
         serialization/subtitles/subtitles_v2.cpp
         serialization/subtitles/subtitles.cpp
@@ -87,9 +87,11 @@ add_library(common
         util/term_util.cpp
         util/Timer.cpp
         util/unicode_util.cpp
-        versions/versions.cpp)
+        util/gltf_util.cpp
+        versions/versions.cpp
+        )
 
-target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs)
+target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs tiny_gltf)
 
 if(WIN32)
     target_link_libraries(common wsock32 ws2_32 windowsapp)

common/cross_sockets/XSocketClient.cpp

@@ -5,6 +5,8 @@
 #include "common/cross_sockets/XSocket.h"
 #include "common/log/log.h"
 
+#include "fmt/core.h"
+
 // clang-format off
 #ifdef _WIN32
 #define NOMINMAX
@@ -13,9 +15,7 @@
 #include <WinSock2.h>
 #include <WS2tcpip.h>
 #endif
-#include "common/repl/nrepl/ReplServer.h"
 
-#include "fmt/core.h"
 // clang-format on
 
 XSocketClient::XSocketClient(int _tcp_port) {

common/cross_sockets/XSocketServer.cpp

@@ -35,7 +35,7 @@ void XSocketServer::shutdown_server() {
   close_server_socket();
 }
 
-bool XSocketServer::init_server() {
+bool XSocketServer::init_server(bool failure_may_occur) {
   listening_socket = open_socket(AF_INET, SOCK_STREAM, 0);
   if (listening_socket < 0) {
     listening_socket = -1;
@@ -76,19 +76,27 @@ bool XSocketServer::init_server() {
   addr.sin_port = htons(tcp_port);
 
   if (bind(listening_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
-    lg::error("[XSocketServer:{}] failed to bind", tcp_port);
+    if (failure_may_occur) {
+      lg::debug("[XSocketServer:{}] failed to bind", tcp_port);
+    } else {
+      lg::error("[XSocketServer:{}] failed to bind", tcp_port);
+    }
     close_server_socket();
     return false;
   }
 
   if (listen(listening_socket, 0) < 0) {
-    lg::error("[XSocketServer:{}] failed to listen", tcp_port);
+    if (failure_may_occur) {
+      lg::debug("[XSocketServer:{}] failed to listen", tcp_port);
+    } else {
+      lg::error("[XSocketServer:{}] failed to listen", tcp_port);
+    }
     close_server_socket();
     return false;
   }
 
   server_initialized = true;
-  lg::info("[XSocketServer:{}] initialized", tcp_port);
+  lg::debug("[XSocketServer:{}] initialized", tcp_port);
   post_init();
   return true;
 }

common/cross_sockets/XSocketServer.h

@@ -20,7 +20,7 @@ class XSocketServer {
   XSocketServer(const XSocketServer&) = delete;
   XSocketServer& operator=(const XSocketServer&) = delete;
 
-  bool init_server();
+  bool init_server(bool failure_may_occur = false);
   void shutdown_server();
   void close_server_socket();
 

common/custom_data/TFrag3Data.cpp

@@ -14,6 +14,11 @@
 
 namespace tfrag3 {
 
+void PackedTimeOfDay::serialize(Serializer& ser) {
+  ser.from_pod_vector(&data);
+  ser.from_ptr(&color_count);
+}
+
 void PackedTieVertices::serialize(Serializer& ser) {
   ser.from_pod_vector(&color_indices);
   ser.from_pod_vector(&matrices);
@@ -72,10 +77,9 @@ void TfragTree::serialize(Serializer& ser) {
     draw.serialize(ser);
   }
 
-  // ser.from_pod_vector(&vertices);
   ser.from_pod_vector(&packed_vertices.vertices);
   ser.from_pod_vector(&packed_vertices.cluster_origins);
-  ser.from_pod_vector(&colors);
+  colors.serialize(ser);
   bvh.serialize(ser);
   ser.from_ptr(&use_strips);
 }
@@ -399,7 +403,7 @@ void TieTree::serialize(Serializer& ser) {
   }
 
   packed_vertices.serialize(ser);
-  ser.from_pod_vector(&colors);
+  colors.serialize(ser);
   bvh.serialize(ser);
 
   ser.from_ptr(&has_per_proto_visibility_toggle);
@@ -407,7 +411,7 @@ void TieTree::serialize(Serializer& ser) {
 }
 
 void ShrubTree::serialize(Serializer& ser) {
-  ser.from_pod_vector(&time_of_day_colors);
+  time_of_day_colors.serialize(ser);
   ser.from_pod_vector(&indices);
   packed_vertices.serialize(ser);
   if (ser.is_saving()) {
@@ -423,6 +427,30 @@ void ShrubTree::serialize(Serializer& ser) {
   ser.from_string_vector(&proto_names);
 }
 
+void HfragmentBucket::serialize(Serializer& ser) {
+  ser.from_pod_vector(&corners);
+  ser.from_ptr(&montage_table);
+}
+
+void Hfragment::serialize(Serializer& ser) {
+  ser.from_pod_vector(&vertices);
+  ser.from_pod_vector(&indices);
+  ser.from_pod_vector(&corners);
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(buckets.size());
+  } else {
+    buckets.resize(ser.load<size_t>());
+  }
+  for (auto& x : buckets) {
+    x.serialize(ser);
+  }
+  time_of_day_colors.serialize(ser);
+  ser.from_ptr(&wang_tree_tex_id);
+  ser.from_ptr(&draw_mode);
+  ser.from_ptr(&occlusion_offset);
+}
+
 void BVH::serialize(Serializer& ser) {
   ser.from_ptr(&first_leaf_node);
   ser.from_ptr(&last_leaf_node);
@@ -464,6 +492,7 @@ void MercDraw::serialize(Serializer& ser) {
   ser.from_ptr(&first_index);
   ser.from_ptr(&index_count);
   ser.from_ptr(&num_triangles);
+  ser.from_ptr(&no_strip);
 }
 
 void Blerc::serialize(Serializer& ser) {
@@ -605,6 +634,8 @@ void Level::serialize(Serializer& ser) {
     tree.serialize(ser);
   }
 
+  hfrag.serialize(ser);
+
   collision.serialize(ser);
   merc_data.serialize(ser);
 
@@ -655,8 +686,7 @@ void PackedShrubVertices::memory_usage(MemoryUsageTracker* tracker) const {
 }
 
 void ShrubTree::memory_usage(MemoryUsageTracker* tracker) const {
-  tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY,
-               sizeof(TimeOfDayColor) * time_of_day_colors.size());
+  tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY, sizeof(u8) * time_of_day_colors.data.size());
   packed_vertices.memory_usage(tracker);
   tracker->add(MemoryUsageCategory::SHRUB_DRAW, sizeof(ShrubDraw) * static_draws.size());
   tracker->add(MemoryUsageCategory::SHRUB_IND, sizeof(u32) * indices.size());
@@ -684,7 +714,7 @@ void TieTree::memory_usage(MemoryUsageTracker* tracker) const {
                  draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
   }
   packed_vertices.memory_usage(tracker);
-  tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+  tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(u8) * colors.data.size());
 
   for (auto& draw : instanced_wind_draws) {
     draw.memory_usage(tracker);
@@ -708,7 +738,7 @@ void TfragTree::memory_usage(MemoryUsageTracker* tracker) const {
                  draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
   }
   packed_vertices.memory_usage(tracker);
-  tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
+  tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(u8) * colors.data.size());
   tracker->add(MemoryUsageCategory::TFRAG_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
 }
 
@@ -721,6 +751,13 @@ void IndexTexture::memory_usage(MemoryUsageTracker* tracker) const {
   tracker->add(MemoryUsageCategory::SPECIAL_TEXTURE, 256 * 4);  // clut
 }
 
+void Hfragment::memory_usage(tfrag3::MemoryUsageTracker* tracker) const {
+  tracker->add(MemoryUsageCategory::HFRAG_VERTS, vertices.size() * sizeof(HfragmentVertex));
+  tracker->add(MemoryUsageCategory::HFRAG_INDEX, indices.size() * sizeof(u32));
+  tracker->add(MemoryUsageCategory::HFRAG_TIME_OF_DAY, time_of_day_colors.data.size() * sizeof(u8));
+  tracker->add(MemoryUsageCategory::HFRAG_CORNERS, corners.size() * sizeof(HfragmentCorner));
+}
+
 void Level::memory_usage(MemoryUsageTracker* tracker) const {
   for (const auto& texture : textures) {
     texture.memory_usage(tracker);
@@ -741,6 +778,7 @@ void Level::memory_usage(MemoryUsageTracker* tracker) const {
   for (const auto& tree : shrub_trees) {
     tree.memory_usage(tracker);
   }
+  hfrag.memory_usage(tracker);
   collision.memory_usage(tracker);
   merc_data.memory_usage(tracker);
 }
@@ -784,6 +822,11 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
       {"merc-mod-draw-1", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_1]},
       {"merc-mod-draw-2", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_2]},
       {"blerc", mem_use.data[tfrag3::MemoryUsageCategory::BLERC]},
+      {"hfrag-verts", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_VERTS]},
+      {"hfrag-index", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_INDEX]},
+      {"hfrag-time-of-day", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_TIME_OF_DAY]},
+      {"hfrag-corners", mem_use.data[tfrag3::MemoryUsageCategory::HFRAG_CORNERS]}
+
   };
   for (auto& known : known_categories) {
     total_accounted += known.second;

common/custom_data/Tfrag3Data.h

@@ -18,7 +18,7 @@ namespace tfrag3 {
 // - if changing any large things (vertices, vis, bvh, colors, textures) update get_memory_usage
 // - if adding a new category to the memory usage, update extract_level to print it.
 
-constexpr int TFRAG3_VERSION = 39;
+constexpr int TFRAG3_VERSION = 41;
 
 enum MemoryUsageCategory {
   TEXTURE,
@@ -61,6 +61,11 @@ enum MemoryUsageCategory {
   MERC_MOD_TABLE,
   BLERC,
 
+  HFRAG_VERTS,
+  HFRAG_INDEX,
+  HFRAG_TIME_OF_DAY,
+  HFRAG_CORNERS,
+
   COLLISION,
 
   NUM_CATEGORIES
@@ -269,18 +274,23 @@ struct BVH {
   void serialize(Serializer& ser);
 };
 
-// A time-of-day color. Each stores 8 colors. At a given "time of day", they are interpolated
-// to find a single color which goes into a color palette.
-struct TimeOfDayColor {
-  math::Vector<u8, 4> rgba[8];
+// This is split into groups of 4 colors.
+// The data in these groups is stored first by palette, then color, then channel.
+struct PackedTimeOfDay {
+  std::vector<u8> data;
+  u32 color_count = 0;
+  void serialize(Serializer& ser);
 
-  bool operator==(const TimeOfDayColor& other) const {
-    for (size_t i = 0; i < 8; i++) {
-      if (rgba[i] != other.rgba[i]) {
-        return false;
-      }
-    }
-    return true;
+  u8 read(int color, int palette, int channel) const {
+    const int color_quad = color / 4;
+    const int color_in_quad = color % 4;
+    return data[color_quad * 4 * 4 * 8 + palette * 4 * 4 + color_in_quad * 4 + channel];
+  }
+
+  u8& read(int color, int palette, int channel) {
+    const int color_quad = color / 4;
+    const int color_in_quad = color % 4;
+    return data[color_quad * 4 * 4 * 8 + palette * 4 * 4 + color_in_quad * 4 + channel];
   }
 };
 
@@ -319,8 +329,8 @@ struct TfragTree {
   TFragmentTreeKind kind;        // our tfrag kind
   std::vector<StripDraw> draws;  // the actual topology and settings
   PackedTfragVertices packed_vertices;
-  std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
-  BVH bvh;                             // the bvh for frustum culling
+  PackedTimeOfDay colors;  // vertex colors (pre-interpolation)
+  BVH bvh;                 // the bvh for frustum culling
   bool use_strips = true;
 
   struct {
@@ -401,7 +411,7 @@ struct TieTree {
   std::array<u32, kNumTieCategories + 1> category_draw_indices;
 
   PackedTieVertices packed_vertices;
-  std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
+  PackedTimeOfDay colors;  // vertex colors (pre-interpolation)
 
   std::vector<InstancedStripDraw> instanced_wind_draws;
   std::vector<TieWindInstance> wind_instance_info;
@@ -422,7 +432,7 @@ struct TieTree {
 
 struct ShrubTree {
   // todo some visibility structure
-  std::vector<TimeOfDayColor> time_of_day_colors;  // multiplier colors
+  PackedTimeOfDay time_of_day_colors;  // multiplier colors
 
   PackedShrubVertices packed_vertices;
   std::vector<ShrubDraw> static_draws;  // the actual topology and settings
@@ -441,6 +451,43 @@ struct ShrubTree {
   void unpack();
 };
 
+struct HfragmentVertex {
+  float height = 0;
+  u32 vi = 0;
+  u16 color_index = 0;
+  u8 u = 0, v = 0;
+  u32 pad = 0;
+};
+
+struct HfragmentCorner {
+  math::Vector<float, 4> bsphere;
+  u32 vis_id = 0;
+  u32 index_start = 0;
+  u32 index_length = 0;
+  u32 num_tris = 0;
+};
+
+struct HfragmentBucket {
+  std::vector<u32> corners;
+  std::array<u16, 16> montage_table;
+  void serialize(Serializer& ser);
+};
+
+struct Hfragment {
+  std::vector<HfragmentVertex> vertices;
+  std::vector<u32> indices;
+  std::vector<HfragmentCorner> corners;
+  std::vector<HfragmentBucket> buckets;
+  PackedTimeOfDay time_of_day_colors;
+
+  std::array<s32, 4> wang_tree_tex_id;
+  DrawMode draw_mode;
+  u32 occlusion_offset;
+
+  void serialize(Serializer& ser);
+  void memory_usage(MemoryUsageTracker* tracker) const;
+};
+
 struct CollisionMesh {
   struct Vertex {
     float x, y, z;
@@ -483,6 +530,8 @@ struct MercDraw {
   u32 first_index;
   u32 index_count;
   u32 num_triangles;
+  // no strip hack for custom models
+  bool no_strip = false;
   void serialize(Serializer& ser);
 };
 
@@ -566,6 +615,7 @@ struct Level {
   std::array<std::vector<TfragTree>, TFRAG_GEOS> tfrag_trees;
   std::array<std::vector<TieTree>, TIE_GEOS> tie_trees;
   std::vector<ShrubTree> shrub_trees;
+  Hfragment hfrag;
   CollisionMesh collision;
   MercModelGroup merc_data;
   u16 version2 = TFRAG3_VERSION;

common/dma/dma.cpp

@@ -4,7 +4,7 @@
 
 #include "fmt/core.h"
 
-std::string DmaTag::print() {
+std::string DmaTag::print() const {
   std::string result;
   const char* mode_names[8] = {"refe", "cnt", "next", "ref", "refs", "call", "ret", "end"};
   result += fmt::format("TAG: 0x{:08x} {:4s} qwc 0x{:04x}", addr, mode_names[(int)kind], qwc);
@@ -15,7 +15,7 @@ std::string DmaTag::print() {
   return result;
 }
 
-std::string VifCode::print() {
+std::string VifCode::print() const {
   std::string result;
 
   switch (kind) {

common/dma/dma.h

@@ -52,7 +52,7 @@ struct DmaTag {
 
   bool operator!=(const DmaTag& other) const { return !((*this) == other); }
 
-  std::string print();
+  std::string print() const;
 };
 
 inline void emulate_dma(const void* source_base, void* dest_base, u32 tadr, u32 dadr) {
@@ -148,7 +148,7 @@ struct VifCode {
   u16 num;
   u16 immediate;
 
-  std::string print();
+  std::string print() const;
 };
 
 struct VifCodeStcycl {

common/formatter/README.md

@@ -12,7 +12,6 @@ Also in general I found it _really_ hard to find modern and easy to understand a
 Atleast for me, it helps to understand the path code takes as it flows through the formatting process.  Originally I tried to do as much at once for sake of efficiency but this just makes things incredibly hard to reason about and fix bugs.  So like any problem that is complex, break it down, the formatter goes through many distinct phases chipping away at the problem:
 
 ```mermaid
-%%{init: {'theme': 'dark', "flowchart" : { "curve" : "basis" } } }%%
 flowchart TB
   subgraph top1 [Build a Formatting Tree]
     direction TB

common/formatter/formatter.cpp

@@ -33,14 +33,16 @@ int hang_indentation_width(const FormatterTreeNode& curr_node) {
   return 1 + hang_indentation_width(first_elt);
 }
 
-// TODO - this doesn't account for paren's width contribution!
 int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
   if (curr_node.token) {
     return curr_node.token->length();
   }
   int width = 1;
-  for (const auto& ref : curr_node.refs) {
-    width += get_total_form_inlined_width(ref);
+  for (int i = 0; i < curr_node.refs.size(); i++) {
+    width += get_total_form_inlined_width(curr_node.refs.at(i));
+    if (i != curr_node.refs.size() - 1) {
+      width += 1;  // add the space between elements
+    }
   }
   return width + 1;
 }
@@ -49,7 +51,9 @@ int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
 void apply_formatting_config(
     FormatterTreeNode& curr_node,
     std::optional<std::shared_ptr<formatter_rules::config::FormFormattingConfig>>
-        config_from_parent = {}) {
+        config_from_parent = {},
+    std::optional<std::shared_ptr<formatter_rules::config::FormFormattingConfig>>
+        config_merge_from_parent = {}) {
   using namespace formatter_rules;
   // node is empty, base-case
   if (curr_node.token || curr_node.refs.empty()) {
@@ -65,6 +69,13 @@ void apply_formatting_config(
       predefined_config = config::opengoal_form_config.at(form_head.value());
       curr_node.formatting_config = predefined_config.value();
     }
+
+    if (config_merge_from_parent) {
+      const auto& merge_config = *config_merge_from_parent.value();
+      curr_node.formatting_config.parent_mutable_extra_indent +=
+          merge_config.parent_mutable_extra_indent;
+      curr_node.formatting_config.prevent_inlining = merge_config.prevent_inlining;
+    }
   } else if (config_from_parent) {
     // TODO - doesn't merge just replaces, a bit inflexible
     predefined_config = *config_from_parent.value();
@@ -104,7 +115,7 @@ void apply_formatting_config(
       if (curr_node.formatting_config.has_constant_pairs) {
         for (int i = 0; i < (int)curr_node.refs.size(); i++) {
           auto& child_ref = curr_node.refs.at(i);
-          const auto type = child_ref.metadata.node_type;
+          const auto& type = child_ref.metadata.node_type;
           if (constant_types.find(type) == constant_types.end() &&
               constant_pairs::is_element_second_in_constant_pair(curr_node, child_ref, i)) {
             child_ref.formatting_config.parent_mutable_extra_indent = 2;
@@ -129,13 +140,16 @@ void apply_formatting_config(
     auto& ref = curr_node.refs.at(i);
     if (!ref.token) {
       // If the child has a pre-defined configuration at that index, we pass it along
-      if (predefined_config &&
-          predefined_config->index_configs.find(i) != predefined_config->index_configs.end()) {
-        apply_formatting_config(ref, predefined_config->index_configs.at(i));
+      if (predefined_config && predefined_config->index_config_override.find(i) !=
+                                   predefined_config->index_config_override.end()) {
+        apply_formatting_config(ref, {}, predefined_config->index_config_override.at(i));
+      } else if (predefined_config && predefined_config->index_configs.find(i) !=
+                                          predefined_config->index_configs.end()) {
+        apply_formatting_config(ref, predefined_config->index_configs.at(i), {});
       } else if (predefined_config && predefined_config->default_index_config) {
-        apply_formatting_config(ref, predefined_config->default_index_config);
+        apply_formatting_config(ref, predefined_config->default_index_config, {});
       } else {
-        apply_formatting_config(ref);
+        apply_formatting_config(ref, {}, {});
       }
     }
   }
@@ -151,10 +165,27 @@ void apply_formatting_config(
         max_columns = field.refs.size();
       }
     }
+    // if only one field has a value in the max col position, it looks weird for it to be indented
+    bool ignore_final_column_width = true;
+    int fields_with_atleast_max_col = 0;
+    for (const auto& field : curr_node.refs) {
+      if ((int)field.refs.size() == max_columns) {
+        fields_with_atleast_max_col++;
+        if (fields_with_atleast_max_col > 1) {
+          ignore_final_column_width = false;
+          break;
+        }
+      }
+    }
     // Now find the column max widths
     std::vector<int> column_max_widths = {};
     for (int col = 0; col < max_columns; col++) {
       column_max_widths.push_back(0);
+      // -2 because its the indentation before the final column that we want to skip
+      if (col >= curr_node.formatting_config.num_columns_to_compute_widths ||
+          (ignore_final_column_width && col == max_columns - 2)) {
+        continue;
+      }
       for (const auto& field : curr_node.refs) {
         if ((int)field.refs.size() > col) {
           const auto width = get_total_form_inlined_width(field.refs.at(col));
@@ -202,6 +233,10 @@ bool form_contains_node_that_prevents_inlining(const FormatterTreeNode& curr_nod
 bool can_node_be_inlined(const FormatterTreeNode& curr_node, int cursor_pos) {
   using namespace formatter_rules;
   if (curr_node.formatting_config.force_inline) {
+    // Ensure there are no comments, this still trumps this
+    if (form_contains_comment(curr_node)) {
+      return false;
+    }
     return true;
   }
   // First off, we cannot inline the top level
@@ -264,13 +299,15 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     // Add new line entry
     if (ref.token) {
       // Cleanup block-comments
-      std::string val = ref.token_str();
       if (ref.metadata.node_type == "block_comment") {
-        // TODO - change this sanitization to return a list of lines instead of a single new-lined
-        // line
-        val = comments::format_block_comment(ref.token_str());
+        const auto comment_lines = comments::format_block_comment(ref.token_str());
+        for (const auto& line : comment_lines) {
+          form_lines.push_back(line);
+        }
+      } else {
+        form_lines.push_back(ref.token_str());
       }
-      form_lines.push_back(val);
+
       if (!curr_node.metadata.is_top_level && i == (int)curr_node.refs.size() - 1 &&
           (ref.metadata.is_comment)) {
         // if there's an inline comment at the end of a form, we have to force the paren to the next
@@ -292,26 +329,41 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
         (curr_node.formatting_config.hang_forms ||
          curr_node.formatting_config.combine_first_two_lines)) {
-      form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+      form_lines.at(0) += fmt::format(" {}", str_util::ltrim(form_lines.at(1)));
       form_lines.erase(form_lines.begin() + 1);
     } else if ((i + 1) < (int)curr_node.refs.size()) {
       const auto& next_ref = curr_node.refs.at(i + 1);
       // combine the next inline comment or constant pair
       if ((next_ref.metadata.node_type == "comment" && next_ref.metadata.is_inline) ||
           (curr_node.formatting_config.has_constant_pairs &&
-           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1))) {
+           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1)) ||
+          constant_pairs::is_element_second_in_constant_pair_new(curr_node.refs.at(i), next_ref)) {
         // TODO
         // has issues with not consolidating first lines, this should probably just be moved to
         // outside this loop for simplicity, do it later
         if (next_ref.token) {
-          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token.value());
+          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token_str());
           i++;
+          // We have to handle hang-consolidation here or else it will never be reached above!
+          if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
+              (curr_node.formatting_config.hang_forms ||
+               curr_node.formatting_config.combine_first_two_lines)) {
+            form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+            form_lines.erase(form_lines.begin() + 1);
+          }
         } else if (can_node_be_inlined(next_ref, cursor_pos)) {
           const auto& lines = apply_formatting(next_ref, {}, cursor_pos);  // TODO - cursor pos
           for (const auto& line : lines) {
             form_lines.at(form_lines.size() - 1) += fmt::format(" {}", line);
           }
           i++;
+          // We have to handle hang-consolidation here or else it will never be reached above!
+          if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&
+              (curr_node.formatting_config.hang_forms ||
+               curr_node.formatting_config.combine_first_two_lines)) {
+            form_lines.at(0) += fmt::format(" {}", form_lines.at(1));
+            form_lines.erase(form_lines.begin() + 1);
+          }
         }
         if (!curr_node.metadata.is_top_level && next_ref.metadata.node_type == "comment" &&
             (i + 1) == (int)curr_node.refs.size()) {
@@ -328,7 +380,9 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
   // Consolidate any lines if the configuration requires it
   // TODO there is a hack here so that multi-line forms that are consolidated still line up properly
   // i have to make consolidate a more first-class feature of the config
-  if (curr_node.formatting_config.inline_until_index(form_lines)) {
+  // TODO - hacky, but prevents a bad situation, clean up
+  if (curr_node.formatting_config.inline_until_index(form_lines) &&
+      !str_util::contains(form_lines.at(0), ";")) {
     std::vector<std::string> new_form_lines = {};
     const auto original_form_head_width = str_util::split(form_lines.at(0), '\n').at(0).length();
     bool consolidating_lines = true;
@@ -374,22 +428,32 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     form_lines[form_lines.size() - 1] =
         fmt::format("{}{}", form_lines[form_lines.size() - 1], form_surround_end);
   }
-  std::string curr_form = "";
-  if (curr_node.formatting_config.parent_mutable_extra_indent > 0) {
-    curr_form += str_util::repeat(curr_node.formatting_config.parent_mutable_extra_indent, " ");
-  }
   if (inline_form) {
-    form_lines = {fmt::format("{}", fmt::join(form_lines, " "))};
+    // NOTE - not sure about this, if we are inlining a form, it always makes sense to eliminate
+    // trailing whitespace the only issue i can foresee is related to strings that span multiple
+    // lines.
+    std::vector<std::string> new_form_lines = {};
+    for (const auto& form_line : form_lines) {
+      new_form_lines.push_back(str_util::ltrim(form_line));
+    }
+    form_lines = {fmt::format("{}", fmt::join(new_form_lines, " "))};
   } else {
+    bool currently_in_block_comment = false;
     for (int i = 0; i < (int)form_lines.size(); i++) {
-      if (i > 0) {
-        auto& line = form_lines.at(i);
+      auto& line = form_lines.at(i);
+      if (str_util::contains(line, "|#")) {
+        currently_in_block_comment = false;
+      }
+      if (i > 0 && !currently_in_block_comment) {
         line = fmt::format("{}{}",
                            str_util::repeat(curr_node.formatting_config.indentation_width_for_index(
                                                 curr_node.formatting_config, i),
                                             " "),
                            line);
       }
+      if (str_util::contains(line, "#|") && !str_util::contains(line, "|#")) {
+        currently_in_block_comment = true;
+      }
     }
   }
   return form_lines;

common/formatter/formatter_tree.cpp

@@ -79,18 +79,18 @@ FormatterTree::FormatterTree(const std::string& source, const TSNode& root_node)
 }
 
 const std::unordered_map<std::string, std::vector<std::string>> node_type_ignorable_contents = {
-    {"list_lit", {"(", ")"}},
-    {"quoting_lit", {"'"}},
-    {"unquoting_lit", {","}},
-    {"quasi_quoting_lit", {"`"}}};
+    {"list_lit", {"(", ")"}}};
 
 // TODO make an imperative version eventually
+// TODO - cleanup duplication
 void FormatterTree::construct_formatter_tree_recursive(const std::string& source,
                                                        TSNode curr_node,
                                                        FormatterTreeNode& tree_node,
                                                        std::optional<std::string> node_prefix) {
   if (ts_node_child_count(curr_node) == 0) {
-    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
+    auto new_node = FormatterTreeNode(source, curr_node);
+    new_node.node_prefix = node_prefix;
+    tree_node.refs.push_back(new_node);
     return;
   }
   const std::string curr_node_type = ts_node_type(curr_node);
@@ -105,11 +105,37 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
     tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
     return;
   } else if (curr_node_type == "quoting_lit") {
-    next_node_prefix = "'";
+    if (node_prefix) {
+      node_prefix.value() += "'";
+    } else {
+      node_prefix = "'";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   } else if (curr_node_type == "unquoting_lit") {
-    next_node_prefix = ",";
+    if (node_prefix) {
+      node_prefix.value() += ",";
+    } else {
+      node_prefix = ",";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   } else if (curr_node_type == "quasi_quoting_lit") {
-    next_node_prefix = "`";
+    if (node_prefix) {
+      node_prefix.value() += "`";
+    } else {
+      node_prefix = "`";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
+  } else if (curr_node_type == "unquote_splicing_lit") {
+    if (node_prefix) {
+      node_prefix.value() += ",@";
+    } else {
+      node_prefix = ",@";
+    }
+    construct_formatter_tree_recursive(source, ts_node_child(curr_node, 1), tree_node, node_prefix);
+    return;
   }
   std::vector<std::string> skippable_nodes = {};
   if (node_type_ignorable_contents.find(curr_node_type) != node_type_ignorable_contents.end()) {
@@ -117,6 +143,7 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
   }
   for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
     const auto child_node = ts_node_child(curr_node, i);
+    auto debug_child = ts_node_string(child_node);
     const auto contents = get_source_code(source, child_node);
     bool skip_node = false;
     for (const auto& skippable_content : skippable_nodes) {
@@ -129,19 +156,22 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
       continue;
     }
     if (curr_node_type == "list_lit") {
-      construct_formatter_tree_recursive(source, child_node, list_node, next_node_prefix);
+      construct_formatter_tree_recursive(source, child_node, list_node, {});
       if (node_prefix) {
         list_node.node_prefix = node_prefix;
       }
     } else {
-      construct_formatter_tree_recursive(source, child_node, tree_node, next_node_prefix);
-      // TODO - im not sure if this is correct
+      construct_formatter_tree_recursive(source, child_node, tree_node, node_prefix);
       if (node_prefix && !tree_node.refs.empty()) {
         tree_node.refs.at(tree_node.refs.size() - 1).node_prefix = node_prefix;
       }
     }
   }
   if (curr_node_type == "list_lit") {
+    // special case for empty lists
+    if (node_prefix && !list_node.node_prefix) {
+      list_node.node_prefix = node_prefix;
+    }
     tree_node.refs.push_back(list_node);
   }
 }

common/formatter/rules/formatting_rules.cpp

@@ -12,12 +12,16 @@ namespace formatter_rules {
 // differentiate between a quoted symbol and a quoted form
 const std::set<std::string> constant_types = {"kwd_lit",  "num_lit",  "str_lit",
                                               "char_lit", "null_lit", "bool_lit"};
+const std::set<std::string> constant_type_forms = {"meters", "seconds", "degrees"};
 
 namespace constant_list {
 bool is_constant_list(const FormatterTreeNode& node) {
   if (!node.is_list() || node.refs.empty()) {
     return false;
   }
+  if (!node.refs.at(0).token) {
+    return true;
+  }
   const auto& type = node.refs.at(0).metadata.node_type;
   return constant_types.find(type) != constant_types.end();
 }
@@ -36,21 +40,28 @@ bool should_insert_blank_line(const FormatterTreeNode& containing_node,
   if (node.metadata.is_comment && node.metadata.num_blank_lines_following == 0) {
     return false;
   }
-  // If the next form is a comment and is inline, don't insert a comment
+  // If the next form is a comment and is inline, don't insert a new line
   if ((index + 1) < (int)containing_node.refs.size() &&
       containing_node.refs.at(index + 1).metadata.is_comment &&
       containing_node.refs.at(index + 1).metadata.is_inline) {
     return false;
   }
 
-  // TODO - only if the form doesn't fit on a single line
+  if (node.formatting_config.elide_top_level_newline) {
+    if ((index + 1) < (int)containing_node.refs.size() &&
+        containing_node.refs.at(index + 1).metadata.is_comment) {
+      return true;
+    }
+    return false;
+  }
+
   return true;
 }
 
 }  // namespace blank_lines
 
 namespace comments {
-std::string format_block_comment(const std::string& comment) {
+std::vector<std::string> format_block_comment(const std::string& comment) {
   // Normalize block comments, remove any trailing or leading whitespace
   // Only allow annotations on the first line, like #|@file
   // Don't mess with internal indentation as the user might intend it to be a certain way.
@@ -71,12 +82,22 @@ std::string format_block_comment(const std::string& comment) {
   // Remove trailing whitespace
   comment_contents = str_util::rtrim(comment_contents);
   // remove |#
-  // TODO - check suffix
-  comment_contents.pop_back();
-  comment_contents.pop_back();
+  if (str_util::ends_with(comment_contents, "|#")) {
+    comment_contents.pop_back();
+    comment_contents.pop_back();
+  }
   comment_contents = str_util::rtrim(comment_contents);
-  new_comment += fmt::format("\n{}\n|#", comment_contents);
-  return new_comment;
+  std::vector<std::string> lines = {new_comment};
+  const auto contents_as_lines = str_util::split_string(comment_contents, "\n");
+  if (contents_as_lines.size() > 1) {
+    for (const auto& line : contents_as_lines) {
+      lines.push_back(line);
+    }
+    lines.push_back("|#");
+  } else {
+    lines.at(0) = fmt::format("{} {} |#", new_comment, str_util::trim(contents_as_lines.at(0)));
+  }
+  return lines;
 }
 }  // namespace comments
 
@@ -101,6 +122,46 @@ bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node
   return true;
 }
 
+// TODO - potentially remove the above
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node) {
+  if (prev_node.metadata.node_type == "kwd_lit") {
+    // Handle standard constant types
+    // TODO - pair up sym_names as well
+    if (constant_types.find(curr_node.metadata.node_type) != constant_types.end()) {
+      if (curr_node.metadata.node_type != "kwd_lit") {
+        // NOTE - there is ambiugity here which cannot be totally solved (i think?)
+        // if the element itself is also a keyword, assume this is two adjacent keywords and they
+        // should not be paired
+        return true;
+      }
+    }
+    // Quoted symbols
+    if (curr_node.metadata.node_type == "sym_name" && curr_node.node_prefix &&
+        (curr_node.node_prefix.value() == "'" || curr_node.node_prefix.value() == ",")) {
+      return true;
+    }
+    if (!curr_node.refs.empty()) {
+      // Constant forms special cases (ie. meters)
+      if (constant_type_forms.find(curr_node.refs.at(0).token_str()) != constant_type_forms.end()) {
+        return true;
+      }
+      // If they are just a list of symbol names (enum or simple method call)
+      bool all_symbols = true;
+      for (const auto& ref : curr_node.refs) {
+        if (ref.metadata.node_type != "sym_name") {
+          all_symbols = false;
+          break;
+        }
+      }
+      if (all_symbols) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 bool form_should_be_constant_paired(const FormatterTreeNode& node) {
   // Criteria for a list to be constant paired:
   // - needs to start with a non-symbol

common/formatter/rules/formatting_rules.h

@@ -36,7 +36,7 @@ bool should_insert_blank_line(const FormatterTreeNode& containing_node,
 //
 // Reference - https://github.com/kkinnear/zprint/blob/main/doc/options/comments.md
 namespace comments {
-std::string format_block_comment(const std::string& comment);
+std::vector<std::string> format_block_comment(const std::string& comment);
 }
 
 // Paired elements in a list will be kept in-line rather than the default new-line indentation
@@ -72,6 +72,8 @@ const static int min_pair_amount = 4;
 bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node,
                                         const FormatterTreeNode& node,
                                         const int index);
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node);
 bool form_should_be_constant_paired(const FormatterTreeNode& node);
 }  // namespace constant_pairs
 

common/formatter/rules/rule_config.cpp

@@ -1,24 +1,146 @@
 #include "rule_config.h"
 
+#include "common/util/string_util.h"
+
 namespace formatter_rules {
 namespace config {
 
+static FormFormattingConfig new_inlinable_simple_flow_rule() {
+  return {.config_set = true, .hang_forms = false};
+}
+
 static FormFormattingConfig new_permissive_flow_rule() {
   return {.config_set = true, .hang_forms = false, .combine_first_two_lines = true};
 }
 
-static FormFormattingConfig new_flow_rule(int start_index) {
+static FormFormattingConfig new_flow_rule(int start_index, bool has_constant_pairs = false) {
   return {.config_set = true,
           .hang_forms = false,
-          .inline_until_index = [start_index](const std::vector<std::string>& /*curr_lines*/) {
-            return start_index;
-          }};
+          .inline_until_index =
+              [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+          .has_constant_pairs = has_constant_pairs};
+}
+
+static FormFormattingConfig new_function_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index =
+          [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+      .has_constant_pairs = has_constant_pairs};
+  auto arg_list_config = std::make_shared<FormFormattingConfig>();
+  arg_list_config->force_inline = true;
+  arg_list_config->hang_forms = false;
+  cfg.index_configs.emplace(2, arg_list_config);
+  return cfg;
+}
+
+static FormFormattingConfig new_inlineable_flow_rule(int start_index,
+                                                     bool has_constant_pairs = false) {
+  return {.config_set = true,
+          .hang_forms = false,
+          .inline_until_index =
+              [start_index](const std::vector<std::string>& curr_lines) {
+                int total_width = 0;
+                for (const auto& line : curr_lines) {
+                  // Check for comments
+                  // TODO - this shows how this isn't really the best strategy but it holds up
+                  if (str_util::contains(line, ";")) {
+                    // Can't inline, there's a comment!
+                    return start_index;
+                  }
+                  total_width += line.length();
+                  // an empty line implies a new-line was forced, this is bleeding implementation
+                  // details, but fine for now
+                  if (line.empty()) {
+                    return start_index;
+                  }
+                }
+                if (total_width <= 120) {
+                  return (int)curr_lines.size();
+                }
+                return start_index;
+              },
+          .has_constant_pairs = has_constant_pairs};
+}
+
+static FormFormattingConfig new_defstate_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index =
+          [start_index](const std::vector<std::string>& /*curr_lines*/) { return start_index; },
+      .has_constant_pairs = has_constant_pairs};
+  std::vector<int> state_handler_indexes = {4,  6, 8, 10,
+                                            12, 14};  // NOTE - not all of these have to be defined
+  for (const auto& index : state_handler_indexes) {
+    auto temp_config = std::make_shared<FormFormattingConfig>();
+    temp_config->prevent_inlining = true;
+    temp_config->parent_mutable_extra_indent = 2;
+    cfg.index_config_override.emplace(index, temp_config);
+  }
+  return cfg;
+}
+
+static FormFormattingConfig new_defmethod_rule(int start_index, bool has_constant_pairs = false) {
+  // TODO - might be nice to have a function that returns a config based on a given index, instead
+  // of hardcoding them!
+  // Right now this only works for non-`new` methods (else we may bleed into the body of a normal
+  // method)
+  auto arg_list_config = std::make_shared<FormFormattingConfig>();
+  arg_list_config->force_inline = true;
+  arg_list_config->hang_forms = false;
+  FormFormattingConfig cfg = {.config_set = true,
+                              .hang_forms = false,
+                              .inline_until_index =
+                                  [start_index](const std::vector<std::string>& curr_lines) {
+                                    if (curr_lines.size() >= 2 && curr_lines.at(1) == "new") {
+                                      // defmethod was changed to omit the type name for everything
+                                      // except the `new` method, so special case.
+                                      return start_index + 1;
+                                    }
+                                    return start_index;
+                                  },
+                              .has_constant_pairs = has_constant_pairs};
+  cfg.index_configs.emplace(2, arg_list_config);
+  return cfg;
+}
+
+static FormFormattingConfig new_lambda_rule(int start_index, bool has_constant_pairs = false) {
+  FormFormattingConfig cfg = {.config_set = true,
+                              .hang_forms = false,
+                              .inline_until_index =
+                                  [start_index](const std::vector<std::string>& curr_lines) {
+                                    if (curr_lines.size() >= 2 && curr_lines.at(1) == ":behavior") {
+                                      // defmethod was changed to omit the type name for everything
+                                      // except the `new` method, so special case.
+                                      return start_index + 2;
+                                    }
+                                    return start_index;
+                                  },
+                              .has_constant_pairs = has_constant_pairs};
+  return cfg;
+}
+
+static FormFormattingConfig new_defenum_rule() {
+  auto temp_list_config = std::make_shared<FormFormattingConfig>();
+  temp_list_config->force_inline = true;
+  temp_list_config->hang_forms = false;
+  return {
+      .config_set = true,
+      .hang_forms = false,
+      .inline_until_index = [](const std::vector<std::string>& /*curr_lines*/) { return 2; },
+      .has_constant_pairs = true,
+      .default_index_config = temp_list_config,
+  };
 }
 
 static FormFormattingConfig new_deftype_rule(
     int start_index,
+    int num_columns_to_compute_widths,
     const std::vector<int>& inlining_preventation_indices) {
   FormFormattingConfig cfg;
+  cfg.has_constant_pairs = true;
   cfg.config_set = true;
   cfg.hang_forms = false;
   cfg.inline_until_index = [start_index](std::vector<std::string> curr_lines) {
@@ -39,6 +161,7 @@ static FormFormattingConfig new_deftype_rule(
     temp_config->default_index_config = temp_list_config;
     if (index == 3) {
       temp_config->determine_column_widths_for_list_elements = true;
+      temp_config->num_columns_to_compute_widths = num_columns_to_compute_widths;
     }
     cfg.index_configs.emplace(index, temp_config);
   }
@@ -74,6 +197,29 @@ static FormFormattingConfig new_binding_rule(int form_head_width) {
   return cfg;
 }
 
+static FormFormattingConfig new_inline_binding_rule(int form_head_width) {
+  FormFormattingConfig cfg;
+  cfg.config_set = true;
+  cfg.hang_forms = false;
+  cfg.combine_first_two_lines = true;
+  auto binding_list_config = std::make_shared<FormFormattingConfig>();
+  binding_list_config->config_set = true;
+  binding_list_config->hang_forms = false;
+  binding_list_config->indentation_width = 1;
+  binding_list_config->indentation_width_for_index = [form_head_width](FormFormattingConfig /*cfg*/,
+                                                                       int index) {
+    if (index == 0) {
+      return 0;
+    }
+    return form_head_width;
+  };
+  binding_list_config->should_prevent_inlining = [](FormFormattingConfig /*config*/, int num_refs) {
+    return false;
+  };
+  cfg.index_configs.emplace(1, binding_list_config);
+  return cfg;
+}
+
 static FormFormattingConfig new_pair_rule(bool combine_first_two_expr) {
   FormFormattingConfig cfg;
   cfg.config_set = true;
@@ -88,23 +234,71 @@ static FormFormattingConfig new_pair_rule(bool combine_first_two_expr) {
   return cfg;
 }
 
+static FormFormattingConfig new_top_level_inline_form(bool elide_new_line) {
+  return {.force_inline = true, .elide_top_level_newline = elide_new_line};
+}
+
 const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config = {
     {"case", new_pair_rule(true)},
+    {"case-str", new_pair_rule(true)},
     {"cond", new_pair_rule(false)},
-    {"defmethod", new_flow_rule(3)},
-    {"deftype", new_deftype_rule(3, {3, 4, 5, 6})},
+    {"#cond", new_pair_rule(false)},
+    {"in-package", new_top_level_inline_form(true)},
+    {"bundles", new_top_level_inline_form(true)},
+    {"require", new_top_level_inline_form(true)},
+    {"def-art-elt", new_top_level_inline_form(true)},
+    {"def-joint-node", new_top_level_inline_form(true)},
+    {"declare-file", new_top_level_inline_form(false)},
+    {"defenum", new_defenum_rule()},
+    {"defmethod", new_defmethod_rule(3)},
+    {"lambda", new_lambda_rule(2)},
+    {"deftype", new_deftype_rule(3, 1, {3, 4, 5, 6})},
     {"defun", new_flow_rule(3)},
+    {"defun-recursive", new_flow_rule(4)},
+    {"defun-debug-recursive", new_flow_rule(4)},
     {"defun-debug", new_flow_rule(3)},
     {"defbehavior", new_flow_rule(4)},
-    {"if", new_permissive_flow_rule()},
+    {"if", new_inlineable_flow_rule(2)},
+    {"#if", new_inlineable_flow_rule(2)},
     {"define", new_permissive_flow_rule()},
+    {"def-mips2c", new_permissive_flow_rule()},
+    {"defconstant", new_permissive_flow_rule()},
+    {"defglobalconstant", new_permissive_flow_rule()},
+    {"defmethod-mips2c", new_permissive_flow_rule()},
     {"define-extern", new_permissive_flow_rule()},
-    {"defmacro", new_flow_rule(3)},
+    {"declare-type", new_permissive_flow_rule()},
+    {"defmacro", new_function_rule(3)},
+    {"desfun", new_function_rule(3)},
+    {"defskelgroup", new_flow_rule(2, true)},
+    {"defpartgroup", new_flow_rule(2, true)},
+    {"defpart", new_flow_rule(2, true)},
+    {"defstate", new_defstate_rule(3, true)},
+    {"behavior", new_flow_rule(2)},
     {"dotimes", new_flow_rule(2)},
+    {"dolist", new_flow_rule(2)},
+    {"process-spawn-function", new_flow_rule(2)},
     {"let", new_binding_rule(4)},
+    {"protect", new_binding_rule(4)},
+    {"let*", new_binding_rule(5)},
     {"rlet", new_binding_rule(5)},
     {"when", new_flow_rule(2)},
+    {"unless", new_flow_rule(2)},
+    {"with-profiler", new_flow_rule(2)},
+    {"with-pc", new_flow_rule(0)},
+    {"#unless", new_flow_rule(2)},
+    {"#when", new_flow_rule(2)},
+    {"countdown", new_flow_rule(2)},
+    {"until", new_flow_rule(2)},
+    {"loop", new_flow_rule(0)},
+    {"while", new_flow_rule(2)},
     {"begin", new_flow_rule(0)},
+    {"seval", new_flow_rule(0)},
+    {"with-pp", new_flow_rule(0)},
+    {"with-gensyms", new_flow_rule(2)},
+    {"with-vf0", new_flow_rule(0)},
+    {"with-vf", new_inline_binding_rule(8)},
+    {"with-cnt-vif-block", new_inline_binding_rule(19)},
+    {"local-vars", new_inlinable_simple_flow_rule()},
     {"with-dma-buffer-add-bucket", new_flow_rule(2)}};
 }  // namespace config
 }  // namespace formatter_rules

common/formatter/rules/rule_config.h

@@ -7,7 +7,6 @@
 #include <unordered_map>
 #include <vector>
 
-// TODO - some way to apply a config to all list elements (index configs with -1?)
 namespace formatter_rules {
 namespace config {
 struct FormFormattingConfig {
@@ -33,9 +32,14 @@ struct FormFormattingConfig {
   int parent_mutable_extra_indent = 0;
   std::optional<std::shared_ptr<FormFormattingConfig>> default_index_config;
   std::unordered_map<int, std::shared_ptr<FormFormattingConfig>> index_configs = {};
+  // TODO / NOTe - not fully implemented, only doing `parent_mutable_extra_indent` right now
+  std::unordered_map<int, std::shared_ptr<FormFormattingConfig>> index_config_override = {};
 
   bool determine_column_widths_for_list_elements = false;
+  int num_columns_to_compute_widths = 0;
   std::vector<int> list_element_column_widths = {};
+
+  bool elide_top_level_newline = false;
 };
 
 extern const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config;

common/global_profiler/GlobalProfiler.cpp

@@ -26,6 +26,8 @@ u64 get_current_tid() {
 }
 #endif
 #include "common/log/log.h"
+#include "common/util/compress.h"
+#include "common/util/string_util.h"
 // clang-format on
 
 u64 get_current_ts() {
@@ -34,12 +36,12 @@ u64 get_current_ts() {
 
 GlobalProfiler::GlobalProfiler() {
   m_t0 = get_current_ts();
-  set_max_events(65536);
+  m_nodes.resize(m_max_events);
 }
 
-void GlobalProfiler::set_max_events(size_t event_count) {
-  ASSERT(!m_enabled);
-  m_nodes.resize(event_count);
+void GlobalProfiler::update_event_buffer_size(size_t new_size) {
+  m_max_events = new_size;
+  m_nodes.resize(m_max_events);
 }
 
 void GlobalProfiler::set_waiting_for_event(const std::string& event_name) {
@@ -72,6 +74,10 @@ void GlobalProfiler::root_event() {
   instant_event("ROOT");
 }
 
+size_t GlobalProfiler::get_next_idx() {
+  return (m_next_idx % m_nodes.size());
+}
+
 void GlobalProfiler::begin_event(const char* name) {
   event(name, ProfNode::BEGIN);
 }
@@ -96,8 +102,10 @@ void GlobalProfiler::set_enable(bool en) {
   m_enabled = en;
 }
 
-void GlobalProfiler::dump_to_json(const std::string& path) {
-  ASSERT(!m_enabled);
+void GlobalProfiler::dump_to_json() {
+  if (m_enabled) {
+    set_enable(false);
+  }
 
   nlohmann::json json;
   auto& trace_events = json["traceEvents"];
@@ -185,7 +193,20 @@ void GlobalProfiler::dump_to_json(const std::string& path) {
               t.second.highest_at_target);
   }
 
-  file_util::write_text_file(path, json.dump());
+  if (m_enable_compression) {
+    const auto json_str = json.dump();
+    const auto compressed_data =
+        compression::compress_zstd_no_header(json_str.data(), json_str.size());
+    auto file_path = file_util::get_jak_project_dir() / "profile_data" /
+                     fmt::format("prof-{}.json.zst", str_util::current_local_timestamp_no_colons());
+    file_util::create_dir_if_needed_for_file(file_path);
+    file_util::write_binary_file(file_path, compressed_data.data(), compressed_data.size());
+  } else {
+    auto file_path = file_util::get_jak_project_dir() / "profile_data" /
+                     fmt::format("prof-{}.json", str_util::current_local_timestamp_no_colons());
+    file_util::create_dir_if_needed_for_file(file_path);
+    file_util::write_text_file(file_path, json.dump());
+  }
 }
 
 GlobalProfiler gprof;

common/global_profiler/GlobalProfiler.h

@@ -17,7 +17,8 @@ struct ProfNode {
 class GlobalProfiler {
  public:
   GlobalProfiler();
-  void set_max_events(size_t event_count);
+  size_t get_max_events() { return m_max_events; }
+  void update_event_buffer_size(size_t new_size);
   void set_waiting_for_event(const std::string& event_name);
   void instant_event(const char* name);
   void begin_event(const char* name);
@@ -25,11 +26,16 @@ class GlobalProfiler {
   void end_event();
   void clear();
   void set_enable(bool en);
-  void dump_to_json(const std::string& path);
+  void dump_to_json();
   void root_event();
+  bool is_enabled() { return m_enabled; }
+  size_t get_next_idx();
+
+  bool m_enable_compression = false;
 
  private:
   std::atomic_bool m_enabled = false;
+  size_t m_max_events = 65536;
   u64 m_t0 = 0;
   std::atomic_size_t m_next_idx = 0;
   std::vector<ProfNode> m_nodes;

common/goos/Reader.cpp

@@ -12,7 +12,6 @@
 #include "Reader.h"
 
 #include "common/log/log.h"
-#include "common/repl/util.h"
 #include "common/util/FileUtil.h"
 #include "common/util/FontUtils.h"
 

common/goos/Reader.h

@@ -18,7 +18,7 @@
 
 #include "common/goos/Object.h"
 #include "common/goos/TextDB.h"
-#include "common/repl/util.h"
+#include "common/repl/repl_wrapper.h"
 #include "common/util/Assert.h"
 
 namespace goos {

common/log/log.cpp

@@ -140,7 +140,7 @@ void printstd(const char* format, va_list arg_list) {
   internal::log_vprintf(format, arg_list);
 }
 
-// how many extra log files for a single program should be kept?
+// how many extra log files for a single program should be kept
 constexpr int LOG_ROTATE_MAX = 10;
 
 void set_file(const std::string& filename,
@@ -163,7 +163,7 @@ void set_file(const std::string& filename,
         file_util::find_files_in_dir(fs::path(complete_filename).parent_path(),
                                      std::regex(fmt::format("{}\\.(\\d\\.)?log", filename)));
     for (const auto& file : old_log_files) {
-      lg::info("removing {}", file.string());
+      lg::debug("removing {}", file.string());
       fs::remove(file);
     }
     // remove the oldest log file if there are more than LOG_ROTATE_MAX
@@ -172,9 +172,9 @@ void set_file(const std::string& filename,
     // sort the names and remove them
     existing_log_files = file_util::sort_filepaths(existing_log_files, true);
     if (existing_log_files.size() > (LOG_ROTATE_MAX - 1)) {
-      lg::info("removing {} log files", existing_log_files.size() - (LOG_ROTATE_MAX - 1));
+      lg::debug("removing {} log files", existing_log_files.size() - (LOG_ROTATE_MAX - 1));
       for (int i = 0; i < (int)existing_log_files.size() - (LOG_ROTATE_MAX - 1); i++) {
-        lg::info("removing {}", existing_log_files.at(i).string());
+        lg::debug("removing {}", existing_log_files.at(i).string());
         fs::remove(existing_log_files.at(i));
       }
     }

common/repl/config.cpp

@@ -7,15 +7,21 @@
 namespace REPL {
 void to_json(json& j, const Config& obj) {
   j = json{
+      {"nreplPort", obj.nrepl_port},
       {"gameVersionFolder", obj.game_version_folder},
       {"numConnectToTargetAttempts", obj.target_connect_attempts},
       {"asmFileSearchDirs", obj.asm_file_search_dirs},
       {"keybinds", obj.keybinds},
       {"perGameHistory", obj.per_game_history},
+      {"permissiveRedefinitions", obj.permissive_redefinitions},
   };
 }
 
 void from_json(const json& j, Config& obj) {
+  // TODO - make a camelCase variant of json_serialize/deserialize macros
+  if (j.contains("nreplPort")) {
+    j.at("nreplPort").get_to(obj.nrepl_port);
+  }
   if (j.contains("gameVersionFolder")) {
     j.at("gameVersionFolder").get_to(obj.game_version_folder);
   }
@@ -55,6 +61,9 @@ void from_json(const json& j, Config& obj) {
   if (j.contains("perGameHistory")) {
     j.at("perGameHistory").get_to(obj.per_game_history);
   }
+  if (j.contains("permissiveRedefinitions")) {
+    j.at("permissiveRedefinitions").get_to(obj.permissive_redefinitions);
+  }
   // if there is game specific configuration, override any values we just set
   if (j.contains(version_to_game_name(obj.game_version))) {
     from_json(j.at(version_to_game_name(obj.game_version)), obj);

common/repl/config.h

@@ -26,11 +26,14 @@ struct KeyBind {
 void to_json(json& j, const KeyBind& obj);
 void from_json(const json& j, KeyBind& obj);
 
+// TODO - per-game config
 struct Config {
   GameVersion game_version;
   Config(GameVersion _game_version) : game_version(_game_version){};
 
   // this is the default REPL configuration
+  int nrepl_port = 8181;
+  int temp_nrepl_port = -1;
   std::string game_version_folder;
   int target_connect_attempts = 30;
   std::vector<std::string> asm_file_search_dirs = {};
@@ -45,6 +48,14 @@ struct Config {
       {KeyBind::Modifier::CTRL, "B", "Displays the most recently caught backtrace", "(:di)"},
       {KeyBind::Modifier::CTRL, "N", "Full build of the game", "(mi)"}};
   bool per_game_history = true;
+  bool permissive_redefinitions = false;
+
+  int get_nrepl_port() {
+    if (temp_nrepl_port != -1) {
+      return temp_nrepl_port;
+    }
+    return nrepl_port;
+  }
 };
 void to_json(json& j, const Config& obj);
 void from_json(const json& j, Config& obj);

common/repl/nrepl/ReplServer.cpp

@@ -16,10 +16,6 @@
 #include "common/log/log.h"
 // clang-format on
 
-// TODO - basically REPL to listen and inject commands into a running REPL
-// - we will need a C++ side client as well which will let us communicate with the repl via for
-// example, ImgUI
-//
 // TODO - The server also needs to eventually return the result of the evaluation
 
 ReplServer::~ReplServer() {
@@ -31,7 +27,18 @@ ReplServer::~ReplServer() {
 
 void ReplServer::post_init() {
   // Add the listening socket to our set of sockets
-  lg::info("[nREPL:{}:{}] awaiting connections", tcp_port, listening_socket);
+  lg::debug("[nREPL:{}:{}] awaiting connections", tcp_port, listening_socket);
+}
+
+void ReplServer::error_response(int socket, const std::string& error) {
+  std::string msg = fmt::format("[ERROR]: {}", error);
+  auto resp = write_to_socket(socket, msg.c_str(), msg.size());
+  if (resp == -1) {
+    lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
+             ntohs(addr.sin_port), socket);
+    close_socket(socket);
+    client_sockets.erase(socket);
+  }
 }
 
 void ReplServer::ping_response(int socket) {
@@ -52,7 +59,6 @@ std::optional<std::string> ReplServer::get_msg() {
 
   // Add the server's main listening socket (where we accept clients from)
   FD_SET(listening_socket, &read_sockets);
-
   int max_sd = listening_socket;
   for (const int& sock : client_sockets) {
     if (sock > max_sd) {
@@ -64,12 +70,11 @@ std::optional<std::string> ReplServer::get_msg() {
   }
 
   // Wait for activity on _something_, with a timeout so we don't get stuck here on exit.
-  struct timeval timeout;
-  timeout.tv_sec = 0;
-  timeout.tv_usec = 100000;
+  struct timeval timeout = {0, 100000};
   auto activity = select(max_sd + 1, &read_sockets, NULL, NULL, &timeout);
-
-  if (activity < 0) {  // TODO - || error!
+  if (activity < 0 && errno != EINTR) {
+    lg::error("[nREPL:{}] select error, returned: {}, errno: {}", tcp_port, activity,
+              strerror(errno));
     return std::nullopt;
   }
 
@@ -78,44 +83,46 @@ std::optional<std::string> ReplServer::get_msg() {
     socklen_t addr_len = sizeof(addr);
     auto new_socket = accept_socket(listening_socket, (sockaddr*)&addr, &addr_len);
     if (new_socket < 0) {
-      // TODO - handle error
+      if (new_socket != -1) {
+        lg::error("[nREPL:{}] accept error, returned: {}, errono: {}", tcp_port, new_socket,
+                  strerror(errno));
+      }
     } else {
       lg::info("[nREPL:{}]: New socket connection: {}:{}:{}", tcp_port, address_to_string(addr),
                ntohs(addr.sin_port), new_socket);
-
       // Say hello
       ping_response(new_socket);
       // Track the new socket
       if ((int)client_sockets.size() < max_clients) {
         client_sockets.insert(new_socket);
       } else {
-        // TODO - Respond with NO
+        // Respond with NO and close the socket
+        lg::warn("[nREPL:{}]: Maximum clients reached. Rejecting connection.", tcp_port);
+        error_response(new_socket, "Maximum clients reached. Rejecting connection.");
+        close_socket(new_socket);
       }
     }
   }
 
-  // otherwise (and no matter what) check all the clients to see if they have sent us anything
-  // else its some IO operation on some other socket
-  //
-  // RACE - the first client wins
-
-  // TODO - there are ways to do this with iterators but, couldn't figure it out!
-  std::vector<int> sockets_to_scan(client_sockets.begin(), client_sockets.end());
-  for (const int& sock : sockets_to_scan) {
+  // Check all clients for activity
+  for (auto it = client_sockets.begin(); it != client_sockets.end();) {
+    int sock = *it;
     if (FD_ISSET(sock, &read_sockets)) {
       // Attempt to read a header
-      // TODO - should this be in a loop?
       auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
-      if (req_bytes == 0) {
-        // Socket disconnected
+      if (req_bytes <= 0) {
         // TODO - add a queue of messages in the REPL::Wrapper so we can print _BEFORE_ the prompt
         // is output
-        lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr),
-                 ntohs(addr.sin_port), sock);
-
+        if (req_bytes == 0) {
+          lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr));
+        } else {
+          lg::warn("[nREPL:{}] Error reading from socket on {}: {}", tcp_port,
+                   address_to_string(addr), strerror(errno));
+        }
         // Cleanup the socket and remove it from our set
         close_socket(sock);
-        client_sockets.erase(sock);
+        it = client_sockets.erase(it);  // Erase and move to the next element
+        continue;
       } else {
         // Otherwise, process the message
         auto* header = (ReplServerHeader*)(header_buffer.data());
@@ -123,7 +130,12 @@ std::optional<std::string> ReplServer::get_msg() {
         int expected_size = header->length;
         int got = 0;
         int tries = 0;
+        bool skip_to_next_socket = false;
         while (got < expected_size) {
+          if (want_exit_callback()) {
+            lg::warn("[nREPL:{}] Terminating nREPL early", tcp_port);
+            return std::nullopt;
+          }
           tries++;
           if (tries > 100) {
             break;
@@ -135,11 +147,25 @@ std::optional<std::string> ReplServer::get_msg() {
                 tcp_port, got, expected_size, buffer.size());
             return std::nullopt;
           }
-          auto x = read_from_socket(sock, buffer.data() + got, expected_size - got);
-          if (want_exit_callback()) {
-            return std::nullopt;
+          auto bytes_read = read_from_socket(sock, buffer.data() + got, expected_size - got);
+          if (bytes_read <= 0) {
+            if (bytes_read == 0) {
+              lg::warn("[nREPL:{}] Client Disconnected: {}", tcp_port, address_to_string(addr));
+            } else {
+              lg::warn("[nREPL:{}] Error reading from socket on {}: {}", tcp_port,
+                       address_to_string(addr), strerror(errno));
+            }
+            close_socket(sock);
+            it = client_sockets.erase(it);  // Erase and move to the next element
+            skip_to_next_socket = true;
+            break;
           }
-          got += x > 0 ? x : 0;
+          got += bytes_read;
+          std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+
+        if (skip_to_next_socket) {
+          continue;
         }
 
         switch (header->type) {
@@ -153,6 +179,7 @@ std::optional<std::string> ReplServer::get_msg() {
         }
       }
     }
+    ++it;
   }
   return std::nullopt;
 }

common/repl/nrepl/ReplServer.h

@@ -27,5 +27,6 @@ class ReplServer : public XSocketServer {
   fd_set read_sockets;
   std::set<int> client_sockets = {};
 
+  void error_response(int socket, const std::string& error);
   void ping_response(int socket);
 };

common/repl/repl_wrapper.cpp

@@ -1,4 +1,4 @@
-#include "util.h"
+#include "repl_wrapper.h"
 
 #include "common/util/FileUtil.h"
 #include "common/util/json_util.h"
@@ -8,36 +8,66 @@
 #include "fmt/color.h"
 #include "fmt/core.h"
 #include "third-party/replxx/include/replxx.hxx"
-// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
-// or perhaps, show the docstring for the current function being used?)
 
 namespace REPL {
 void Wrapper::clear_screen() {
   repl.clear_screen();
 }
 
-void Wrapper::print_welcome_message() {
-  // TODO - dont print on std-out
-  // Welcome message / brief intro for documentation
-  std::string ascii;
-  ascii += " _____             _____ _____ _____ __    \n";
-  ascii += "|     |___ ___ ___|   __|     |  _  |  |   \n";
-  ascii += "|  |  | . | -_|   |  |  |  |  |     |  |__ \n";
-  ascii += "|_____|  _|___|_|_|_____|_____|__|__|_____|\n";
-  ascii += "      |_|                                  \n";
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
-
-  fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
-             versions::GOAL_VERSION_MINOR);
-  fmt::print("Run {} or {} for help with common commands and REPL usage.\n",
-             fmt::styled("(repl-help)", fmt::emphasis::bold | fg(fmt::color::cyan)),
-             fmt::styled("(repl-keybinds)", fmt::emphasis::bold | fg(fmt::color::cyan)));
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
-  fmt::print(" to connect to the local target.\n");
-  fmt::print("Run ");
-  fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
-  fmt::print(" to rebuild the entire game.\n\n");
+void Wrapper::print_welcome_message(const std::vector<std::string>& loaded_projects) {
+  std::string message;
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "        ..:::::..\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "     .:-----------:.\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "         .-----.");
+  message += fmt::format(fmt::emphasis::bold, "         Welcome to OpenGOAL {}.{} [{}]",
+                         versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR,
+                         fmt::format(fg(fmt::color::gray), "{}", build_revision()));
+  if (!username.empty() && username != "#f" && username != "unknown") {
+    message += fmt::format(fg(fmt::color::light_green), " {}", username);
+  }
+  message += "!\n";
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "          .---.");
+  if (repl_config.game_version == GameVersion::Jak1) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::orange), "jak1"));
+  } else if (repl_config.game_version == GameVersion::Jak2) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::purple), "jak2"));
+  } else if (repl_config.game_version == GameVersion::Jak3) {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::gold), "jak3"));
+  } else {
+    message += fmt::format("          [{}]: ", fmt::format(fg(fmt::color::magenta), "jakx"));
+  }
+  const auto loaded_projects_str = fmt::format("{}", fmt::join(loaded_projects, ","));
+  message += fmt::format(fg(fmt::color::gray), "{}\n", loaded_projects_str);
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  .        ---        .");
+  message +=
+      fmt::format("  Project Path: {}\n",
+                  fmt::format(fg(fmt::color::gray), file_util::get_jak_project_dir().string()));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  -       :===:       -");
+  message += "  nREPL:";
+  if (!nrepl_alive) {
+    message += fmt::format(fg(fmt::color::red), "DISABLED\n");
+  } else {
+    message += fmt::format(fg(fmt::color::light_green), " Listening on {}\n",
+                           repl_config.get_nrepl_port());
+  }
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  --.   .--: :--.   .--");
+  message += "  Source File Search Dirs: ";
+  const auto search_dir_string =
+      fmt::format("{}", fmt::join(repl_config.asm_file_search_dirs, ","));
+  message += fmt::format("[{}]\n", fmt::format(fg(fmt::color::gray), search_dir_string));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "  .=======.    =======.");
+  message += fmt::format("  {} or {} for basic help and usage\n",
+                         fmt::format(fg(fmt::color::cyan), "(repl-help)"),
+                         fmt::format(fg(fmt::color::cyan), "(repl-keybinds)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "   .-=====-. .-=====-");
+  message +=
+      fmt::format("    {} to connect to the game\n", fmt::format(fg(fmt::color::cyan), "(lt)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "     .-===========-.");
+  message += fmt::format("     {} to recompile the active project.\n",
+                         fmt::format(fg(fmt::color::cyan), "(mi)"));
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "         .-===-.\n");
+  message += fmt::format(fmt::emphasis::bold | fg(fmt::color::orange), "            .\n");
+  fmt::print("{}", message);
 }
 
 void Wrapper::print_to_repl(const std::string& str) {
@@ -242,20 +272,24 @@ StartupFile load_user_startup_file(const std::string& username, const GameVersio
   return startup_file;
 }
 
-REPL::Config load_repl_config(const std::string& username, const GameVersion game_version) {
+REPL::Config load_repl_config(const std::string& username,
+                              const GameVersion game_version,
+                              const int nrepl_port) {
   auto repl_config_path =
       file_util::get_jak_project_dir() / "goal_src" / "user" / username / "repl-config.json";
+  REPL::Config loaded_config(game_version);
   if (file_util::file_exists(repl_config_path.string())) {
     try {
-      REPL::Config config(game_version);
       auto repl_config_data =
           parse_commented_json(file_util::read_text_file(repl_config_path), "repl-config.json");
-      from_json(repl_config_data, config);
-      return config;
+      from_json(repl_config_data, loaded_config);
+      loaded_config.temp_nrepl_port = nrepl_port;
+      return loaded_config;
     } catch (std::exception& e) {
-      REPL::Config config(game_version);
+      // do nothing
     }
   }
-  return REPL::Config(game_version);
+  loaded_config.temp_nrepl_port = nrepl_port;
+  return loaded_config;
 }
 }  // namespace REPL

common/repl/repl_wrapper.h

@@ -17,18 +17,23 @@ struct StartupFile {
 };
 
 class Wrapper {
-  replxx::Replxx repl;
-
  public:
   std::string username;
   Config repl_config;
   StartupFile startup_file;
+  bool nrepl_alive = false;
   std::vector<std::string> examples{};
   std::vector<std::pair<std::string, replxx::Replxx::Color>> regex_colors{};
 
   Wrapper(GameVersion version) : repl_config(version) {}
-  Wrapper(const std::string& _username, const Config& config, const StartupFile& startup)
-      : username(_username), repl_config(config), startup_file(startup) {}
+  Wrapper(const std::string& _username,
+          const Config& config,
+          const StartupFile& startup,
+          bool _nrepl_alive)
+      : username(_username),
+        repl_config(config),
+        startup_file(startup),
+        nrepl_alive(_nrepl_alive) {}
   replxx::Replxx& get_repl() { return repl; }
   void init_settings();
   void reload_startup_file();
@@ -36,7 +41,7 @@ class Wrapper {
   // Functionality / Commands
   void clear_screen();
   void print_to_repl(const std::string& str);
-  void print_welcome_message();
+  void print_welcome_message(const std::vector<std::string>& loaded_projects);
   void set_history_max_size(size_t len);
   const char* readline(const std::string& prompt);
   void add_to_history(const std::string& line);
@@ -47,11 +52,14 @@ class Wrapper {
   std::pair<std::string, bool> get_current_repl_token(std::string const& context);
 
  private:
+  replxx::Replxx repl;
   replxx::Replxx::key_press_handler_t commit_text_action(std::string text_to_commit);
   std::vector<REPL::KeyBind> keybindings = {};
 };
 
 std::string find_repl_username();
 StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version);
-REPL::Config load_repl_config(const std::string& username, const GameVersion game_version);
+REPL::Config load_repl_config(const std::string& username,
+                              const GameVersion game_version,
+                              const int nrepl_port);
 }  // namespace REPL

common/serialization/subtitles/subtitles_v2.cpp

@@ -89,7 +89,9 @@ const std::unordered_map<std::string, u16> jak2_speaker_name_to_enum_val = {
     {"citizen-male", 31},
     {"citizen-female", 32},
     {"oracle", 33},
-    {"precursor", 34}};
+    {"precursor", 34},
+    {"metalkor-before-consite", 35},
+    {"metalkor-intro", 36}};
 
 GameSubtitlePackage read_json_files_v2(const GameSubtitleDefinitionFile& file_info) {
   GameSubtitlePackage package;

common/texture/texture_slots.cpp

@@ -42,7 +42,153 @@ std::vector<std::string> jak2_slots = {
     "cas-toxic-slime-scroll-dest",
 };
 
-std::vector<std::string> jak3_slots = {};
+std::vector<std::string> jak3_slots = {
+    "skull-gem-dest",
+    "jakc-arm",
+    "jakc-eyebrow",
+    "jakc-face",
+    "jakc-finger",
+    "jakc-hair",
+    "jakchires-arm",
+    "jakchires-eye",
+    "jakchires-eyebrow",
+    "jakchires-eyelid",
+    "jakchires-facelft",
+    "jakchires-facert",
+    "jakchires-hair",
+    // default-water
+    "bomb-gradient",
+    "blue-beam-dest",
+    "lightjak-wings",
+    "mushroom-dest",
+    // default-warp
+    "shield-env-rim-dest",
+    // templea-water
+    "templea-waterfall-dest",
+    // templea-warp
+    "holograph-env-rim-dest",
+    // hanga-sprite
+    "glider-ring-dest2",
+    "glider-ring-dest",
+    // foresta-water
+    "fora-water-dest",
+    "fora-waterfall-01-dest",
+    "fora-water-wave-01-dest",
+    // forestb-water
+    "forb-water-dest",
+    "forb-waterfall-01-dest",
+    "forb-water-wave-01-dest",
+    // lforplnt-pris
+    "mh-gem-dest",
+    // lmhcitya-tfrag
+    "mhcitya-base-goo-01-dest",
+    // lmhcityb-tfrag
+    "mhcityb-base-goo-01-dest",
+    // mhcitya-pris
+    // "mhcity-de-door-skin-01-dest",
+    // templec-water
+    "tplc-water-dest",
+    // sewc-water
+    "sewer-water-01-c-dest",
+    "sewer-waterfall-01-c-dest",
+    "sewer-waterfall-02-c-dest",
+    "sewer-water-wave-01-c-dest",
+    "sewer-water-highlight-01-c-dest",
+    // sewd-water
+    "sewer-water-01-d-dest",
+    "sewer-waterfall-02-d-dest",
+    "sewer-water-highlight-01-d-dest",
+    "sewer-water-wave-01-d-dest",
+    "sewer-water-wave-02-d-dest",
+    "sewer-water-still-01-d-dest",
+    // sewe-water
+    "sewer-water-01-e-dest",
+    "sewer-waterfall-01-e-dest",
+    "sewer-waterfall-02-e-dest",
+    "sewer-water-highlight-01-e-dest",
+    // sewg-water
+    "sewer-water-01-g-dest",
+    "sewer-waterfall-02-g-dest",
+    "sewer-water-wave-01-g-dest",
+    // sewh-water
+    "sewer-water-01-h-dest",
+    "sewer-waterfall-02-h-dest",
+    "sewer-water-wave-02-h-dest",
+    "sewer-watefall-froth-01-h-dest",
+    // sewi-water
+    "sewer-water-still-01-i-dest",
+    "sewer-waterfall-02-i-dest",
+    "sewer-water-wave-01-i-dest",
+    // sewj-water
+    "sewer-waterfall-02-j-dest",
+    "sewer-watefall-froth-01-j-dest",
+    // sewl-water
+    "sewer-waterfall-02-l-dest",
+    "sewer-watefall-froth-01-l-dest",
+    // sewm-water
+    "sewer-water-01-m-dest",
+    "sewer-waterfall-01-m-dest",
+    "sewer-waterfall-02-m-dest",
+    "sewer-water-highlight-01-m-dest",
+    "sewer-water-wave-01-m-dest",
+    "sewer-water-still-01-m-dest",
+    "sewer-watefall-froth-01-m-dest",
+    // sewn-water
+    "sewer-waterfall-01-n-dest",
+    "sewer-waterfall-02-n-dest",
+    // "sewer-water-highlight-01-n-dest",
+    "sewer-water-wave-01-n-dest",
+    "sewer-water-still-01-n-dest",
+    // hanga-water
+    // "des-thermal-01-dest",
+    // desresc-warp
+    "sat-shield-dest",
+    // security
+    "security-env-dest",
+    "security-dot-dest",
+    // lgunnorm-water
+    "kg-target-c-forcefield-01-dest",
+    // templex-water
+    "temple-waterfall-dest",
+    // desertd-water
+    "des-waterfall-dest",
+    // towerb-water
+    "tow-energy-bridge-dest",
+    // factoryb-water
+    "hemi-gradient-dest",
+    "hemi-gradient-flames-dest",
+    // factoryc-alpha
+    "facc-convey-dest",
+    "facc-convey-02-dest",
+    // waspal-water
+    "waspala-water-dest",
+    "waspala-waterfall-dest",
+    // rubblea-water
+    "rub-water-dest",
+    // rubblea2-water
+    "rub-water-desta2",
+    // rubbleb-water
+    "rub-water-destb",
+    // rubblec-water
+    "rub-water-destc",
+    // nstb-quicksand
+    "nstb-quicksand-dest",
+    // ctyslumb-water
+    "ctyslumb-water-dest",
+    "ctyslumb-fountain-fall-dest",
+    // ctyslumc-water
+    "ctyslumc-water-dest",
+    "ctyslumc-fountain-fall-dest",
+    // mined-tfrag
+    "mined-pillar-side-dest",
+    "mined-pillar-top2side-dest",
+    "mined-pillar-top-dest",
+    // volcanoa-alpha
+    "vola-lava-01-dest",
+    "vola-lava-fall-dest",
+    // wasstada-alpha
+    "wstd-lava-base-dest",
+};
 
 }  // namespace
 

common/type_system/Type.h

@@ -28,8 +28,12 @@ struct MethodInfo {
   std::string name;
   TypeSpec type;
   std::string defined_in_type;
+  // TODO - this might be redundant, but preventing an unknown breaking change, clean it up later
+  std::string type_name;
   bool no_virtual = false;
+  // TODO - this is no longer used anymore...but was not removed from th4e struct?
   bool overrides_parent = false;
+  // TODO - this is no longer used anymore...but was not removed from th4e struct?
   bool only_overrides_docstring = false;
   std::optional<std::string> docstring;
   std::optional<std::string> overlay_name;

common/type_system/TypeSpec.h

@@ -124,10 +124,12 @@ class TypeSpec {
     ASSERT(m_arguments);
     return m_arguments->at(idx);
   }
+
   TypeSpec& get_arg(int idx) {
     ASSERT(m_arguments);
     return m_arguments->at(idx);
   }
+
   const TypeSpec& last_arg() const {
     ASSERT(m_arguments);
     ASSERT(!m_arguments->empty());

common/type_system/TypeSystem.cpp

@@ -548,6 +548,7 @@ MethodInfo TypeSystem::override_method(Type* type,
                            existing_info.name,
                            existing_info.type,
                            type->get_name(),
+                           type->get_name(),
                            existing_info.no_virtual,
                            false,
                            true,
@@ -607,6 +608,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
                              method_name,
                              ts,
                              type->get_name(),
+                             type->get_name(),
                              no_virtual,
                              true,
                              false,
@@ -645,6 +647,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
                                method_name,
                                ts,
                                type->get_name(),
+                               type->get_name(),
                                no_virtual,
                                false,
                                false,
@@ -678,8 +681,8 @@ MethodInfo TypeSystem::overlay_method(Type* type,
   }
 
   // use the existing ID.
-  return type->add_method({existing_info.id, method_name, ts, type->get_name(), false, true, false,
-                           docstring, std::make_optional(method_overlay_name)});
+  return type->add_method({existing_info.id, method_name, ts, type->get_name(), type->get_name(),
+                           false, true, false, docstring, std::make_optional(method_overlay_name)});
 }
 
 MethodInfo TypeSystem::define_method(const std::string& type_name,
@@ -709,8 +712,10 @@ MethodInfo TypeSystem::define_method(Type* type,
   // look up the method
   MethodInfo existing_info;
   bool got_existing = try_lookup_method(type, method_name, &existing_info);
-
   if (got_existing) {
+    // The lookup will return a parents method, but the type should be equal
+    // to the type in question (it's a child)
+    existing_info.type_name = type->get_name();
     // Update the docstring
     existing_info.docstring = docstring;
     int bad_arg_idx = -99;
@@ -758,7 +763,7 @@ MethodInfo TypeSystem::add_new_method(Type* type,
     return existing;
   } else {
     return type->add_new_method(
-        {0, "new", ts, type->get_name(), false, false, false, docstring, {}});
+        {0, "new", ts, type->get_name(), type->get_name(), false, false, false, docstring, {}});
   }
 }
 

common/type_system/defenum.cpp

@@ -58,10 +58,11 @@ EnumType* parse_defenum(const goos::Object& defenum,
   auto& enum_name_obj = car(iter);
   iter = cdr(iter);
   // check for docstring
+  std::optional<std::string> maybe_docstring;
   if (iter->is_pair() && car(iter).is_string()) {
-    // TODO - docstring - store and use docstring if coming from the compiler
     if (symbol_metadata) {
-      symbol_metadata->docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
+      maybe_docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
+      symbol_metadata->docstring = maybe_docstring.value();
     }
     iter = cdr(iter);
   }
@@ -157,6 +158,7 @@ EnumType* parse_defenum(const goos::Object& defenum,
   if (is_type("integer", base_type, ts)) {
     auto parent = ts->get_type_of_type<ValueType>(base_type.base_type());
     auto new_type = std::make_unique<EnumType>(parent, name, is_bitfield, entries);
+    new_type->m_metadata.docstring = maybe_docstring;
     new_type->set_runtime_type(parent->get_runtime_name());
     return dynamic_cast<EnumType*>(ts->add_type(name, std::move(new_type)));
   } else {

common/util/Assert.h

@@ -36,6 +36,9 @@
 
 #define ASSERT_NOT_REACHED_MSG(STR) \
   (void)((private_assert_failed("not reached", __FILE__, __LINE__, __PRETTY_FUNCTION__, STR), 0))
+
+#define ASSERT_EQ_IMM(EXPR, EXPECTED) \
+  ASSERT_MSG((EXPR) == (EXPECTED), fmt::format("result was {}, expected {}", (EXPR), (EXPECTED)))
 #else
 
 #define ASSERT(EX) ((void)0)

common/util/FileUtil.cpp

@@ -185,9 +185,11 @@ std::optional<std::string> try_get_jak_project_path() {
   return try_get_project_path_from_path(get_current_executable_path());
 }
 
-std::optional<fs::path> try_get_data_dir() {
+std::optional<fs::path> try_get_data_dir(bool skip_logs) {
   fs::path my_path = get_current_executable_path();
-  lg::info("Current executable directory - {}", my_path.string());
+  if (!skip_logs) {
+    lg::debug("Current executable directory - {}", my_path.string());
+  }
   auto data_dir = my_path.parent_path() / "data";
   if (fs::exists(data_dir) && fs::is_directory(data_dir)) {
     return std::make_optional(data_dir);
@@ -196,7 +198,7 @@ std::optional<fs::path> try_get_data_dir() {
   }
 }
 
-bool setup_project_path(std::optional<fs::path> project_path_override) {
+bool setup_project_path(std::optional<fs::path> project_path_override, bool skip_logs) {
   if (g_file_path_info.initialized) {
     return true;
   }
@@ -204,16 +206,20 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
   if (project_path_override) {
     g_file_path_info.path_to_data_folder = fs::absolute(project_path_override.value());
     g_file_path_info.initialized = true;
-    lg::info("Using explicitly set project path: {}",
-             g_file_path_info.path_to_data_folder.string());
+    if (!skip_logs) {
+      lg::debug("Using explicitly set project path: {}",
+                g_file_path_info.path_to_data_folder.string());
+    }
     return true;
   }
 
-  auto data_path = try_get_data_dir();
+  auto data_path = try_get_data_dir(skip_logs);
   if (data_path) {
     g_file_path_info.path_to_data_folder = *data_path;
     g_file_path_info.initialized = true;
-    lg::info("Using data path: {}", data_path->string());
+    if (!skip_logs) {
+      lg::debug("Using data path: {}", data_path->string());
+    }
     return true;
   }
 
@@ -221,7 +227,9 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
   if (development_repo_path) {
     g_file_path_info.path_to_data_folder = *development_repo_path;
     g_file_path_info.initialized = true;
-    lg::info("Using development repo path: {}", *development_repo_path);
+    if (!skip_logs) {
+      lg::debug("Using development repo path: {}", *development_repo_path);
+    }
     return true;
   }
 

common/util/FileUtil.h

@@ -43,7 +43,7 @@ bool create_dir_if_needed_for_file(const std::string& path);
 bool create_dir_if_needed_for_file(const fs::path& path);
 std::string get_current_executable_path();
 std::optional<std::string> try_get_project_path_from_path(const std::string& path);
-bool setup_project_path(std::optional<fs::path> project_path_override);
+bool setup_project_path(std::optional<fs::path> project_path_override, bool skip_logs = false);
 void override_user_config_dir(fs::path user_config_dir_override,
                               bool use_overridden_config_dir_for_saves);
 std::string get_file_path(const std::vector<std::string>& path);

common/util/FontUtils.cpp

@@ -985,6 +985,8 @@ static std::vector<ReplaceInfo> s_replace_info_jak2 = {
     {"o~Y~-26H~-4V'~Z", "ó"},
     {"U~Y~-24H~-3V'~Z", "Ú"},
     {"u~Y~-24H~-3V'~Z", "ú"},
+    {"Z~Y~-24H~-3V'~Z", "Ź"},
+    {"z~Y~-24H~-3V'~Z", "ź"},
 
     // circumflex
     {"A~Y~-20H~-4V^~Z", "Â"},
@@ -1015,6 +1017,7 @@ static std::vector<ReplaceInfo> s_replace_info_jak2 = {
     {"a~Y~-25H~-5V¨~Z", "ä"},
     {"E~Y~-20H~-5V¨~Z", "Ë"},
     {"I~Y~-19H~-5V¨~Z", "Ï"},
+    {"i~Y~-26H~-4V¨~Z", "ï"},
     {"O~Y~-26H~-8V¨~Z", "Ö"},
     {"o~Y~-26H~-4V¨~Z", "ö"},
     {"U~Y~-25H~-8V¨~Z", "Ü"},

common/util/compress.cpp

@@ -46,4 +46,17 @@ std::vector<u8> decompress_zstd(const void* data, size_t size) {
   ASSERT(decomp_size == decompressed_size);
   return result;
 }
+
+std::vector<u8> compress_zstd_no_header(const void* data, size_t size) {
+  size_t max_compressed = ZSTD_compressBound(size);
+  std::vector<u8> result(max_compressed);
+
+  size_t compressed_size = ZSTD_compress(result.data(), max_compressed, data, size, 1);
+  if (ZSTD_isError(compressed_size)) {
+    ASSERT_MSG(false, fmt::format("ZSTD error: {}", ZSTD_getErrorName(compressed_size)));
+  }
+
+  result.resize(compressed_size);
+  return result;
+}
 }  // namespace compression

common/util/compress.h

@@ -8,4 +8,5 @@ namespace compression {
 // compress and decompress data with zstd
 std::vector<u8> compress_zstd(const void* data, size_t size);
 std::vector<u8> decompress_zstd(const void* data, size_t size);
-}  // namespace compression
+std::vector<u8> compress_zstd_no_header(const void* data, size_t size);
+}  // namespace compression

common/util/fnv.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <string>
+
+#include "common/common_types.h"
+
+inline u64 fnv64(const void* data, u64 len) {
+  u64 ret = 0xcbf29ce484222325;
+  const auto* ptr = (const u8*)data;
+  for (u64 i = 0; i < len; i++) {
+    ret = 1099511628211 * (((u64)*ptr) ^ ret);
+    ptr++;
+  }
+  return ret;
+}
+
+inline u64 fnv64(const std::string& str) {
+  return fnv64(str.data(), str.length());
+}

common/util/gltf_util.cpp

@@ -0,0 +1,467 @@
+#include "gltf_util.h"
+
+#include "common/log/log.h"
+
+namespace gltf_util {
+
+/*!
+ * Convert a GLTF position buffer or similar to std::vector<Vec3f>
+ */
+std::vector<math::Vector3f> extract_vec3f(const u8* data, u32 count, u32 stride) {
+  std::vector<math::Vector3f> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    memcpy(&result.emplace_back(), data, sizeof(math::Vector3f));
+    data += stride;
+  }
+  return result;
+}
+
+std::vector<math::Vector2f> extract_vec2f(const u8* data, u32 count, u32 stride) {
+  std::vector<math::Vector2f> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    memcpy(&result.emplace_back(), data, sizeof(math::Vector2f));
+    data += stride;
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF color buffer (float format) to u8 colors.
+ */
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_float(const u8* data,
+                                                               u32 count,
+                                                               u32 stride) {
+  std::vector<math::Vector<u8, 4>> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    math::Vector<float, 4> temp;
+    memcpy(&temp, data, sizeof(math::Vector<float, 4>));
+    data += stride;
+    result.emplace_back(temp.x() * 255, temp.y() * 255, temp.z() * 255, temp.w() * 255);
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF color buffer (u16 format) to u8 colors.
+ */
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_u16(const u8* data,
+                                                             u32 count,
+                                                             u32 stride) {
+  std::vector<math::Vector<u8, 4>> result;
+  result.reserve(count);
+  for (u32 i = 0; i < count; i++) {
+    math::Vector<u16, 4> temp;
+    memcpy(&temp, data, sizeof(math::Vector<u16, 4>));
+    data += stride;
+    result.emplace_back(temp.x() >> 8, temp.y() >> 8, temp.z() >> 8, temp.w() >> 8);
+  }
+  return result;
+}
+
+/*!
+ * Convert a GLTF index buffer
+ */
+std::vector<u32> gltf_index_buffer(const tinygltf::Model& model,
+                                   int indices_idx,
+                                   u32 index_offset) {
+  const auto& indices_accessor = model.accessors[indices_idx];
+  const auto& buffer_view = model.bufferViews[indices_accessor.bufferView];
+  const auto& buffer = model.buffers[buffer_view.buffer];
+  const auto data_ptr = buffer.data.data() + buffer_view.byteOffset + indices_accessor.byteOffset;
+  const auto stride = indices_accessor.ByteStride(buffer_view);
+  const auto count = indices_accessor.count;
+
+  switch (indices_accessor.componentType) {
+    case TINYGLTF_COMPONENT_TYPE_BYTE:
+      return index_list_to_u32<s8>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE:
+      return index_list_to_u32<u8>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_SHORT:
+      return index_list_to_u32<s16>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT:
+      return index_list_to_u32<u16>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_INT:
+      return index_list_to_u32<s32>(data_ptr, count, index_offset, stride);
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT:
+      return index_list_to_u32<u32>(data_ptr, count, index_offset, stride);
+    default:
+      ASSERT_MSG(false, "unsupported component type");
+  }
+}
+
+/*!
+ * Extract positions, colors, and normals from a mesh.
+ */
+ExtractedVertices gltf_vertices(const tinygltf::Model& model,
+                                const std::map<std::string, int>& attributes,
+                                const math::Matrix4f& w_T_local,
+                                bool get_colors,
+                                bool get_normals,
+                                const std::string& debug_name) {
+  std::vector<tfrag3::PreloadedVertex> result;
+  std::vector<math::Vector<u8, 4>> vtx_colors;
+
+  {
+    const auto& position_attrib = attributes.find("POSITION");
+    ASSERT_MSG(position_attrib != attributes.end(), "Did not find position attribute.");
+
+    const auto attrib_accessor = model.accessors[position_attrib->second];
+    const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+    const auto& buffer = model.buffers[buffer_view.buffer];
+    const auto data_ptr = buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+    const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+    const auto count = attrib_accessor.count;
+
+    ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC3, "POSITION wasn't vec3");
+    ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+               "POSITION wasn't float");
+    // for (auto& attrib : attributes) {
+    // lg::print("attrib: {}\n", attrib.first);
+    //}
+    auto mesh_verts = extract_vec3f(data_ptr, count, byte_stride);
+    result.reserve(mesh_verts.size());
+    for (auto& vert : mesh_verts) {
+      auto& new_vert = result.emplace_back();
+      math::Vector4f v_in(vert.x(), vert.y(), vert.z(), 1);
+      math::Vector4f v_w = w_T_local * v_in;
+      new_vert.x = v_w.x() * 4096;
+      new_vert.y = v_w.y() * 4096;
+      new_vert.z = v_w.z() * 4096;
+    }
+  }
+
+  if (get_colors) {
+    const auto& color_attrib = attributes.find("COLOR_0");
+    if (color_attrib == attributes.end()) {
+      lg::error("Mesh {} didn't have any colors, using white", debug_name);
+      for (size_t i = 0; i < result.size(); i++) {
+        vtx_colors.emplace_back(0x80, 0x80, 0x80, 0xff);
+      }
+    } else {
+      const auto attrib_accessor = model.accessors[color_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC4, "COLOR_0 wasn't vec4");
+      std::vector<math::Vector<u8, 4>> colors;
+      switch (attrib_accessor.componentType) {
+        case TINYGLTF_COMPONENT_TYPE_FLOAT:
+          colors = extract_color_from_vec4_float(data_ptr, count, byte_stride);
+          break;
+        case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT:
+          colors = extract_color_from_vec4_u16(data_ptr, count, byte_stride);
+          break;
+        default:
+          lg::die("Unknown component type for COLOR_0: {}", attrib_accessor.componentType);
+      }
+      vtx_colors.insert(vtx_colors.end(), colors.begin(), colors.end());
+    }
+  }
+
+  bool got_texture = false;
+  {
+    const auto& texcoord_attrib = attributes.find("TEXCOORD_0");
+    if (texcoord_attrib != attributes.end()) {
+      const auto attrib_accessor = model.accessors[texcoord_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC2, "TEXCOORD wasn't vec2");
+      ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+                 "TEXCOORD wasn't float");
+      auto mesh_verts = extract_vec2f(data_ptr, count, byte_stride);
+      ASSERT(mesh_verts.size() == result.size());
+      got_texture = true;
+      for (size_t i = 0; i < mesh_verts.size(); i++) {
+        result[i].s = mesh_verts[i].x();
+        result[i].t = mesh_verts[i].y();
+      }
+    } else {
+      if (!get_normals) {
+        // don't warn if we're just getting collision
+        lg::warn("No texcoord attribute for mesh: {}", debug_name);
+      }
+    }
+  }
+
+  std::vector<math::Vector3f> normals;
+  if (get_normals) {
+    const auto& normal_attrib = attributes.find("NORMAL");
+    if (normal_attrib != attributes.end()) {
+      const auto attrib_accessor = model.accessors[normal_attrib->second];
+      const auto& buffer_view = model.bufferViews[attrib_accessor.bufferView];
+      const auto& buffer = model.buffers[buffer_view.buffer];
+      const auto data_ptr =
+          buffer.data.data() + buffer_view.byteOffset + attrib_accessor.byteOffset;
+      const auto byte_stride = attrib_accessor.ByteStride(buffer_view);
+      const auto count = attrib_accessor.count;
+
+      ASSERT_MSG(attrib_accessor.type == TINYGLTF_TYPE_VEC3, "NORMAL wasn't vec3");
+      ASSERT_MSG(attrib_accessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT,
+                 "NORMAL wasn't float");
+      normals = extract_vec3f(data_ptr, count, byte_stride);
+      for (auto& nrm : normals) {
+        math::Vector4f nrm4(nrm.x(), nrm.y(), nrm.z(), 0.f);
+        nrm = (w_T_local * nrm4).xyz();
+      }
+      ASSERT(normals.size() == result.size());
+    } else {
+      lg::error("No NORMAL attribute for mesh: {}", debug_name);
+    }
+  }
+
+  for (auto& v : result) {
+    v.color_index = 0;
+    if (!got_texture) {
+      v.s = 0;
+      v.t = 0;
+    }
+  }
+  // TODO: other properties
+  return {result, vtx_colors, normals};
+}
+
+DrawMode make_default_draw_mode() {
+  DrawMode mode;
+  mode.set_depth_write_enable(true);
+  mode.set_depth_test(GsTest::ZTest::GEQUAL);
+  mode.set_alpha_blend(DrawMode::AlphaBlend::DISABLED);
+  mode.set_aref(0);
+  mode.set_alpha_fail(GsTest::AlphaFail::KEEP);
+  mode.set_clamp_s_enable(false);
+  mode.set_clamp_t_enable(false);
+  mode.disable_filt();  // for checkerboard...
+  mode.enable_tcc();    // ?
+  mode.disable_at();
+  mode.enable_zt();
+  mode.disable_ab();
+  mode.disable_decal();
+  mode.enable_fog();
+  return mode;
+}
+
+int texture_pool_debug_checker(TexturePool* pool) {
+  const auto& existing = pool->textures_by_name.find("DEBUG_CHECKERBOARD");
+  if (existing == pool->textures_by_name.end()) {
+    size_t idx = pool->textures_by_idx.size();
+    pool->textures_by_name["DEBUG_CHECKERBOARD"] = idx;
+    auto& tex = pool->textures_by_idx.emplace_back();
+    tex.w = 16;
+    tex.h = 16;
+    tex.debug_name = "DEBUG_CHECKERBOARD";
+    tex.debug_tpage_name = "DEBUG";
+    tex.load_to_pool = false;
+    tex.combo_id = 0;  // doesn't matter, not a pool tex
+    tex.data.resize(16 * 16);
+    u32 c0 = 0xa0303030;
+    u32 c1 = 0xa0e0e0e0;
+    for (int i = 0; i < 16; i++) {
+      for (int j = 0; j < 16; j++) {
+        tex.data[i * 16 + j] = (((i / 4) & 1) ^ ((j / 4) & 1)) ? c1 : c0;
+      }
+    }
+    return idx;
+  } else {
+    return existing->second;
+  }
+}
+
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
+  const auto& existing = pool->textures_by_name.find(tex.name);
+  if (existing != pool->textures_by_name.end()) {
+    lg::info("Reusing image: {}", tex.name);
+    return existing->second;
+  } else {
+    lg::info("adding new texture: {}, size {} kB", tex.name, tex.width * tex.height * 4 / 1024);
+  }
+
+  ASSERT(tex.bits == 8);
+  ASSERT(tex.component == 4);
+  ASSERT(tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+
+  size_t idx = pool->textures_by_idx.size();
+  pool->textures_by_name[tex.name] = idx;
+  auto& tt = pool->textures_by_idx.emplace_back();
+  tt.w = tex.width;
+  tt.h = tex.height;
+  tt.debug_name = tex.name;
+  tt.debug_tpage_name = "custom-level";
+  tt.load_to_pool = false;
+  tt.combo_id = 0;  // doesn't matter, not a pool tex
+  tt.data.resize(tt.w * tt.h);
+  ASSERT(tex.image.size() >= tt.data.size());
+  memcpy(tt.data.data(), tex.image.data(), tt.data.size() * 4);
+  return idx;
+}
+
+math::Matrix4f affine_translation(const math::Vector3f& translation) {
+  math::Matrix4f result = math::Matrix4f::identity();
+  result(0, 3) = translation[0];
+  result(1, 3) = translation[1];
+  result(2, 3) = translation[2];
+  result(3, 3) = 1;
+  return result;
+}
+
+math::Matrix4f affine_scale(const math::Vector3f& scale) {
+  math::Matrix4f result = math::Matrix4f::zero();
+  result(0, 0) = scale[0];
+  result(1, 1) = scale[1];
+  result(2, 2) = scale[2];
+  result(3, 3) = 1;
+  return result;
+}
+
+math::Matrix4f affine_rot_qxyzw(const math::Vector4f& quat) {
+  math::Matrix4f result = math::Matrix4f::zero();
+  result(3, 3) = 1;
+  result(0, 0) = 1.0 - 2.0 * (quat.y() * quat.y() + quat.z() * quat.z());
+  result(0, 1) = 2.0 * (quat.x() * quat.y() - quat.z() * quat.w());
+  result(0, 2) = 2.0 * (quat.x() * quat.z() + quat.y() * quat.w());
+  result(1, 0) = 2.0 * (quat.x() * quat.y() + quat.z() * quat.w());
+  result(1, 1) = 1.0 - 2.0 * (quat.x() * quat.x() + quat.z() * quat.z());
+  result(1, 2) = 2.0 * (quat.y() * quat.z() - quat.x() * quat.w());
+  result(2, 0) = 2.0 * (quat.x() * quat.z() - quat.y() * quat.w());
+  result(2, 1) = 2.0 * (quat.y() * quat.z() + quat.x() * quat.w());
+  result(2, 2) = 1.0 - 2.0 * (quat.x() * quat.x() + quat.y() * quat.y());
+  return result;
+}
+
+math::Vector3f vector3f_from_gltf(const std::vector<double>& in) {
+  ASSERT(in.size() == 3);
+  return math::Vector3f{in[0], in[1], in[2]};
+}
+
+math::Vector4f vector4f_from_gltf(const std::vector<double>& in) {
+  ASSERT(in.size() == 4);
+  return math::Vector4f{in[0], in[1], in[2], in[3]};
+}
+
+math::Matrix4f matrix_from_node(const tinygltf::Node& node) {
+  if (!node.matrix.empty()) {
+    math::Matrix4f result;
+    for (int i = 0; i < 16; i++) {
+      result.data()[i] = node.matrix[i];
+    }
+    return result;
+  } else {
+    // from trs
+    math::Matrix4f t, r, s;
+    if (!node.translation.empty()) {
+      t = affine_translation(vector3f_from_gltf(node.translation));
+    } else {
+      t = math::Matrix4f::identity();
+    }
+
+    if (!node.rotation.empty()) {
+      r = affine_rot_qxyzw(vector4f_from_gltf(node.rotation));
+    } else {
+      r = math::Matrix4f::identity();
+    }
+
+    if (!node.scale.empty()) {
+      s = affine_scale(vector3f_from_gltf(node.scale));
+    } else {
+      s = math::Matrix4f::identity();
+    }
+
+    return t * r * s;
+  }
+}
+
+/*!
+ * Recursively walk the tree of nodes, flatten, and compute w_T_node for each.
+ */
+void node_find_helper(const tinygltf::Model& model,
+                      const math::Matrix4f& w_T_parent,
+                      int node_idx,
+                      std::vector<NodeWithTransform>* out) {
+  const auto& node = model.nodes.at(node_idx);
+  math::Matrix4f w_T_node = w_T_parent * matrix_from_node(node);
+  out->push_back({node_idx, w_T_node});
+  for (auto& child : node.children) {
+    node_find_helper(model, w_T_node, child, out);
+  }
+}
+
+std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Model& model) {
+  std::vector<NodeWithTransform> out;
+  for (auto& scene : model.scenes) {
+    for (auto& nidx : scene.nodes) {
+      math::Matrix4f identity = math::Matrix4f::identity();
+      node_find_helper(model, identity, nidx, &out);
+    }
+  }
+  return out;
+}
+
+void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
+                    std::vector<tfrag3::PreloadedVertex>& vertices_out,
+                    std::vector<u32>& old_to_new_out) {
+  ASSERT(vertices_out.empty());
+  ASSERT(old_to_new_out.empty());
+  old_to_new_out.resize(vertices_in.size(), -1);
+
+  std::unordered_map<tfrag3::PreloadedVertex, u32, tfrag3::PreloadedVertex::hash> vtx_to_new;
+
+  for (size_t in_idx = 0; in_idx < vertices_in.size(); in_idx++) {
+    auto& vtx = vertices_in[in_idx];
+    const auto& lookup = vtx_to_new.find(vtx);
+    if (lookup == vtx_to_new.end()) {
+      // first time seeing this one
+      size_t new_idx = vertices_out.size();
+      vertices_out.push_back(vtx);
+      old_to_new_out[in_idx] = new_idx;
+      vtx_to_new[vtx] = new_idx;
+    } else {
+      old_to_new_out[in_idx] = lookup->second;
+    }
+  }
+}
+
+DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler) {
+  DrawMode mode = make_default_draw_mode();
+  if (sampler.magFilter == TINYGLTF_TEXTURE_FILTER_NEAREST) {
+    ASSERT(sampler.minFilter == TINYGLTF_TEXTURE_FILTER_NEAREST);
+    mode.set_filt_enable(false);
+  } else {
+    ASSERT(sampler.minFilter != TINYGLTF_TEXTURE_FILTER_NEAREST);
+    mode.set_filt_enable(true);
+  }
+
+  switch (sampler.wrapS) {
+    case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
+      mode.set_clamp_s_enable(true);
+      break;
+    case TINYGLTF_TEXTURE_WRAP_REPEAT:
+      mode.set_clamp_s_enable(false);
+      break;
+    default:
+      ASSERT(false);
+  }
+
+  switch (sampler.wrapT) {
+    case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
+      mode.set_clamp_t_enable(true);
+      break;
+    case TINYGLTF_TEXTURE_WRAP_REPEAT:
+      mode.set_clamp_t_enable(false);
+      break;
+    default:
+      ASSERT(false);
+  }
+
+  return mode;
+}
+}  // namespace gltf_util

common/util/gltf_util.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/common_types.h"
+#include "common/custom_data/Tfrag3Data.h"
+#include "common/math/Vector.h"
+
+#include "third-party/tiny_gltf/tiny_gltf.h"
+
+namespace gltf_util {
+
+/*!
+ * Convert a GLTF index buffer to std::vector<u32>
+ */
+template <typename T>
+std::vector<u32> index_list_to_u32(const u8* data, u32 num_verts, u32 offset, u32 stride) {
+  std::vector<u32> result;
+  result.reserve(num_verts);
+  for (u32 i = 0; i < num_verts; i++) {
+    T val;
+    memcpy(&val, data, sizeof(T));
+    result.push_back(offset + val);
+    data += stride;
+  }
+  return result;
+}
+
+std::vector<math::Vector3f> extract_vec3f(const u8* data, u32 count, u32 stride);
+std::vector<math::Vector2f> extract_vec2f(const u8* data, u32 count, u32 stride);
+std::vector<math::Vector<u8, 4>> extract_color_from_vec4_u16(const u8* data, u32 count, u32 stride);
+std::vector<u32> gltf_index_buffer(const tinygltf::Model& model, int indices_idx, u32 index_offset);
+
+struct ExtractedVertices {
+  std::vector<tfrag3::PreloadedVertex> vtx;
+  std::vector<math::Vector<u8, 4>> vtx_colors;
+  std::vector<math::Vector3f> normals;
+};
+
+ExtractedVertices gltf_vertices(const tinygltf::Model& model,
+                                const std::map<std::string, int>& attributes,
+                                const math::Matrix4f& w_T_local,
+                                bool get_colors,
+                                bool get_normals,
+                                const std::string& debug_name);
+DrawMode make_default_draw_mode();
+
+struct TexturePool {
+  std::unordered_map<std::string, int> textures_by_name;
+  std::vector<tfrag3::Texture> textures_by_idx;
+};
+
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex);
+int texture_pool_debug_checker(TexturePool* pool);
+
+struct NodeWithTransform {
+  int node_idx;
+  math::Matrix4f w_T_node;
+};
+
+void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
+                    std::vector<tfrag3::PreloadedVertex>& vertices_out,
+                    std::vector<u32>& old_to_new_out);
+
+std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Model& model);
+
+DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler);
+
+}  // namespace gltf_util

common/util/json_util.h

@@ -27,11 +27,11 @@ Range<int> parse_json_optional_integer_range(const nlohmann::json& json);
   }
 
 template <typename T>
-void json_get_optional(const nlohmann::json& json,
+void json_get_optional(const nlohmann::json& j,
                        const std::string& key,
                        std::optional<T>& optionalValue) {
-  if (json.contains(key) && !json[key].is_null()) {
-    optionalValue = json[key].get<T>();
+  if (j.contains(key) && !j[key].is_null()) {
+    optionalValue = j[key].get<T>();
   }
 }
 

common/util/os.cpp

@@ -3,20 +3,29 @@
 #include "common/common_types.h"
 #include "common/log/log.h"
 
-#ifdef __linux__
-
+#ifdef _WIN32
+// clang-format off
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <psapi.h>
+// clang-format on
+size_t get_peak_rss() {
+  HANDLE hProcess = GetCurrentProcess();
+  PROCESS_MEMORY_COUNTERS pmc;
+  if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) {
+    return pmc.PeakWorkingSetSize;
+  } else {
+    return 0;
+  }
+}
+#else
 #include <sys/resource.h>
-
 size_t get_peak_rss() {
   rusage x;
   getrusage(RUSAGE_SELF, &x);
   return x.ru_maxrss * 1024;
 }
-
-#else
-size_t get_peak_rss() {
-  return 0;
-}
 #endif
 
 #ifdef _WIN32

common/util/os.h

@@ -3,7 +3,6 @@
 #include <cstddef>
 #include <string>
 
-// Note: these are not implemented on windows and will return zero.
 size_t get_peak_rss();
 void setup_cpu_info();
 

common/util/trie_with_duplicates.h

@@ -7,9 +7,6 @@
 
 // A normal Trie does not allow for duplicate keys, however this one does
 // It allows for insertion and removal
-//
-// Note, keys assume only basic ASCII, which is _fine_ as OpenGOAL itself has this
-// limitation as well.
 template <typename T>
 class TrieWithDuplicates {
  private:

custom_assets/README.md

@@ -5,13 +5,13 @@ Disclaimer: custom levels are still in development and are missing most features
 The first three steps are already done for "test zone", so this can be used as a starting point.
 
 # 1: File Setup
-To create a custom level, copy the layout of `custom_levels/test-zone`. See `test-zone.jsonc` for information on how to name things. The `.gd` file also contains the level name.
+To create a custom level, copy the layout of `custom_assets/jak1/levels/test-zone`. See `test-zone.jsonc` for information on how to name things. The `.gd` file also contains the level name.
 
 # 2: Modify the engine
-Modify `goal_src/engine/level/level-info.gc` to add level info for each custom level. There is level info for `test-zone` at the bottom that can be used as an example.
+Modify `goal_src/jak1/engine/level/level-info.gc` to add level info for each custom level. There is level info for `test-zone` at the bottom that can be used as an example.
 
 # 3: Modify the build system
-Modify `goal_src/game.gp` and add a custom level target:
+Modify `goal_src/jak1/game.gp` and add a custom level target:
 ```lisp
 (build-custom-level "test-zone")
 ;; the DGO file
@@ -21,7 +21,7 @@ Modify `goal_src/game.gp` and add a custom level target:
 # 4: Export the GLTF file from blender.
 For now, all meshes are displayed and treated as ground collision. This causes buggy collision because walls shouldn't use "floor" mode.
 
-Blender will create a `.glb` file, which must have the name specified in the `.jsonc` file and should be located in `custom_level/your_level`
+Blender will create a `.glb` file, which must have the name specified in the `.jsonc` file and should be located in `custom_assets/jak1/levels/your_level`
 
 # 5: Rebuild the game
 Any time the `.glb` file is changed, you must rebuild the game. Launch the compiler (`goalc`) and run `(mi)` to rebuild everything. It's recommended to leave the compiler open - it will remember files that haven't changed and skip rebuilding them.

custom_assets/jak1/levels/test-zone/test-zone.jsonc

@@ -10,7 +10,7 @@
   // Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
   // and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
   // only have 1.
-  "gltf_file": "custom_levels/jak1/test-zone/test-zone2.glb",
+  "gltf_file": "custom_assets/jak1/levels/test-zone/test-zone2.glb",
 
   // automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
   "automatic_wall_detection": true,
@@ -55,20 +55,22 @@
   // adds a 'symbol' tag (using the 'type' and 'string' lump types works the same way):
   // "symbol-list": ["symbol", "sym-1", "sym-2"]
 
-   // The base actor id for your custom level. If you have multiple levels this should be unique!
-   "base_id": 100,
+  // The base actor id for your custom level. If you have multiple levels, this should be unique!
+  "base_id": 100,
 
   // All art groups you want to use in your custom level. Will add their models and corresponding textures to the FR3 file.
   // Note: You will still have to add them to your level's .gd file.
-  // Removed so that the release builds don't have to double-decompile the game
   // "art_groups": ["plat-ag"],
-  "art_groups": [],
 
-   // Any textures you want to include in your custom level. This is mainly useful for things such as the zoomer HUD,
-   // which is not in the common level files and has no art group associated with it.
-   // To get a list of all the textures, you can extract all of the game's textures
-   // by setting "save_texture_pngs" to true in the decompiler config.
-   "textures": [
+  // If you have any custom models in the "custom_assets/jak1/models/custom_levels" folder that you want to use in your level, add them to this list.
+  // Note: Like with art groups, these should also be added to your level's .gd file.
+  "custom_models": ["test-actor"],
+
+  // Any textures you want to include in your custom level. This is mainly useful for things such as the zoomer HUD,
+  // which is not in the common level files and has no art group associated with it.
+  // To get a list of all the textures, you can extract all of the game's textures
+  // by setting "save_texture_pngs" to true in the decompiler config.
+  "textures": [
     // all textures required for the zoomer HUD
     // "zoomerhud",
     // "zoomerhud-dial",
@@ -130,6 +132,16 @@
       "lump": {
         "name": "test-eco"
       }
+    },
+    {
+      "trans": [-5.41, 3.5, 28.42], // translation
+      "etype": "test-actor",  // actor type
+      "game_task": 0, // associated game task (for powercells, etc)
+      "quat": [0, 0, 0, 1], // quaternion
+      "bsphere": [-7.41, 3.5, 28.42, 10], // bounding sphere
+      "lump": {
+        "name": "test-actor"
+      }
     }
     // {
     //   "trans": [-7.41, 3.5, 28.42], // translation

custom_assets/jak1/levels/test-zone/testzone.gd

@@ -4,6 +4,8 @@
 ;; the actual file name still needs to be 8.3
 ("TSZ.DGO"
  ("static-screen.o"
-  "test-zone.go"
+  "test-zone-obs.o"
   "plat-ag.go"
+  "test-actor-ag.go"
+  "test-zone.go"
  ))

custom_assets/jak2/levels/test-zone/test-zone.jsonc

@@ -10,7 +10,7 @@
   // Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
   // and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
   // only have 1.
-  "gltf_file": "custom_levels/jak2/test-zone/test-zone2.glb",
+  "gltf_file": "custom_assets/jak2/levels/test-zone/test-zone2.glb",
 
   // automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
   "automatic_wall_detection": true,

custom_assets/jak3/levels/test-zone/test-zone.jsonc

@@ -0,0 +1,90 @@
+{
+  // The "in-game" name of the level. Should be lower case, with dashes (GOAL symbol name)
+  // the name of this file, and the folder this file is in must have the same name.
+  "long_name": "test-zone",
+  // The file name, should be upper case and 8 characters or less.
+  "iso_name": "TESTZONE",
+  // The nickname, should be exactly 3 characters
+  "nickname": "tsz", // 3 char name, all lowercase
+  // Background mesh file.
+  // Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
+  // and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
+  // only have 1.
+  "gltf_file": "custom_assets/jak3/levels/test-zone/test-zone2.glb",
+
+  // automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
+  "automatic_wall_detection": true,
+  "automatic_wall_angle": 45.0,
+
+  // if your mesh has triangles with incorrect orientation, set this to make all collision mesh triangles double sided
+  // this makes collision 2x slower and bigger, so only use if really needed
+  "double_sided_collide": false,
+
+  // available res-lump tag types:
+  // integer types: int32, uint32, enum-int32, enum-uint32
+  // float types: float, meters (1 meter = 4096.0 units), degrees (65536.0 = 360°)
+  // vector types: vector (normal floats), vector4m (meters), vector3m (meters with w set to 1.0), vector-vol (normal floats with w in meters), movie-pos (meters with w in degrees)
+  // special types: symbol, type, string, eco-info, cell-info, buzzer-info, water-height
+  //
+  // examples:
+  //
+  // adds a float tag 'spring-height' with value of 200 meters (1 meter = 4096.0 units):
+  // "spring-height": ["meters", 200.0]
+  //
+  // adds a degrees tag 'rotoffset':
+  // "rotoffset": ["degrees", -45.0]
+  //
+  // adds a movie-pos tag:
+  // "movie-pos": ["movie-pos", [100.22, -25.3, 99.5, 180.0]]
+  //
+  // adds an enum tag 'options':
+  // "options": ["enum-int32", "(fact-options large)"]
+  //
+  // adds a water-height tag:
+  // "water-height": ["water-height", 25.0, 0.5, 2.0, "(water-flags can-swim can-wade)"]
+  //
+  // adds an eco-info tag:
+  // "eco-info": ["eco-info", "(pickup-type health)", 2]
+  //
+  // adds a 'type' tag (using the "symbol" and "string" lump types works the same way):
+  // "spawn-types": ["type", "spyder", "juicer"]
+
+   // The base actor id for your custom level. If you have multiple levels, this should be unique!
+   "base_id": 100,
+
+  // All art groups you want to use in your custom level. Will add their models and corresponding textures to the FR3 file.
+  // "art_groups": [],
+
+   // Any textures you want to include in your custom level.
+   // This is mainly useful for textures which are not in the common level files and have no art group associated with them.
+   // To get a list of all the textures, you can extract all of the game's textures
+   // by setting "save_texture_pngs" to true in the decompiler config.
+   // "textures": [],
+
+  "actors" : [
+    {
+      "trans": [-15.2818, 15.2461, 17.1360], // translation
+      "etype": "crate",  // actor type
+      "game_task": "(game-task none)", // associated game task (for powercells, etc)
+      "kill_mask": 0,
+      "quat": [0, 0, 0, 1], // quaternion
+      "bsphere": [-15.2818, 15.2461, 17.1360, 10], // bounding sphere
+      "lump": {
+        "name": "test-crate",
+        "eco-info": ["eco-info", "(pickup-type gem)", 1]
+      }
+    },
+
+    {
+      "trans": [-5.4630, 17.4553, 1.6169], // translation
+      "etype": "eco-yellow",  // actor type
+      "game_task": "(game-task none)", // associated game task (for powercells, etc)
+      "kill_mask": 0,
+      "quat": [0, 0, 0, 1], // quaternion
+      "bsphere": [-5.4630, 17.4553, 1.6169, 10], // bounding sphere
+      "lump": {
+        "name": "test-eco"
+      }
+    }
+  ]
+}

custom_assets/jak3/levels/test-zone/testzone.gd

@@ -0,0 +1,8 @@
+;; DGO definition file for Awful Village level
+;; We use the convention of having a longer DGO name for levels without precomputed visibility.
+
+;; the actual file name still needs to be 8.3
+("TSZ.DGO"
+ ("test-zone.go"
+  )
+ )

decompiler/CMakeLists.txt

@@ -58,6 +58,7 @@ add_library(
         level_extractor/extract_actors.cpp
         level_extractor/extract_collide_frags.cpp
         level_extractor/extract_common.cpp
+        level_extractor/extract_hfrag.cpp
         level_extractor/extract_joint_group.cpp
         level_extractor/extract_level.cpp
         level_extractor/extract_merc.cpp
@@ -67,6 +68,7 @@ add_library(
         level_extractor/fr3_to_gltf.cpp
         level_extractor/MercData.cpp
         level_extractor/tfrag_tie_fixup.cpp
+        level_extractor/merc_replacement.cpp
 
         ObjectFile/LinkedObjectFile.cpp
         ObjectFile/LinkedObjectFileCreation.cpp
@@ -88,7 +90,8 @@ add_library(
         VuDisasm/VuDisassembler.cpp
         VuDisasm/VuInstruction.cpp
 
-        config.cpp)
+        config.cpp
+        decompilation_process.cpp)
 
 target_link_libraries(decomp
         lzokay

decompiler/ObjectFile/LinkedObjectFile.h

@@ -25,7 +25,7 @@ namespace decompiler {
  */
 class LinkedObjectFile {
  public:
-  LinkedObjectFile(GameVersion version) : version(version){};
+  LinkedObjectFile(GameVersion _version) : version(_version) {}
   void set_segment_count(int n_segs);
   void push_back_word_to_segment(uint32_t word, int segment);
   int get_label_id_for(int seg, int offset);

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -5,6 +5,7 @@
 
 #include "ObjectFileDB.h"
 
+#include "common/formatter/formatter.h"
 #include "common/goos/PrettyPrinter.h"
 #include "common/link_types.h"
 #include "common/log/log.h"
@@ -872,9 +873,22 @@ void ObjectFileDB::ir2_write_results(const fs::path& output_dir,
     auto file_name = output_dir / (obj.to_unique_name() + "_ir2.asm");
     file_util::write_text_file(file_name, file_text);
 
-    auto final = ir2_final_out(obj, imports, {});
+    auto unformatted_code = ir2_final_out(obj, imports, {});
     auto final_name = output_dir / (obj.to_unique_name() + "_disasm.gc");
-    file_util::write_text_file(final_name, final);
+    if (config.format_code) {
+      const auto formatted_code = formatter::format_code(unformatted_code);
+      if (!formatted_code) {
+        lg::error(
+            "Was unable to format the decompiled result of {}, make a github issue. Writing "
+            "unformatted code",
+            obj.to_unique_name());
+        file_util::write_text_file(final_name, unformatted_code);
+      } else {
+        file_util::write_text_file(final_name, formatted_code.value());
+      }
+    } else {
+      file_util::write_text_file(final_name, unformatted_code);
+    }
   }
 }
 

decompiler/analysis/final_output.cpp

@@ -54,12 +54,8 @@ void append_body_to_function_definition(goos::Object* top_form,
   body_elements.insert(body_elements.end(), inline_body.begin(), inline_body.end());
   // If the first element in the body is a docstring, add it first
   if (body_elements.size() > 0 && body_elements.at(0).is_string()) {
-    if (version > GameVersion::Jak2) {
-      initial_top_level_forms.push_back(
-          goos::StringObject::make_new(fix_docstring_indent(inline_body.at(0).as_string()->data)));
-    } else {
-      initial_top_level_forms.push_back(inline_body.at(0));
-    }
+    initial_top_level_forms.push_back(
+        goos::StringObject::make_new(fix_docstring_indent(inline_body.at(0).as_string()->data)));
     body_elements.erase(body_elements.begin());
   }
 

decompiler/config.cpp

@@ -64,7 +64,6 @@ Config make_config_via_json(nlohmann::json& json) {
         inputs_json.at("str_art_file_names").get<std::vector<std::string>>();
   }
 
-  config.audio_dir_file_name = inputs_json.at("audio_dir_file_name").get<std::string>();
   config.streamed_audio_file_names =
       inputs_json.at("streamed_audio_file_names").get<std::vector<std::string>>();
 
@@ -96,6 +95,9 @@ Config make_config_via_json(nlohmann::json& json) {
   }
   config.disassemble_code = json.at("disassemble_code").get<bool>();
   config.decompile_code = json.at("decompile_code").get<bool>();
+  if (json.contains("format_code")) {
+    config.format_code = json.at("format_code").get<bool>();
+  }
   config.write_hex_near_instructions = json.at("write_hex_near_instructions").get<bool>();
   config.write_scripts = json.at("write_scripts").get<bool>();
   config.disassemble_data = json.at("disassemble_data").get<bool>();
@@ -318,6 +320,9 @@ Config make_config_via_json(nlohmann::json& json) {
   if (json.contains("save_texture_pngs")) {
     config.save_texture_pngs = json.at("save_texture_pngs").get<bool>();
   }
+  if (json.contains("rip_streamed_audio")) {
+    config.rip_streamed_audio = json.at("rip_streamed_audio").get<bool>();
+  }
 
   if (inputs_json.contains("animated_textures")) {
     config.animated_textures =

decompiler/config.h

@@ -102,8 +102,6 @@ struct Config {
   std::vector<std::string> str_file_names;
   std::vector<std::string> str_texture_file_names;
   std::vector<std::string> str_art_file_names;
-
-  std::string audio_dir_file_name;
   std::vector<std::string> streamed_audio_file_names;
 
   std::string obj_file_name_map_file;
@@ -111,6 +109,7 @@ struct Config {
 
   bool disassemble_code = false;
   bool decompile_code = false;
+  bool format_code = false;
   bool write_scripts = false;
   bool disassemble_data = false;
   bool process_tpages = false;
@@ -174,6 +173,7 @@ struct Config {
   std::vector<std::string> levels_to_extract;
   bool levels_extract;
   bool save_texture_pngs = false;
+  bool rip_streamed_audio = false;
 
   DecompileHacks hacks;
 

decompiler/config/jak1/all-types.gc

@@ -21102,7 +21102,7 @@
   :size-assert         #x94
   :flag-assert         #xf00300094
   (:methods
-    (dumb-15 (_type_) none))
+    (spawn-particles! (_type_) none))
   (:states
     hud-normal
     hud-coming-in
@@ -22303,20 +22303,20 @@
 
 ;; - Functions
 
-(define-extern calculate-rotation-and-color-for-slice (function int float int int int hud-particle none))
-(define-extern part-hud-health-01-func (function basic basic hud-particle none)) ;; TODO - i figured this out on another branch...all these particle functions get called by the particle system with the same beginning types iirc
-(define-extern part-hud-health-02-func (function basic basic hud-particle none))
-(define-extern part-hud-health-03-func (function basic basic hud-particle none))
+(define-extern calculate-rotation-and-color-for-slice (function int float int int int matrix none))
+(define-extern part-hud-health-01-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-health-02-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-health-03-func (function sparticle-system sparticle-cpuinfo matrix none))
 (define-extern fuel-cell-hud-orbit-callback (function sparticle-system sparticle-cpuinfo matrix none))
 (define-extern fuel-cell-hud-starburst-3-callback (function sparticle-system sparticle-cpuinfo matrix none))
 (define-extern fuel-cell-hud-starburst-4-callback (function sparticle-system sparticle-cpuinfo matrix none))
 (define-extern fuel-cell-hud-center-callback (function sparticle-system sparticle-cpuinfo matrix none))
-(define-extern part-hud-buzzer-func (function basic basic hud-particle none))
-(define-extern part-hud-eco-timer-01-func (function basic basic hud-particle none))
-(define-extern part-hud-eco-timer-02-func (function basic basic hud-particle none))
-(define-extern part-hud-eco-timer-03-func (function basic basic hud-particle none))
-(define-extern part-hud-eco-timer-backing-func (function basic basic hud-particle none))
-(define-extern part-hud-eco-timer-func (function basic basic hud-particle none))
+(define-extern part-hud-buzzer-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-eco-timer-01-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-eco-timer-02-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-eco-timer-03-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-eco-timer-backing-func (function sparticle-system sparticle-cpuinfo matrix none))
+(define-extern part-hud-eco-timer-func (function sparticle-system sparticle-cpuinfo matrix none))
 (define-extern show-hud (function none))
 
 ;; - Unknowns

decompiler/config/jak1/jak1_config.jsonc

@@ -23,6 +23,10 @@
   // run the first pass of the decompiler
   "find_functions": true,
 
+  // will attempt to run the decompiled output through the OpenGOAL formatter
+  // this will be skipped in offline tests
+  "format_code": true,
+
   ////////////////////////////
   // DATA ANALYSIS OPTIONS
   ////////////////////////////
@@ -118,6 +122,9 @@
   // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
 
+  // whether or not to dump out streamed audio files to decompiler_out/<game>/audio
+  "rip_streamed_audio": false,
+
   ////////////////////////////
   // PATCHING OPTIONS
   ////////////////////////////

decompiler/config/jak1/ntsc_v1/inputs.jsonc

@@ -258,10 +258,6 @@
     "TEXT/6COMMON.TXT"
   ],
 
-  // uncomment the next line to extract audio to wave files.
-  //"audio_dir_file_name": "jak1/VAG",
-  "audio_dir_file_name": "",
-
   "streamed_audio_file_names": [
     "VAGWAD.ENG",
     "VAGWAD.FRE",

decompiler/config/jak1_demo/default/inputs.jsonc

@@ -228,9 +228,5 @@
     "TEXT/6COMMON.TXT"
   ],
 
-  // uncomment the next line to extract audio to wave files.
-  //"audio_dir_file_name": "jak1/VAG",
-  "audio_dir_file_name": "",
-
   "streamed_audio_file_names": ["VAGWAD.ENG", "VAGWAD.JAP"]
 }

decompiler/config/jak2/all-types.gc

@@ -2515,6 +2515,7 @@
   (iop-mem             48)
   (cancel-dgo          49)
   (set-stereo-mode     50) ;; sound-rpc-set-stereo-mode
+  (set-mirror         201)
   )
 
 (defenum sound-group
@@ -2829,6 +2830,17 @@
   :flag-assert         #x900000004
   )
 
+; added
+(defenum sound-mirror-mode
+  :type uint8
+  (normal)
+  (mirrored)
+  )
+
+; added for mirror mode
+(deftype sound-rpc-set-mirror-mode (sound-rpc-cmd)
+  ((mirror sound-mirror-mode)))
+
 (deftype sound-rpc-union (structure)
   ((data              uint32                      20 :offset-assert 0)
    (load-bank         sound-rpc-load-bank            :offset        0)
@@ -2852,6 +2864,7 @@
    (shutdown          sound-rpc-shutdown             :offset        0)
    (list-sounds       sound-rpc-list-sounds          :offset        0)
    (unload-music      sound-rpc-unload-music         :offset        0)
+   (mirror-mode        sound-rpc-set-mirror-mode    :overlay-at (-> data 0))
    )
   :method-count-assert 9
   :size-assert         #x50
@@ -8774,6 +8787,7 @@ Unlike [[engine]], users can use [[engine-pers]] as a parent class."
   (progress-graphics-ps2-lod-dist #x1337)
   (progress-graphics-msaa-off #x1338)
   (progress-graphics-display #x1339)
+  (language-name-finnish #x133a)
 )
 ;; ---text-id-h:text-id
 

decompiler/config/jak2/jak2_config.jsonc

@@ -127,6 +127,9 @@
   "rip_collision": false,
   // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
+  
+  // whether or not to dump out streamed audio files to decompiler_out/<game>/audio
+  "rip_streamed_audio": false,
 
   ////////////////////////////
   // PATCHING OPTIONS

decompiler/config/jak2/ntsc_v1/inputs.jsonc

@@ -441,10 +441,6 @@
     "TEXT/7COMMON.TXT"
   ],
 
-  // uncomment the next line to extract audio to wave files.
-  // "audio_dir_file_name": "jak2/VAG",
-  "audio_dir_file_name": "",
-
   "streamed_audio_file_names": [
     "VAGWAD.ENG",
     "VAGWAD.FRE",

decompiler/config/jak3/jak3_config.jsonc

@@ -127,6 +127,9 @@
   // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
 
+  // whether or not to dump out streamed audio files to decompiler_out/<game>/audio
+  "rip_streamed_audio": false,
+
   ////////////////////////////
   // PATCHING OPTIONS
   ////////////////////////////

decompiler/config/jak3/ntsc_v1/anonymous_function_types.jsonc

@@ -138,7 +138,7 @@
     [7, "(function none)"],
     [3, "(function symbol :behavior process)"]
   ],
-  "scene": [[4, "(function none :behavior scene-player)"]],
+  "scene": [[4, "(function symbol :behavior scene-player)"]],
   "pov-camera": [
     [
       7,
@@ -864,5 +864,141 @@
     [22, "(function game-task object)"],
     [26, "(function vector :behavior manipy)"],
     [30, "(function object :behavior manipy)"]
+  ],
+  "ff-squad-control": [
+    [6, "(function process-focusable traffic-object-type-info none)"],
+    [7, "(function process-focusable traffic-object-type-info none)"],
+    [8, "(function process-focusable traffic-object-type-info none)"],
+    [9, "(function crimson-guard traffic-object-type-info none)"]
+  ],
+  "guard": [
+    [31, "(function crimson-guard collide-shape vector symbol)"],
+    [49, "(function process city-attacker-info int)"]
+  ],
+  "guard-tazer": [
+    [7, "(function collide-shape-prim none)"],
+    [13, "(function collide-shape-prim none)"]
+  ],
+  "roboguard-city": [
+    [13, "(function cspace transformq none)"],
+    [78, "(function roboguard-city symbol quaternion :behavior process)"]
+  ],
+  "ctywide-obs": [[93, "(function symbol)"]],
+  "ctywide-scenes": [
+    [1, "(function none :behavior scene-player)"],
+    [2, "(function none :behavior scene-player)"],
+    [3, "(function none :behavior scene-player)"]
+  ],
+  "ctyport-obs": [[5, "(function collide-shape-prim none)"]],
+  "ctyport-scenes": [
+    [3, "(function none :behavior scene-player)"],
+    [4, "(function none :behavior scene-player)"],
+    [5, "(function none :behavior scene-player)"],
+    [6, "(function none :behavior scene-player)"]
+  ],
+  "ctyport-attack": [[45, "(function object :behavior process)"]],
+  "intro-scenes": [
+    [0, "(function none :behavior scene-player)"],
+    [1, "(function none :behavior scene-player)"],
+    [2, "(function none :behavior scene-player)"],
+    [3, "(function none :behavior scene-player)"],
+    [4, "(function none :behavior scene-player)"],
+    [5, "(function none :behavior scene-player)"],
+    [6, "(function none :behavior scene-player)"],
+    [7, "(function none :behavior scene-player)"],
+    [8, "(function none :behavior scene-player)"],
+    [9, "(function none :behavior scene-player)"],
+    [10, "(function none :behavior scene-player)"],
+    [11, "(function none :behavior scene-player)"],
+    [12, "(function none :behavior scene-player)"],
+    [13, "(function none :behavior scene-player)"],
+    [14, "(function none :behavior scene-player)"],
+    [15, "(function none :behavior scene-player)"],
+    [16, "(function none :behavior scene-player)"],
+    [17, "(function none :behavior scene-player)"],
+    [18, "(function none :behavior scene-player)"],
+    [19, "(function none :behavior scene-player)"],
+    [20, "(function none :behavior scene-player)"],
+    [21, "(function none :behavior scene-player)"],
+    [22, "(function none :behavior scene-player)"],
+    [23, "(function none :behavior scene-player)"],
+    [24, "(function none :behavior scene-player)"],
+    [25, "(function none :behavior scene-player)"],
+    [26, "(function none :behavior scene-player)"],
+    [27, "(function none :behavior scene-player)"],
+    [28, "(function none :behavior scene-player)"],
+    [29, "(function none :behavior scene-player)"],
+    [30, "(function none :behavior scene-player)"],
+    [31, "(function none :behavior scene-player)"],
+    [32, "(function none :behavior scene-player)"],
+    [33, "(function none :behavior scene-player)"],
+    [34, "(function none :behavior scene-player)"],
+    [35, "(function none :behavior scene-player)"],
+    [36, "(function none :behavior scene-player)"],
+    [37, "(function none :behavior scene-player)"],
+    [38, "(function none :behavior scene-player)"],
+    [39, "(function none :behavior scene-player)"]
+  ],
+  "freehq-scenes": [[0, "(function none :behavior scene-player)"]],
+  "hiphog-scenes": [
+    [0, "(function none :behavior scene-player)"],
+    [1, "(function none :behavior scene-player)"],
+    [2, "(function none :behavior scene-player)"],
+    [3, "(function none :behavior scene-player)"]
+  ],
+  "rubble-attack": [[7, "(function int)"]],
+  "rublcst-scenes": [
+    [0, "(function none :behavior scene-player)"],
+    [1, "(function none :behavior scene-player)"],
+    [2, "(function none :behavior scene-player)"],
+    [3, "(function none :behavior scene-player)"],
+    [4, "(function none :behavior scene-player)"],
+    [5, "(function none :behavior scene-player)"],
+    [6, "(function none :behavior scene-player)"],
+    [7, "(function none :behavior scene-player)"],
+    [8, "(function none :behavior scene-player)"],
+    [9, "(function none :behavior scene-player)"],
+    [10, "(function none :behavior scene-player)"],
+    [11, "(function none :behavior scene-player)"],
+    [12, "(function none :behavior scene-player)"],
+    [13, "(function none :behavior scene-player)"]
+  ],
+  "vinroom-scenes": [
+    [0, "(function none :behavior scene-player)"],
+    [1, "(function none :behavior scene-player)"]
+  ],
+  "gungame-manager": [[6, "(function process symbol)"]],
+  "blow-tower-obs2": [
+    [16, "(function bt-mh-flyer cspace float float vector vector int none)"],
+    [117, "(function bt-roboguard symbol quaternion)"],
+    [124, "(function cspace transformq none)"]
+  ],
+  "cty-blow-tower": [
+    [39, "(function collide-shape-prim none)"],
+    [27, "(function process symbol)"],
+    [46, "(function collide-shape-prim none)"],
+    [69, "(function process object)"]
+  ],
+  "assault-enemies": [
+    [36, "(function assault-crimson-guard city-attacker-info int)"]
+  ],
+  "assault-task": [[11, "(function symbol :behavior process)"]],
+  "precura-obs": [[98, "(function symbol :behavior process)"]],
+  "precurd-scenes": [
+    [0, "(function none :behavior scene-player)"],
+    [1, "(function none :behavior scene-player)"],
+    [2, "(function none :behavior scene-player)"],
+    [3, "(function none :behavior scene-player)"],
+    [4, "(function none :behavior scene-player)"],
+    [5, "(function none :behavior scene-player)"],
+    [6, "(function none :behavior scene-player)"],
+    [7, "(function none :behavior scene-player)"],
+    [8, "(function none :behavior scene-player)"],
+    [9, "(function none :behavior scene-player)"],
+    [10, "(function none :behavior scene-player)"],
+    [11, "(function none :behavior scene-player)"],
+    [12, "(function none :behavior scene-player)"],
+    [13, "(function none :behavior scene-player)"],
+    [14, "(function none :behavior scene-player)"]
   ]
 }

decompiler/config/jak3/ntsc_v1/art_info.jsonc

@@ -25,7 +25,21 @@
     "maker": "dm-robot-ag",
     "mh-wasp": "neo-wasp-ag",
     "factory-boss": "errol-lowres-ag",
-    "fac-robotank-turret": "fac-robotank-ag"
+    "fac-robotank-turret": "fac-robotank-ag",
+    "neo-sat-shield": "neo-satellite-shield-ag",
+    "neo-sat": "neo-satellite-ag",
+    "power-game-switcher": "switcher-ag",
+    "power-game-player": "daxter-pac-man-ag",
+    "power-game-glyph": "cipher-ag",
+    "power-game-rings": "pow-rings-ag",
+    "power-game-chaser": "flitter-ag",
+    "power-game-zapper": "grunt-head-ag",
+    "gungame-door": "fort-entry-gate-ag",
+    "bt-mh-flyer": "bt-wasp-ag",
+    "gunship-exploder": "kg-robot-transport-break",
+    "gunship-engine": "kg-robot-transport-bomb",
+    "protect-gunship": "kg-robot-transport",
+    "gunship-missile": "cty-homing-missile"
   },
 
   // remap names for types in an entire file (higher priority)

decompiler/config/jak3/ntsc_v1/hacks.jsonc

@@ -78,10 +78,6 @@
     "blerc-execute",
     "foreground-check-longest-edge-asm",
     "generic-light-proc",
-    "shadow-add-single-edges",
-    "shadow-add-facing-single-tris",
-    "shadow-add-double-tris",
-    "shadow-add-double-edges",
     "(method 17 collide-edge-work)",
     "(method 10 collide-cache-prim)",
     "(method 17 collide-cache)",
@@ -107,8 +103,6 @@
     // jak 3
     "(method 10 manipulator)",
     "(method 46 ff-squad-control)",
-    "(method 33 rub-tower)",
-    "(method 261 crimson-guard)",
     "memcpy"
   ],
 
@@ -225,7 +219,10 @@
     "update-sound-info",
     "insert-into-sound-list",
     // title-obs
-    "(anon-function 22 title-obs)"
+    "(anon-function 22 title-obs)",
+    "cty-faction-evaluate-commands",
+    "traffic-manager-event-handler",
+    "(method 20 cty-faction-manager)"
   ],
 
   // If format is used with the wrong number of arguments,
@@ -271,7 +268,8 @@
     "~10Hfg-wa: ~8,,0m": 1,
     "~140Hfg-sh: ~8,,0m": 1,
     "~267Hfg-p2: ~8,,0m~%": 1,
-    "~30Hp2: ~8D~131Hhf: ~8D~%~1K": 2
+    "~30Hp2: ~8D~131Hhf: ~8D~%~1K": 2,
+    "Current time (~d:~d) [mission-percentage ~f~%": 2
   },
 
   "blocks_ending_in_asm_branch": {
@@ -448,7 +446,23 @@
     "generic-merc-execute-all": [7, 15],
     "check-enemy": [0, 1],
     "(method 91 h-warf)": [0],
-    "(method 51 h-warf)": [5]
+    "(method 51 h-warf)": [5],
+    "(method 44 nav-graph)": [0, 7, 8, 26, 34],
+    "(method 15 city-level-info)": [0, 1, 2, 6, 7, 9, 11, 13],
+    "(method 10 traffic-suppressor)": [0, 1, 2, 4],
+    "(method 18 traffic-tracker)": [5, 6, 7, 8],
+    "(method 21 traffic-tracker)": [0],
+    "(method 9 cty-faction-manager)": [8],
+    "(method 46 traffic-engine)": [0, 1, 2, 4],
+    "(method 42 traffic-engine)": [6],
+    "(method 17 traffic-manager)": [7, 19, 27, 28, 29, 30, 41],
+    "(anon-function 10 guard-rifle)": [9],
+    "(method 261 crimson-guard)": [52, 64],
+    "(anon-function 13 metalhead-predator)": [24, 25],
+    "(anon-function 90 ctywide-obs)": [4],
+    "(anon-function 10 cty-sniper-turret)": [44],
+    "(method 33 rub-tower)": [9, 10],
+    "(method 30 gungame-manager)": [0, 4, 5, 7]
   },
 
   // Sometimes the game might use format strings that are fetched dynamically,
@@ -544,7 +558,27 @@
     "(method 32 task-manager-dark-punch-training)": [[53, 0]],
     "(method 32 task-manager-lightjak-training-shield)": [[53, 0]],
     "(trans credits highres-viewer-manager)": [[185, 0]],
-    "(trans idle hirez-viewer)": [[356, 0]]
+    "(trans idle hirez-viewer)": [[356, 0]],
+    "(trans idle burning-bush)": [
+      [171, 1],
+      [202, 0]
+    ],
+    "(method 33 task-manager-bbush-board)": [[86, 0]],
+    "(method 17 board-score-hud)": [[71, 0]],
+    "(method 23 gungame-manager)": [
+      [52, 0],
+      [90, 0],
+      [128, 0],
+      [164, 0],
+      [194, 0],
+      [235, 0],
+      [273, 0],
+      [305, 0],
+      [346, 0],
+      [378, 0],
+      [413, 0]
+    ],
+    "(trans carry precur-bomb)": [[60, 0]]
   },
 
   "mips2c_functions_by_name": [
@@ -571,13 +605,14 @@
     "draw-large-polygon",
     "render-sky-quad",
     "render-sky-tri",
-    "(method 16 sky-work)",
     "(method 17 sky-work)",
+    "(method 18 sky-work)",
     "(method 32 sky-work)",
-    "(method 33 sky-work)",
-    // "(method 28 sky-work)",
+    "(method 31 sky-work)",
+    "(method 34 sky-work)",
     "(method 29 sky-work)",
     "(method 30 sky-work)",
+    "(method 35 sky-work)",
     "(method 11 collide-hash)",
     "(method 12 collide-hash)",
     "fill-bg-using-box-new",
@@ -673,11 +708,26 @@
     "(method 21 cloth-system)",
     "debug-line-clip?",
     "(method 9 font-work)",
-    "(method 9 prim-strip)",
     "live-func-curve",
     "birth-func-curve",
     "sparticle-motion-blur-dirt",
-    "foreground-draw-hud"
+    "foreground-draw-hud",
+    "shadow-execute",
+    "shadow-add-double-edges",
+    "shadow-add-single-edges",
+    "shadow-add-facing-single-tris",
+    "shadow-add-double-tris",
+    "shadow-xform-verts",
+    "shadow-calc-dual-verts",
+    "shadow-scissor-edges",
+    "shadow-scissor-top",
+    "shadow-init-vars",
+    "shadow-find-facing-single-tris",
+    "shadow-find-single-edges",
+    "shadow-find-facing-double-tris",
+    "shadow-find-double-edges",
+    "shadow-add-verts",
+    "shadow-add-single-tris"
   ],
 
   "mips2c_jump_table_functions": {},
@@ -690,11 +740,9 @@
     ["lcit", 0, 0],
     ["pow", 0, 0],
     ["wasintro", 0, 0],
-    ["wasintro-vis", 0, 0],
     ["lfacctyb", 0, 0],
     ["intpfall", 0, 0],
     ["lfaccity", 0, 0],
-    ["lfacctyb-vis", 0, 0],
     ["ltowcity", 0, 0],
     ["powergd", 0, 0],
     ["lcitysml", 0, 0]

decompiler/config/jak3/ntsc_v1/inputs.jsonc

@@ -37,34 +37,34 @@
     "DGO/DESA.DGO",
     "DGO/DESB.DGO",
     "DGO/DESBATTL.DGO",
-    // "DGO/DESBCST.DGO",
+    "DGO/DESBCST.DGO",
     "DGO/DESBOSS1.DGO",
     "DGO/DESBOSS2.DGO",
-    // "DGO/DESC.DGO",
+    "DGO/DESC.DGO",
     "DGO/DESCHASE.DGO",
     "DGO/DESD.DGO",
-    // "DGO/DESE.DGO",
-    // "DGO/DESERROL.DGO",
+    "DGO/DESE.DGO",
+    "DGO/DESERROL.DGO",
     "DGO/DESF.DGO",
     "DGO/DESG.DGO",
-    // "DGO/DESH.DGO",
+    "DGO/DESH.DGO",
     "DGO/DESHOVER.DGO",
-    // "DGO/DESHUNT.DGO",
+    "DGO/DESHUNT.DGO",
     "DGO/DESINTER.DGO",
-    // "DGO/DESJUMP.DGO",
+    "DGO/DESJUMP.DGO",
     "DGO/DESLIZ.DGO",
     "DGO/DESOASIS.DGO",
     "DGO/DESRACE1.DGO",
     "DGO/DESRACE2.DGO",
     "DGO/DESRALLY.DGO",
     "DGO/DESRESC.DGO",
-    // "DGO/DESRESCC.DGO",
-    // "DGO/DESRESCG.DGO",
+    "DGO/DESRESCC.DGO",
+    "DGO/DESRESCG.DGO",
     "DGO/DESTRACK.DGO",
     "DGO/DESW.DGO",
     "DGO/DST.DGO",
     "DGO/OASISCST.DGO",
-    // "DGO/WARPCAST.DGO", // air train
+    "DGO/WARPCAST.DGO", // air train
     // nest
     "DGO/NSA.DGO",
     "DGO/NSB.DGO",
@@ -117,10 +117,10 @@
     "DGO/GUNGAME2.DGO",
     "DGO/POWERGD.DGO",
     "DGO/FREEHQ.DGO",
-    // "DGO/FREECAST.DGO",
-    // "DGO/CITYCAST.DGO",
-    // "DGO/GRIDCST.DGO", // city-destroy-grid-res
-    // "DGO/SLUMBSET.DGO", // sewer-met-hum-intro
+    "DGO/FREECAST.DGO",
+    "DGO/CITYCAST.DGO",
+    "DGO/GRIDCST.DGO", // city-destroy-grid-res
+    "DGO/SLUMBSET.DGO", // sewer-met-hum-intro
     // sewer
     "DGO/SEA.DGO",
     "DGO/SEB.DGO",
@@ -182,32 +182,32 @@
     "DGO/RAILE.DGO",
     "DGO/RAILF.DGO",
     "DGO/RAILX.DGO",
-    // // precursor
-    // "DGO/PRECA.DGO",
-    // "DGO/PRECB.DGO",
-    // "DGO/PRECC.DGO",
+    // precursor
+    "DGO/PRECA.DGO",
+    "DGO/PRECB.DGO",
+    "DGO/PRECC.DGO",
     // "DGO/PRECD.DGO",
-    // title/intro
+    // // title/intro
     "DGO/WIN.DGO", // wasintro
     "DGO/TITLE.DGO",
     "DGO/INTTITLE.DGO",
-    // "DGO/INTPALRF.DGO", // intro-palace-roof
-    // "DGO/IPF.DGO", // intro-palace-fall
+    "DGO/INTPALRF.DGO", // intro-palace-roof
+    "DGO/IPF.DGO", // intro-palace-fall
     "DGO/INTROCST.DGO",
     // // outro
-    // "DGO/OUTCAST3.DGO",
-    // "DGO/OUTROCST.DGO",
+    "DGO/OUTCAST3.DGO",
+    "DGO/OUTROCST.DGO",
     // // museum
-    // "DGO/MUSEUM.DGO",
-    // "DGO/MUSEUM2.DGO",
-    // "DGO/MUSEUM3.DGO",
-    // "DGO/MUSEUM3B.DGO",
-    // "DGO/MUSEUM4.DGO",
-    // "DGO/MUSEUM4B.DGO",
+    "DGO/MUSEUM.DGO",
+    "DGO/MUSEUM2.DGO",
+    "DGO/MUSEUM3.DGO",
+    "DGO/MUSEUM3B.DGO",
+    "DGO/MUSEUM4.DGO",
+    "DGO/MUSEUM4B.DGO",
     // test
     "DGO/HALFPIPE.DGO",
     // borrow
-    // "DGO/LASHELIN.DGO",
+    "DGO/LASHELIN.DGO",
     "DGO/LBBRING1.DGO",
     "DGO/LBBRING2.DGO",
     "DGO/LBBRING3.DGO",
@@ -225,27 +225,27 @@
     "DGO/LBBTCHA2.DGO",
     "DGO/LBBTCHA3.DGO",
     "DGO/LBIPED.DGO",
-    // "DGO/LBLOWCST.DGO",
-    // "DGO/LBLOWTKG.DGO",
-    // "DGO/LBLOWTMH.DGO",
-    // "DGO/LBOMBBOT.DGO",
-    // "DGO/LCITYSML.DGO",
-    // "DGO/LCTYASS.DGO",
-    // "DGO/LCTYBLOW.DGO",
+    "DGO/LBLOWCST.DGO",
+    "DGO/LBLOWTKG.DGO",
+    "DGO/LBLOWTMH.DGO",
+    "DGO/LBOMBBOT.DGO",
+    "DGO/LCITYSML.DGO",
+    "DGO/LCTYASS.DGO",
+    "DGO/LCTYBLOW.DGO",
     "DGO/LCTYDEST.DGO",
-    // "DGO/LCTYHIJK.DGO",
-    // "DGO/LCTYPALT.DGO",
-    // "DGO/LCTYPATK.DGO",
-    // "DGO/LCTYPROT.DGO",
-    // "DGO/LCTYSNPR.DGO",
-    // "DGO/LDAMKLEV.DGO",
-    // "DGO/LDAMPECK.DGO",
-    // "DGO/LDAMPKSM.DGO",
-    // "DGO/LDAMSIG.DGO",
+    "DGO/LCTYHIJK.DGO",
+    "DGO/LCTYPALT.DGO",
+    "DGO/LCTYPATK.DGO",
+    "DGO/LCTYPROT.DGO",
+    "DGO/LCTYSNPR.DGO",
+    "DGO/LDAMKLEV.DGO",
+    "DGO/LDAMPECK.DGO",
+    "DGO/LDAMPKSM.DGO",
+    "DGO/LDAMSIG.DGO",
     "DGO/LDAX.DGO",
-    // "DGO/LDESGCST.DGO",
-    // "DGO/LDMPCKGN.DGO",
-    // "DGO/LERROL.DGO",
+    "DGO/LDESGCST.DGO",
+    "DGO/LDMPCKGN.DGO",
+    "DGO/LERROL.DGO",
     "DGO/LFACB.DGO",
     "DGO/LFACCAR.DGO",
     "DGO/LFACCITY.DGO",
@@ -255,23 +255,23 @@
     "DGO/LFACTORY.DGO",
     "DGO/LFORM.DGO",
     "DGO/LFORP.DGO",
-    // "DGO/LFORRING.DGO",
+    "DGO/LFORRING.DGO",
     "DGO/LFREEOUT.DGO",
-    // "DGO/LGUNNORM.DGO",
-    // "DGO/LGUNRNC.DGO",
+    "DGO/LGUNNORM.DGO",
+    "DGO/LGUNRNC.DGO",
     "DGO/LJAK.DGO",
     "DGO/LJAKC.DGO",
     "DGO/LJAKCKLV.DGO",
     "DGO/LJAKKLEV.DGO",
     "DGO/LJAKNDAX.DGO",
     "DGO/LJAKSIG.DGO",
-    // "DGO/LJINX.DGO",
-    // "DGO/LJKCDMKL.DGO",
-    // "DGO/LJKDMPK.DGO",
-    // "DGO/LJKDXVIN.DGO",
+    "DGO/LJINX.DGO",
+    "DGO/LJKCDMKL.DGO",
+    "DGO/LJKDMPK.DGO",
+    "DGO/LJKDXVIN.DGO",
     "DGO/LJKFEET.DGO",
-    // "DGO/LJNDKLEV.DGO",
-    // "DGO/LKEIRA.DGO",
+    "DGO/LJNDKLEV.DGO",
+    "DGO/LKEIRA.DGO",
     "DGO/LKLEEVER.DGO",
     "DGO/LMECH.DGO",
     "DGO/LMHCA.DGO",
@@ -280,34 +280,34 @@
     "DGO/LNSTOA.DGO",
     "DGO/LNSTOBB.DGO",
     "DGO/LNSTOBC.DGO",
-    // "DGO/LONINSIM.DGO",
-    // "DGO/LOUTRO.DGO",
-    // "DGO/LOUTRO2.DGO",
-    // "DGO/LOUTRO3.DGO",
-    // "DGO/LPATK.DGO",
-    // "DGO/LPATKCS.DGO",
-    // "DGO/LPRECC.DGO",
-    // "DGO/LPRENME.DGO",
-    // "DGO/LPTRL.DGO",
-    // "DGO/LSAMOS.DGO",
+    "DGO/LONINSIM.DGO",
+    "DGO/LOUTRO.DGO",
+    "DGO/LOUTRO2.DGO",
+    "DGO/LOUTRO3.DGO",
+    "DGO/LPATK.DGO",
+    "DGO/LPATKCS.DGO",
+    "DGO/LPRECC.DGO",
+    "DGO/LPRENME.DGO",
+    "DGO/LPTRL.DGO",
+    "DGO/LSAMOS.DGO",
     "DGO/LSEEMWCA.DGO",
     "DGO/LSIG.DGO",
     "DGO/LSIGJAKC.DGO",
     "DGO/LSIGKLV.DGO",
     "DGO/LSNKWHLS.DGO",
-    // "DGO/LTNFXHIP.DGO",
-    // "DGO/LTNJXHIP.DGO",
+    "DGO/LTNFXHIP.DGO",
+    "DGO/LTNJXHIP.DGO",
     "DGO/LTORN.DGO",
-    // "DGO/LTORNJNX.DGO",
-    // "DGO/LTORNSAM.DGO",
+    "DGO/LTORNJNX.DGO",
+    "DGO/LTORNSAM.DGO",
     "DGO/LTOWA.DGO",
     "DGO/LTOWB.DGO",
     "DGO/LTOWCITY.DGO",
     "DGO/LTRTWHLS.DGO",
-    // "DGO/LVINCST.DGO",
+    "DGO/LVINCST.DGO",
     "DGO/LWASBBV.DGO",
     "DGO/LWASSIG.DGO",
-    // "DGO/LWLANDM.DGO",
+    "DGO/LWLANDM.DGO",
     "DGO/LWSTDPCK.DGO"
   ],
 
@@ -333,10 +333,6 @@
     "TEXT/11COMMON.TXT"
   ],
 
-  // uncomment the next line to extract audio to wave files.
-  // "audio_dir_file_name": "jak3/VAG",
-  "audio_dir_file_name": "",
-
   "streamed_audio_file_names": [
     "VAGWAD.ENG",
     "VAGWAD.FRE",
@@ -347,6 +343,335 @@
     "VAGWAD.INT"
   ],
 
+  "animated_textures": [
+    // dark jak
+    "jakc-arm",
+    "jakc-eyebrow",
+    "jakc-face",
+    "jakc-finger",
+    "jakc-hair",
+    "jakc-arm-norm",
+    "jakc-eyebrow-norm",
+    "jakc-face-norm",
+    "jakc-finger-norm",
+    "jakc-hair-norm",
+    "jakc-arm-dark",
+    "jakc-eyebrow-dark",
+    "jakc-face-dark",
+    "jakc-finger-dark",
+    "jakc-hair-dark",
+
+    // darkjak-highres
+    "jakchires-arm",
+    "jakchires-arm-norm",
+    "jakchires-arm-dark",
+    "jakchires-eye",
+    "jakchires-eye-norm",
+    "jakchires-eye-dark",
+    "jakchires-eyebrow",
+    "jakchires-eyebrow-norm",
+    "jakchires-eyebrow-dark",
+    "jakchires-eyelid",
+    "jakchires-eyelid-norm",
+    "jakchires-eyelid-dark",
+    "jakchires-facelft",
+    "jakchires-facelft-norm",
+    "jakchires-facelft-dark",
+    "jakchires-facert",
+    "jakchires-facert-norm",
+    "jakchires-facert-dark",
+    "jakchires-hair",
+    "jakchires-hair-norm",
+    "jakchires-hair-dark",
+
+    // Skull Gem
+    "skull-gem-dest",
+    "skull-gem-alpha-00",
+    "skull-gem-alpha-01",
+    "skull-gem-alpha-02",
+
+    // default-water
+    "bomb-gradient",
+    "bomb-gradient-rim",
+    "bomb-gradient-flames",
+    "lightjak-wings",
+    "lighjak-wings-u-src",
+    "lightjak-wings-v-src",
+    "mushroom-dest",
+    "mushroom-src",
+
+    // default-warp
+    "shield-env-rim-dest",
+    "common-white",
+    "shield-env-uscroll",
+    "shield-env-rim-src",
+
+    // hanga-sprite
+    "glider-ring-dest2",
+    "glider-ring-dest",
+    "splash-foam",
+    "racegate",
+
+    // templea-water
+    "templea-waterfall-dest",
+    "templea-waterfall",
+
+    // templea-warp
+    "holograph-env-rim-dest",
+    "holograph-env-noise",
+    "holograph-env-scan",
+    "holograph-env-rim",
+    "environment-phong-rim",
+
+    // foresta-water
+    "fora-water-dest",
+    "fora-water",
+    "fora-waterfall-01-dest",
+    "fora-waterfall-01",
+    "fora-water-wave-01-dest",
+    "fora-water-wave-01",
+
+    // forestb-water
+    "forb-water-dest",
+    "forb-water",
+    "forb-waterfall-01-dest",
+    "forb-waterfall-01",
+    "forb-water-wave-01-dest",
+    "forb-water-wave-01",
+
+    // lforplnt-pris
+    "mh-gem-dest",
+    "mh-gem",
+    "mh-gem-alpha-01",
+    "mh-gem-alpha-02",
+
+    // lmhcitya-tfrag
+    "mhcitya-base-goo-01-dest",
+    "mhcitya-base-goo-01",
+
+    // lmhcityb-tfrag
+    "mhcityb-base-goo-01-dest",
+    "mhcityb-base-goo-01",
+
+    // mhcitya-pris
+    // "mhcity-de-door-skin-01-dest",
+    // "mhcity-de-door-skin-01",
+    // "mhcity-de-door-skin-02"
+
+    // templec-water
+    "tplc-water-dest",
+    "tplc-water",
+
+    // sewc-water
+    "sewer-water-01-c-dest",
+    "sewer-water-01-c",
+    "sewer-waterfall-01-c-dest",
+    "sewer-waterfall-01-c",
+    "sewer-waterfall-02-c-dest",
+    "sewer-waterfall-02-c",
+    "sewer-water-wave-01-c-dest",
+    "sewer-water-wave-01-c",
+    "sewer-water-highlight-01-c-dest",
+    "sewer-water-highlight-01-c",
+
+    // sewd-water
+    "sewer-water-01-d-dest",
+    "sewer-water-01-d",
+    "sewer-waterfall-02-d-dest",
+    "sewer-waterfall-02-d",
+    "sewer-water-highlight-01-d-dest",
+    "sewer-water-highlight-01-d",
+    "sewer-water-wave-01-d-dest",
+    "sewer-water-wave-01-d",
+    "sewer-water-wave-02-d-dest",
+    "sewer-water-wave-02-d",
+    "sewer-water-still-01-d-dest",
+    "sewer-water-still-01-d",
+
+    // sewe-water
+    "sewer-water-01-e-dest",
+    "sewer-water-01-e",
+    "sewer-waterfall-01-e-dest",
+    "sewer-waterfall-01-e",
+    "sewer-waterfall-02-e-dest",
+    "sewer-waterfall-02-e",
+    "sewer-water-highlight-01-e-dest",
+    "sewer-water-highlight-01-e",
+
+    // sewg-water
+    "sewer-water-01-g-dest",
+    "sewer-water-01-g",
+    "sewer-waterfall-02-g-dest",
+    "sewer-waterfall-02-g",
+    "sewer-water-wave-01-g-dest",
+    "sewer-water-wave-01-g",
+
+    // sewh-water
+    "sewer-water-01-h-dest",
+    "sewer-water-01-h",
+    "sewer-waterfall-02-h-dest",
+    "sewer-waterfall-02-h",
+    "sewer-water-wave-02-h-dest",
+    "sewer-water-wave-02-h",
+    "sewer-watefall-froth-01-h-dest",
+    "sewer-watefall-froth-01-h",
+
+    // sewi-water
+    "sewer-water-still-01-i-dest",
+    "sewer-water-still-01-i",
+    "sewer-waterfall-02-i-dest",
+    "sewer-waterfall-02-i",
+    "sewer-water-wave-01-i-dest",
+    "sewer-water-wave-01-i",
+
+    // sewj-water
+    "sewer-waterfall-02-j-dest",
+    "sewer-waterfall-02-j",
+    "sewer-watefall-froth-01-j-dest",
+    "sewer-watefall-froth-01-j",
+
+    // sewl-water
+    "sewer-waterfall-02-l-dest",
+    "sewer-waterfall-02-l",
+    "sewer-watefall-froth-01-l-dest",
+    "sewer-watefall-froth-01-l",
+
+    // sewm-water
+    "sewer-water-01-m-dest",
+    "sewer-water-01-m",
+    "sewer-waterfall-01-m-dest",
+    "sewer-waterfall-01-m",
+    "sewer-waterfall-02-m-dest",
+    "sewer-waterfall-02-m",
+    "sewer-water-highlight-01-m-dest",
+    "sewer-water-highlight-01-m",
+    "sewer-water-wave-01-m-dest",
+    "sewer-water-wave-01-m",
+    "sewer-water-still-01-m-dest",
+    "sewer-water-still-01-m",
+    "sewer-watefall-froth-01-m-dest",
+    "sewer-watefall-froth-01-m",
+
+    // sewn-water
+    "sewer-waterfall-01-n-dest",
+    "sewer-waterfall-01-n",
+    "sewer-waterfall-02-n-dest",
+    "sewer-waterfall-02-n",
+    // "sewer-water-highlight-01-n-dest",
+    // "sewer-water-highlight-01-n",
+    "sewer-water-wave-01-n-dest",
+    "sewer-water-wave-01-n",
+    "sewer-water-still-01-n-dest",
+    "sewer-water-still-01-n",
+
+    // hanga-water
+    // "des-thermal-01-dest",
+    // "des-thermal-01"
+
+    // desresc-warp
+    "sat-shield-dest",
+    "sat-shield-env-uvscroll",
+    "sat-shield",
+
+    // security
+    "security-env-dest",
+    "security-env-uscroll",
+    "security-dot-dest",
+    "security-dot-src",
+
+    // lgunnorm-water
+    "kg-target-c-forcefield-01-dest",
+    "kg-target-c-forcefield-01",
+
+    // templex-water
+    "temple-waterfall-dest",
+    "temple-waterfall",
+
+    // desertd-water
+    "des-waterfall-dest",
+    "des-waterfall",
+
+    // towerb-water
+    "tow-energy-bridge-dest",
+    "tow-energy-bridge",
+
+    // factoryb-water
+    "hemi-gradient-dest",
+    "hemi-gradient-rim",
+    "hemi-gradient-flames-dest",
+    "hemi-gradient-flames",
+
+    // factoryc-alpha
+    "facc-convey-dest",
+    "facc-convey",
+    "facc-convey-02-dest",
+    "facc-convey-02",
+
+    // waspal-water
+    "waspala-water-dest",
+    "waspala-water",
+    "waspala-waterfall-dest",
+    "waspala-waterfall",
+
+    // rubblea-water
+    "rub-water-dest",
+    "rub-water",
+
+    // rubblea2-water
+    "rub-water-desta2",
+    "rub-watera2",
+
+    // rubbleb-water
+    "rub-water-destb",
+    "rub-waterb",
+
+    // rubblec-water
+    "rub-water-destc",
+    "rub-waterc",
+
+    // nstb-quicksand
+    "nstb-quicksand-dest",
+    "nstb-quicksand-scroll",
+
+    // ctyslumb-water
+    "ctyslumb-water-dest",
+    "ctyslumb-water",
+    "ctyslumb-fountain-fall-dest",
+    "ctyslumb-fountain-fall",
+
+    // ctyslumc-water
+    "ctyslumc-water-dest",
+    "ctyslumc-water",
+    "ctyslumc-fountain-fall-dest",
+    "ctyslumc-fountain-fall",
+
+    // mined-tfrag
+    "mined-pillar-side-dest",
+    "mined-pillar-side-cold",
+    "mined-pillar-side-cooling",
+    "mined-pillar-side-hot",
+    "mined-pillar-molten",
+    "mined-pillar-top2side-dest",
+    "mined-pillar-top2side-cold",
+    "mined-pillar-top2side-cooling",
+    "mined-pillar-top2side-hot",
+    "mined-pillar-top-dest",
+    "mined-pillar-top-cold",
+    "mined-pillar-top-cooling",
+    "mined-pillar-top-hot",
+    "mined-pillar-molten-top",
+
+    // volcanoa-alpha
+    "vola-lava-01-dest",
+    "vola-lava-01",
+    "vola-lava-fall-dest",
+    "vola-lava-fall",
+
+    // wasstada-alpha
+    "wstd-lava-base-dest",
+    "wstd-lava-base"
+  ],
+
   "levels_to_extract": [
     // wascity
     "WASALL.DGO",

decompiler/config/jak3/ntsc_v1/label_types.jsonc

@@ -575,6 +575,15 @@
     ["L23", "(inline-array sky-vertex)", 648],
     ["L21", "(inline-array sky-vertex)", 144]
   ],
+  "sky-tng": [
+    ["L130", "vector"],
+    ["L131", "uint64", true],
+    ["L132", "uint64", true],
+    ["L133", "uint64", true],
+    ["L134", "uint64", true],
+    ["L135", "uint64", true],
+    ["L136", "uint64", true]
+  ],
   "mood-funcs": [
     ["L198", "uint64", true],
     ["L197", "uint64", true],
@@ -943,7 +952,7 @@
     ["L19", "vector"],
     ["L18", "vector"],
     ["L17", "vector"],
-    ["L24", "vector"],
+    ["L24", "(inline-array vector)", 2],
     ["L15", "vector"],
     ["L14", "vector"]
   ],
@@ -1551,7 +1560,909 @@
     ["L345", "vector"],
     ["L346", "vector"]
   ],
-  "title-obs": [
-    ["L651", "vector"]
+  "title-obs": [["L651", "vector"]],
+  "formations": [["L7", "(inline-array formation-info)", 2]],
+  "citizen": [["L302", "attack-info"]],
+  "desert-rescue": [
+    ["L552", "(inline-array talker-speech-class)", 85],
+    ["L422", "(inline-array vector)", 50]
+  ],
+  "neo-satellite": [
+    ["L808", "uint64", true],
+    ["L806", "uint64", true],
+    ["L809", "uint64", true],
+    ["L805", "uint64", true],
+    ["L807", "uint64", true],
+    ["L804", "uint64", true],
+    ["L681", "uint64", true],
+    ["L683", "uint64", true],
+    ["L666", "uint64", true],
+    ["L678", "uint64", true],
+    ["L516", "attack-info"],
+    ["L667", "uint64", true],
+    ["L665", "uint64", true],
+    ["L670", "uint64", true],
+    ["L668", "uint64", true],
+    ["L671", "uint64", true],
+    ["L680", "uint64", true],
+    ["L672", "uint64", true],
+    ["L677", "uint64", true],
+    ["L682", "uint64", true],
+    ["L674", "uint64", true],
+    ["L673", "uint64", true],
+    ["L679", "uint64", true],
+    ["L499", "attack-info"],
+    ["L676", "uint64", true],
+    ["L675", "uint64", true],
+    ["L669", "uint64", true],
+    ["L500", "attack-info"],
+    ["L501", "attack-info"]
+  ],
+  "desert-rescue-bbush": [["L67", "(inline-array vector)", 50]],
+  "guard-states": [
+    ["L238", "uint64", true],
+    ["L237", "uint64", true],
+    ["L239", "uint64", true]
+  ],
+  "guard-grenade": [
+    ["L139", "uint64", true],
+    ["L138", "uint64", true],
+    ["L123", "uint64", true],
+    ["L124", "uint64", true]
+  ],
+  "guard-rifle": [
+    ["L93", "uint64", true],
+    ["L92", "uint64", true]
+  ],
+  "cty-guard-projectile": [["L95", "attack-info"]],
+  "bike": [
+    ["L31", "(inline-array vector)", 2],
+    ["L32", "(inline-array vector)", 2]
+  ],
+  "metalhead-flitter": [
+    ["L170", "vector"],
+    ["L171", "vector"]
+  ],
+  "metalhead-grunt": [
+    ["L159", "vector"],
+    ["L158", "vector"]
+  ],
+  "metalhead-predator": [
+    ["L189", "uint64", true],
+    ["L190", "uint64", true],
+    ["L163", "uint64", true],
+    ["L166", "uint64", true],
+    ["L165", "uint64", true],
+    ["L164", "uint64", true],
+    ["L142", "vector"],
+    ["L143", "vector"]
+  ],
+  "roboguard-city": [
+    ["L441", "uint64", true],
+    ["L442", "uint64", true],
+    ["L403", "uint64", true],
+    ["L402", "uint64", true],
+    ["L401", "uint64", true],
+    ["L404", "uint64", true],
+    ["L405", "uint64", true]
+  ],
+  "spydroid": [
+    ["L243", "vector"],
+    ["L242", "vector"],
+    ["L241", "vector"],
+    ["L246", "vector"],
+    ["L247", "vector"],
+    ["L251", "vector"],
+    ["L250", "vector"],
+    ["L252", "vector"],
+    ["L253", "vector"],
+    ["L254", "vector"],
+    ["L255", "vector"],
+    ["L256", "vector"],
+    ["L257", "vector"],
+    ["L258", "vector"],
+    ["L259", "vector"],
+    ["L260", "vector"]
+  ],
+  "ctywide-obs": [
+    ["L972", "uint64", true],
+    ["L969", "uint64", true],
+    ["L977", "uint64", true],
+    ["L970", "uint64", true],
+    ["L966", "uint64", true],
+    ["L964", "uint64", true],
+    ["L971", "uint64", true],
+    ["L973", "uint64", true],
+    ["L965", "uint64", true],
+    ["L975", "uint64", true],
+    ["L974", "uint64", true],
+    ["L968", "uint64", true],
+    ["L976", "uint64", true],
+    ["L967", "uint64", true],
+    ["L978", "uint64", true],
+    ["L743", "uint64", true],
+    ["L593", "vector"],
+    ["L751", "uint64", true],
+    ["L744", "uint64", true],
+    ["L746", "uint64", true],
+    ["L720", "vector"],
+    ["L726", "attack-info"],
+    ["L745", "uint64", true],
+    ["L748", "uint64", true],
+    ["L750", "uint64", true],
+    ["L749", "uint64", true],
+    ["L742", "uint64", true],
+    ["L747", "uint64", true]
+  ],
+  "ctywide-scenes": [["L968", "sphere"]],
+  "ctyport-part": [
+    ["L507", "uint64", true],
+    ["L506", "uint64", true],
+    ["L224", "vector4w"],
+    ["L225", "vector4w"],
+    ["L227", "vector4w"],
+    ["L228", "vector4w"],
+    ["L229", "vector4w"],
+    ["L230", "vector4w"],
+    ["L232", "vector4w"],
+    ["L233", "vector4w"],
+    ["L234", "vector4w"],
+    ["L235", "vector4w"],
+    ["L236", "vector4w"],
+    ["L237", "vector4w"],
+    ["L238", "vector4w"],
+    ["L239", "vector4w"],
+    ["L240", "vector4w"],
+    ["L242", "vector4w"],
+    ["L243", "vector4w"],
+    ["L244", "vector4w"],
+    ["L245", "vector4w"],
+    ["L246", "vector4w"],
+    ["L247", "vector4w"],
+    ["L248", "vector4w"],
+    ["L249", "vector4w"],
+    ["L250", "vector4w"],
+    ["L251", "vector4w"],
+    ["L253", "vector4w"],
+    ["L254", "vector4w"],
+    ["L255", "vector4w"],
+    ["L256", "vector4w"],
+    ["L257", "vector4w"],
+    ["L258", "vector4w"],
+    ["L259", "vector4w"],
+    ["L260", "vector4w"],
+    ["L261", "vector4w"],
+    ["L262", "vector4w"],
+    ["L263", "vector4w"],
+    ["L264", "vector4w"],
+    ["L265", "vector4w"],
+    ["L266", "vector4w"],
+    ["L267", "vector4w"],
+    ["L268", "vector4w"],
+    ["L270", "vector4w"],
+    ["L271", "vector4w"],
+    ["L272", "vector4w"],
+    ["L273", "vector4w"],
+    ["L274", "vector4w"],
+    ["L275", "vector4w"],
+    ["L276", "vector4w"],
+    ["L277", "vector4w"],
+    ["L278", "vector4w"],
+    ["L279", "vector4w"],
+    ["L280", "vector4w"],
+    ["L281", "vector4w"],
+    ["L282", "vector4w"],
+    ["L283", "vector4w"],
+    ["L285", "vector4w"],
+    ["L287", "vector4w"]
+  ],
+  "ctyport-obs": [["L86", "attack-info"]],
+  "h-torpedo": [
+    ["L191", "(inline-array talker-speech-class)", 5],
+    ["L184", "attack-info"]
+  ],
+  "krimson-wall": [["L62", "(inline-array vector)", 2]],
+  "ctyport-attack": [
+    ["L727", "uint64", true],
+    ["L726", "uint64", true],
+    ["L725", "uint64", true],
+    ["L729", "uint64", true],
+    ["L724", "uint64", true],
+    ["L730", "uint64", true],
+    ["L728", "uint64", true],
+    ["L723", "uint64", true],
+    ["L593", "(inline-array talker-speech-class)", 49],
+    ["L493", "(inline-array vector)", 100],
+    ["L645", "uint64", true],
+    ["L646", "uint64", true],
+    ["L643", "uint64", true],
+    ["L644", "uint64", true],
+    ["L406", "vector"]
+  ],
+  "wcar-catapult": [
+    ["L168", "vector"],
+    ["L169", "vector"],
+    ["L170", "vector"],
+    ["L171", "vector"],
+    ["L172", "vector"],
+    ["L173", "vector"],
+    ["L174", "vector"],
+    ["L167", "attack-info"],
+    ["L166", "vector"],
+    ["L165", "attack-info"]
+  ],
+  "desert-jump": [
+    ["L222", "attack-info"],
+    ["L221", "vector"],
+    ["L225", "attack-info"],
+    ["L224", "vector"]
+  ],
+  "desert-chase": [
+    ["L491", "attack-info"],
+    ["L475", "vector"]
+  ],
+  "bombbot": [
+    ["L836", "uint64", true],
+    ["L831", "uint64", true],
+    ["L834", "uint64", true],
+    ["L835", "uint64", true],
+    ["L830", "uint64", true],
+    ["L833", "uint64", true],
+    ["L832", "uint64", true],
+    ["L827", "uint64", true],
+    ["L829", "uint64", true],
+    ["L828", "uint64", true],
+    ["L647", "attack-info"],
+    ["L646", "attack-info"],
+    ["L687", "uint64", true],
+    ["L685", "uint64", true],
+    ["L677", "uint64", true],
+    ["L542", "vector"],
+    ["L683", "uint64", true],
+    ["L682", "uint64", true],
+    ["L692", "uint64", true],
+    ["L690", "uint64", true],
+    ["L686", "uint64", true],
+    ["L610", "vector"],
+    ["L680", "uint64", true],
+    ["L693", "uint64", true],
+    ["L684", "uint64", true],
+    ["L681", "uint64", true],
+    ["L689", "uint64", true],
+    ["L691", "uint64", true],
+    ["L688", "uint64", true],
+    ["L679", "uint64", true],
+    ["L678", "uint64", true],
+    ["L620", "vector"],
+    ["L619", "attack-info"],
+    ["L556", "(inline-array kg-squad-member-settings)", 4]
+  ],
+  "ctyport-attack-bbush": [
+    ["L143", "uint64", true],
+    ["L72", "vector"]
+  ],
+  "cty-hijack-missile": [["L207", "attack-info"]],
+  "cty-hijack": [["L526", "(inline-array talker-speech-class)", 52]],
+  "kg-squad-control": [
+    ["L92", "(inline-array kg-squad-member-settings)", 4],
+    ["L91", "(inline-array kg-squad-member-settings)", 4],
+    ["L90", "(inline-array kg-squad-member-settings)", 4]
+  ],
+  "mh-squad-control": [
+    ["L40", "(inline-array kg-squad-member-settings)", 3],
+    ["L39", "(inline-array kg-squad-member-settings)", 3]
+  ],
+  "intro-scenes": [
+    ["L7337", "uint64", true],
+    ["L7338", "uint64", true],
+    ["L7339", "uint64", true],
+    ["L7340", "uint64", true]
+  ],
+  "cty-destroy-grid": [
+    ["L466", "(inline-array kg-squad-member-settings)", 4],
+    ["L465", "(inline-array mh-squad-member-settings)", 3],
+    ["L446", "vector"],
+    ["L442", "vector"],
+    ["L438", "vector"]
+  ],
+  "cty-sniper-battery": [
+    ["L282", "uint64", true],
+    ["L283", "uint64", true],
+    ["L281", "uint64", true],
+    ["L249", "uint64", true],
+    ["L251", "uint64", true],
+    ["L230", "vector"],
+    ["L250", "uint64", true],
+    ["L246", "uint64", true],
+    ["L247", "uint64", true],
+    ["L248", "uint64", true],
+    ["L236", "vector"],
+    ["L235", "vector"],
+    ["L234", "vector"],
+    ["L233", "vector"],
+    ["L232", "vector"],
+    ["L231", "vector"]
+  ],
+  "cty-sniper-turret": [
+    ["L705", "uint64", true],
+    ["L709", "uint64", true],
+    ["L708", "uint64", true],
+    ["L707", "uint64", true],
+    ["L710", "uint64", true],
+    ["L706", "uint64", true],
+    ["L579", "(inline-array kg-squad-member-settings)", 4],
+    ["L426", "vector"],
+    ["L424", "vector"],
+    ["L587", "uint64", true],
+    ["L583", "uint64", true],
+    ["L594", "uint64", true],
+    ["L591", "uint64", true],
+    ["L584", "uint64", true],
+    ["L586", "uint64", true],
+    ["L585", "uint64", true],
+    ["L582", "uint64", true],
+    ["L588", "uint64", true],
+    ["L592", "uint64", true],
+    ["L581", "uint64", true],
+    ["L589", "uint64", true],
+    ["L590", "uint64", true],
+    ["L580", "uint64", true],
+    ["L593", "uint64", true],
+    ["L595", "uint64", true]
+  ],
+  "hfrag-vu1": [
+    ["L250", "uint64", true],
+    ["L249", "uint64", true],
+    ["L248", "uint64", true],
+    ["L247", "uint64", true],
+    ["L246", "uint64", true],
+    ["L245", "uint64", true],
+    ["L244", "uint64", true],
+    ["L243", "uint64", true],
+    ["L242", "uint64", true],
+    ["L241", "uint64", true],
+    ["L240", "uint64", true],
+    ["L239", "uint64", true],
+    ["L238", "uint64", true],
+    ["L237", "uint64", true],
+    ["L236", "uint64", true],
+    ["L235", "uint64", true],
+    ["L234", "uint64", true],
+    ["L233", "uint64", true],
+    ["L232", "uint64", true],
+    ["L231", "uint64", true],
+    ["L230", "uint64", true],
+    ["L229", "uint64", true]
+  ],
+  "hiphog-part": [
+    ["L219", "uint64", true],
+    ["L26", "(inline-array ripple-wave)", 6]
+  ],
+  "king-rider": [["L33", "attack-info"]],
+  "rubble-obs": [["L193", "(inline-array vector)", 2]],
+  "power-game": [
+    ["L971", "uint64", true],
+    ["L966", "uint64", true],
+    ["L967", "uint64", true],
+    ["L968", "uint64", true],
+    ["L972", "uint64", true],
+    ["L965", "uint64", true],
+    ["L969", "uint64", true],
+    ["L970", "uint64", true],
+    ["L783", "(inline-array power-game-cell)", 256],
+    ["L732", "(inline-array talker-speech-class)", 50],
+    ["L687", "vector"],
+    ["L841", "uint64", true],
+    ["L838", "uint64", true],
+    ["L848", "uint64", true],
+    ["L853", "uint64", true],
+    ["L846", "uint64", true],
+    ["L845", "uint64", true],
+    ["L847", "uint64", true],
+    ["L843", "uint64", true],
+    ["L837", "uint64", true],
+    ["L855", "uint64", true],
+    ["L854", "uint64", true],
+    ["L852", "uint64", true],
+    ["L730", "vector"],
+    ["L729", "vector"],
+    ["L728", "vector"],
+    ["L727", "vector"],
+    ["L836", "uint64", true],
+    ["L842", "uint64", true],
+    ["L840", "uint64", true],
+    ["L844", "uint64", true],
+    ["L849", "uint64", true],
+    ["L839", "uint64", true],
+    ["L851", "uint64", true],
+    ["L850", "uint64", true]
+  ],
+  "gun-dummy": [
+    ["L463", "uint64", true],
+    ["L460", "uint64", true],
+    ["L458", "uint64", true],
+    ["L462", "uint64", true],
+    ["L461", "uint64", true],
+    ["L457", "uint64", true],
+    ["L459", "uint64", true],
+    ["L306", "vector"],
+    ["L312", "uint64", true],
+    ["L323", "uint64", true],
+    ["L322", "uint64", true],
+    ["L316", "uint64", true],
+    ["L317", "uint64", true],
+    ["L321", "uint64", true],
+    ["L315", "uint64", true],
+    ["L314", "uint64", true],
+    ["L310", "uint64", true],
+    ["L313", "uint64", true],
+    ["L311", "uint64", true],
+    ["L319", "uint64", true],
+    ["L318", "uint64", true],
+    ["L320", "uint64", true]
+  ],
+  "gungame-obs": [
+    ["L91", "uint64", true],
+    ["L79", "uint64", true],
+    ["L76", "uint64", true],
+    ["L78", "uint64", true],
+    ["L77", "uint64", true]
+  ],
+  "gungame-data": [["L2388", "uint64", true]],
+  "gungame-manager": [
+    ["L580", "uint64", true],
+    ["L579", "uint64", true],
+    ["L583", "uint64", true],
+    ["L584", "uint64", true],
+    ["L582", "uint64", true],
+    ["L581", "uint64", true],
+    ["L396", "uint64", true]
+  ],
+  "mood-funcs2": [
+    ["L115", "uint64", true],
+    ["L116", "uint64", true],
+    ["L117", "uint64", true],
+    ["L118", "uint64", true],
+    ["L120", "uint64", true],
+    ["L119", "uint64", true],
+    ["L121", "uint64", true]
+  ],
+  "blow-tower-shared": [["L34", "(inline-array talker-speech-class)", 60]],
+  "blow-tower-obs": [
+    ["L428", "uint64", true],
+    ["L437", "uint64", true],
+    ["L436", "uint64", true],
+    ["L435", "uint64", true],
+    ["L431", "uint64", true],
+    ["L433", "uint64", true],
+    ["L429", "uint64", true],
+    ["L434", "uint64", true],
+    ["L430", "uint64", true],
+    ["L438", "uint64", true],
+    ["L432", "uint64", true],
+    ["L322", "uint64", true],
+    ["L326", "uint64", true],
+    ["L323", "uint64", true],
+    ["L321", "uint64", true],
+    ["L320", "uint64", true],
+    ["L324", "uint64", true],
+    ["L309", "attack-info"],
+    ["L325", "uint64", true],
+    ["L303", "attack-info"],
+    ["L302", "attack-info"],
+    ["L304", "attack-info"]
+  ],
+  "blow-tower-obs2": [
+    ["L681", "uint64", true],
+    ["L683", "uint64", true],
+    ["L686", "uint64", true],
+    ["L684", "uint64", true],
+    ["L680", "uint64", true],
+    ["L685", "uint64", true],
+    ["L682", "uint64", true],
+    ["L565", "uint64", true],
+    ["L563", "uint64", true],
+    ["L562", "uint64", true],
+    ["L559", "uint64", true],
+    ["L571", "uint64", true],
+    ["L560", "uint64", true],
+    ["L572", "uint64", true],
+    ["L574", "uint64", true],
+    ["L570", "uint64", true],
+    ["L577", "uint64", true],
+    ["L585", "uint64", true],
+    ["L549", "vector"],
+    ["L548", "vector"],
+    ["L547", "vector"],
+    ["L567", "uint64", true],
+    ["L587", "uint64", true],
+    ["L566", "uint64", true],
+    ["L579", "uint64", true],
+    ["L564", "uint64", true],
+    ["L581", "uint64", true],
+    ["L573", "uint64", true],
+    ["L569", "uint64", true],
+    ["L580", "uint64", true],
+    ["L576", "uint64", true],
+    ["L521", "attack-info"],
+    ["L586", "uint64", true],
+    ["L578", "uint64", true],
+    ["L575", "uint64", true],
+    ["L582", "uint64", true],
+    ["L583", "uint64", true],
+    ["L558", "uint64", true],
+    ["L584", "uint64", true],
+    ["L561", "uint64", true],
+    ["L533", "attack-info"],
+    ["L568", "uint64", true]
+  ],
+  "cty-blow-tower": [
+    ["L1038", "bombbot-path"],
+    ["L1283", "uint64", true],
+    ["L1288", "uint64", true],
+    ["L1281", "uint64", true],
+    ["L1278", "uint64", true],
+    ["L1282", "uint64", true],
+    ["L1285", "uint64", true],
+    ["L1287", "uint64", true],
+    ["L1279", "uint64", true],
+    ["L1286", "uint64", true],
+    ["L1284", "uint64", true],
+    ["L1280", "uint64", true],
+    ["L1277", "uint64", true],
+    ["L1051", "uint64", true],
+    ["L1054", "uint64", true],
+    ["L998", "(inline-array kg-squad-member-settings)", 4],
+    ["L1053", "uint64", true],
+    ["L1034", "(pointer float)", 4],
+    ["L1043", "uint64", true],
+    ["L1046", "uint64", true],
+    ["L1048", "uint64", true],
+    ["L1057", "uint64", true],
+    ["L1045", "uint64", true],
+    ["L1052", "uint64", true],
+    ["L1042", "uint64", true],
+    ["L1055", "uint64", true],
+    ["L1050", "uint64", true],
+    ["L1058", "uint64", true],
+    ["L1041", "uint64", true],
+    ["L1059", "uint64", true],
+    ["L1049", "uint64", true],
+    ["L1010", "attack-info"],
+    ["L1047", "uint64", true],
+    ["L1056", "uint64", true],
+    ["L1044", "uint64", true]
+  ],
+  "protect-gunship": [
+    ["L786", "uint64", true],
+    ["L783", "uint64", true],
+    ["L780", "uint64", true],
+    ["L784", "uint64", true],
+    ["L779", "uint64", true],
+    ["L778", "uint64", true],
+    ["L785", "uint64", true],
+    ["L782", "uint64", true],
+    ["L776", "uint64", true],
+    ["L781", "uint64", true],
+    ["L777", "uint64", true],
+    ["L604", "uint64", true],
+    ["L599", "uint64", true],
+    ["L510", "(inline-array gunship-info)", 2],
+    ["L608", "uint64", true],
+    ["L603", "uint64", true],
+    ["L614", "uint64", true],
+    ["L464", "attack-info"],
+    ["L463", "attack-info"],
+    ["L612", "uint64", true],
+    ["L615", "uint64", true],
+    ["L601", "uint64", true],
+    ["L598", "uint64", true],
+    ["L602", "uint64", true],
+    ["L610", "uint64", true],
+    ["L606", "uint64", true],
+    ["L611", "uint64", true],
+    ["L600", "uint64", true],
+    ["L609", "uint64", true],
+    ["L607", "uint64", true],
+    ["L613", "uint64", true],
+    ["L605", "uint64", true],
+    ["L456", "attack-info"]
+  ],
+  "cty-protect": [["L142", "attack-info"]],
+  "assault-shared": [["L1", "(inline-array talker-speech-class)", 22]],
+  "assault-path": [
+    ["L34", "assault-room"],
+    ["L33", "assault-room-collision"],
+    ["L29", "assault-room"],
+    ["L28", "assault-room-collision"],
+    ["L23", "assault-room"],
+    ["L22", "assault-room-collision"],
+    ["L17", "assault-room"],
+    ["L16", "assault-room-collision"],
+    ["L11", "assault-room"],
+    ["L10", "assault-room-collision"]
+  ],
+  "assault-enemies": [
+    ["L333", "attack-info"],
+    ["L332", "attack-info"]
+  ],
+  "precurd-obs": [
+    ["L610", "uint64", true],
+    ["L612", "uint64", true],
+    ["L613", "uint64", true],
+    ["L611", "uint64", true],
+    ["L544", "uint64", true],
+    ["L543", "uint64", true],
+    ["L547", "uint64", true],
+    ["L542", "uint64", true],
+    ["L541", "uint64", true],
+    ["L548", "uint64", true],
+    ["L356", "attack-info"],
+    ["L549", "uint64", true],
+    ["L550", "uint64", true],
+    ["L545", "uint64", true],
+    ["L546", "uint64", true]
+  ],
+  "precura-obs": [
+    ["L734", "uint64", true],
+    ["L728", "uint64", true],
+    ["L729", "uint64", true],
+    ["L732", "uint64", true],
+    ["L731", "uint64", true],
+    ["L730", "uint64", true],
+    ["L725", "uint64", true],
+    ["L735", "uint64", true],
+    ["L726", "uint64", true],
+    ["L727", "uint64", true],
+    ["L733", "uint64", true],
+    ["L608", "uint64", true],
+    ["L620", "uint64", true],
+    ["L618", "uint64", true],
+    ["L617", "uint64", true],
+    ["L623", "uint64", true],
+    ["L610", "uint64", true],
+    ["L605", "uint64", true],
+    ["L611", "uint64", true],
+    ["L621", "uint64", true],
+    ["L614", "uint64", true],
+    ["L613", "uint64", true],
+    ["L606", "uint64", true],
+    ["L619", "uint64", true],
+    ["L607", "uint64", true],
+    ["L577", "attack-info"],
+    ["L576", "attack-info"],
+    ["L622", "uint64", true],
+    ["L432", "vector"],
+    ["L431", "vector"],
+    ["L430", "vector"],
+    ["L609", "uint64", true],
+    ["L612", "uint64", true],
+    ["L615", "uint64", true],
+    ["L616", "uint64", true]
+  ],
+  "precura-obs2": [
+    ["L499", "uint64", true],
+    ["L489", "uint64", true],
+    ["L603", "uint64", true],
+    ["L602", "uint64", true],
+    ["L597", "uint64", true],
+    ["L596", "uint64", true],
+    ["L598", "uint64", true],
+    ["L601", "uint64", true],
+    ["L595", "uint64", true],
+    ["L600", "uint64", true],
+    ["L599", "uint64", true],
+    ["L484", "uint64", true],
+    ["L488", "uint64", true],
+    ["L491", "uint64", true],
+    ["L496", "uint64", true],
+    ["L494", "uint64", true],
+    ["L498", "uint64", true],
+    ["L394", "attack-info"],
+    ["L495", "uint64", true],
+    ["L493", "uint64", true],
+    ["L492", "uint64", true],
+    ["L486", "uint64", true],
+    ["L497", "uint64", true],
+    ["L487", "uint64", true],
+    ["L485", "uint64", true],
+    ["L490", "uint64", true],
+    ["L446", "vector"],
+    ["L416", "attack-info"],
+    ["L415", "attack-info"]
+  ],
+  "neon-baron-part": [
+    ["L592", "uint64", true],
+    ["L579", "(pointer uint64)", 1],
+    ["L83", "(pointer uint32)", 40],
+    ["L384", "vector4w"],
+    ["L385", "vector4w"],
+    ["L386", "vector4w"],
+    ["L387", "vector4w"],
+    ["L388", "vector4w"],
+    ["L389", "vector4w"],
+    ["L390", "vector4w"],
+    ["L391", "vector4w"],
+    ["L392", "vector4w"],
+    ["L393", "vector4w"],
+    ["L394", "vector4w"],
+    ["L395", "vector4w"],
+    ["L396", "vector4w"],
+    ["L397", "vector4w"],
+    ["L398", "vector4w"],
+    ["L399", "vector4w"],
+    ["L400", "vector4w"],
+    ["L401", "vector4w"],
+    ["L402", "vector4w"],
+    ["L403", "vector4w"],
+    ["L404", "vector4w"],
+    ["L405", "vector4w"],
+    ["L406", "vector4w"],
+    ["L407", "vector4w"],
+    ["L408", "vector4w"],
+    ["L409", "vector4w"],
+    ["L410", "vector4w"],
+    ["L411", "vector4w"],
+    ["L412", "vector4w"],
+    ["L413", "vector4w"],
+    ["L414", "vector4w"],
+    ["L415", "vector4w"],
+    ["L416", "vector4w"],
+    ["L417", "vector4w"],
+    ["L418", "vector4w"],
+    ["L419", "vector4w"],
+    ["L420", "vector4w"],
+    ["L421", "vector4w"],
+    ["L422", "vector4w"],
+    ["L423", "vector4w"],
+    ["L424", "vector4w"],
+    ["L425", "vector4w"],
+    ["L426", "vector4w"],
+    ["L427", "vector4w"],
+    ["L428", "vector4w"],
+    ["L429", "vector4w"],
+    ["L430", "vector4w"],
+    ["L431", "vector4w"],
+    ["L432", "vector4w"],
+    ["L433", "vector4w"],
+    ["L434", "vector4w"],
+    ["L435", "vector4w"],
+    ["L436", "vector4w"],
+    ["L437", "vector4w"],
+    ["L438", "vector4w"],
+    ["L439", "vector4w"],
+    ["L440", "vector4w"],
+    ["L441", "vector4w"],
+    ["L442", "vector4w"],
+    ["L443", "vector4w"],
+    ["L444", "vector4w"],
+    ["L445", "vector4w"],
+    ["L446", "vector4w"],
+    ["L447", "vector4w"],
+    ["L448", "vector4w"],
+    ["L449", "vector4w"],
+    ["L450", "vector4w"],
+    ["L451", "vector4w"],
+    ["L452", "vector4w"],
+    ["L453", "vector4w"],
+    ["L454", "vector4w"],
+    ["L455", "vector4w"],
+    ["L456", "vector4w"],
+    ["L457", "vector4w"],
+    ["L458", "vector4w"],
+    ["L459", "vector4w"],
+    ["L460", "vector4w"],
+    ["L461", "vector4w"],
+    ["L462", "vector4w"],
+    ["L463", "vector4w"],
+    ["L464", "vector4w"],
+    ["L465", "vector4w"],
+    ["L466", "vector4w"],
+    ["L467", "vector4w"],
+    ["L468", "vector4w"],
+    ["L469", "vector4w"],
+    ["L470", "vector4w"],
+    ["L471", "vector4w"],
+    ["L472", "vector4w"],
+    ["L473", "vector4w"],
+    ["L474", "vector4w"],
+    ["L475", "vector4w"],
+    ["L476", "vector4w"],
+    ["L477", "vector4w"],
+    ["L478", "vector4w"],
+    ["L479", "vector4w"],
+    ["L480", "vector4w"],
+    ["L481", "vector4w"],
+    ["L482", "vector4w"],
+    ["L483", "vector4w"],
+    ["L484", "vector4w"],
+    ["L485", "vector4w"],
+    ["L486", "vector4w"],
+    ["L487", "vector4w"],
+    ["L488", "vector4w"],
+    ["L489", "vector4w"],
+    ["L490", "vector4w"],
+    ["L491", "vector4w"],
+    ["L492", "vector4w"],
+    ["L493", "vector4w"],
+    ["L494", "vector4w"],
+    ["L495", "vector4w"],
+    ["L496", "vector4w"],
+    ["L497", "vector4w"],
+    ["L498", "vector4w"],
+    ["L499", "vector4w"],
+    ["L500", "vector4w"],
+    ["L501", "vector4w"],
+    ["L502", "vector4w"],
+    ["L503", "vector4w"],
+    ["L505", "vector4w"],
+    ["L507", "vector4w"],
+    ["L509", "vector4w"],
+    ["L511", "vector4w"],
+    ["L513", "vector4w"],
+    ["L514", "vector4w"],
+    ["L515", "vector4w"],
+    ["L516", "vector4w"],
+    ["L517", "vector4w"],
+    ["L518", "vector4w"],
+    ["L519", "vector4w"],
+    ["L520", "vector4w"],
+    ["L521", "vector4w"],
+    ["L522", "vector4w"],
+    ["L523", "vector4w"],
+    ["L524", "vector4w"],
+    ["L526", "vector4w"],
+    ["L527", "vector4w"],
+    ["L528", "vector4w"],
+    ["L529", "vector4w"],
+    ["L530", "vector4w"],
+    ["L531", "vector4w"],
+    ["L532", "vector4w"],
+    ["L533", "vector4w"],
+    ["L534", "vector4w"],
+    ["L535", "vector4w"],
+    ["L536", "vector4w"],
+    ["L537", "vector4w"],
+    ["L538", "vector4w"],
+    ["L539", "vector4w"],
+    ["L540", "vector4w"],
+    ["L541", "vector4w"],
+    ["L542", "vector4w"],
+    ["L543", "vector4w"],
+    ["L544", "vector4w"],
+    ["L545", "vector4w"],
+    ["L546", "vector4w"],
+    ["L547", "vector4w"],
+    ["L548", "vector4w"],
+    ["L549", "vector4w"],
+    ["L550", "vector4w"],
+    ["L551", "vector4w"],
+    ["L552", "vector4w"],
+    ["L554", "vector4w"],
+    ["L556", "vector4w"],
+    ["L558", "vector4w"],
+    ["L559", "vector4w"],
+    ["L560", "vector4w"],
+    ["L561", "vector4w"],
+    ["L562", "vector4w"],
+    ["L563", "vector4w"],
+    ["L564", "vector4w"],
+    ["L565", "vector4w"],
+    ["L566", "vector4w"],
+    ["L567", "vector4w"],
+    ["L568", "vector4w"],
+    ["L569", "vector4w"],
+    ["L570", "vector4w"],
+    ["L571", "vector4w"],
+    ["L572", "vector4w"],
+    ["L573", "vector4w"],
+    ["L574", "vector4w"],
+    ["L575", "vector4w"],
+    ["L576", "vector4w"],
+    ["L577", "vector4w"]
+  ],
+  "texture-anim": [
+    ["L198", "(pointer uint8)", 16],
+    ["L197", "(pointer uint16)", 48]
   ]
 }

decompiler/config/jak3/ntsc_v1/process_stack_size_overrides.jsonc

@@ -5,5 +5,7 @@
   "(method 11 elevator)": 1024,
   "scene-player-init": 1024,
   "task-manager-init-by-other": 2048,
-  "race-manager-init-by-other": 1024
+  "race-manager-init-by-other": 1024,
+  "neo-sat-shield-init-by-other": 64,
+  "bt-gun-manager-init-by-other": 256
 }

decompiler/config/jak3/ntsc_v1/stack_structures.jsonc

@@ -1,5 +1,5 @@
 {
-  "quaternion-smooth-seek!": [[16, "quaternion"]],
+  "quaternion-smooth-seek!": [[16, ["inline-array", "quaternion", 2]]],
   "eul->matrix": [[16, "vector"]],
   "segment-segment-find-intersection-xz": [[16, "vector"]],
   "vector-line-xz-distance-point!": [[16, "matrix"]],
@@ -1945,5 +1945,573 @@
     [16, ["array", "symbol", 10]],
     [64, "event-message-block"], // this memory is reused as both mc-slot-info and event-message-block
     [368, ["array", "symbol", 10]]
+  ],
+  "(method 9 nav-node)": [[48, ["inline-array", "vector", 2]]],
+  "(method 10 nav-node)": [[16, "vector"]],
+  "(method 9 nav-graph)": [[16, "matrix"]],
+  "(method 9 grid-info)": [[16, "vector"]],
+  "(method 11 grid-info)": [[16, "bounding-box"]],
+  "(method 13 grid-info)": [[16, "bounding-box"]],
+  "(method 9 city-level-info)": [
+    [16, "vis-ray"],
+    [192, "vector"],
+    [208, "vector"],
+    [224, "vector"],
+    [128, "vector"],
+    [144, "vector"]
+  ],
+  "(method 11 city-level-info)": [[16, "vector"]],
+  "(method 12 city-level-info)": [[16, ["inline-array", "vector", 7]]],
+  "(method 17 city-level-info)": [[16, ["inline-array", "vis-grid-pos", 2]]],
+  "(method 16 city-level-info)": [
+    [48, "vis-grid-box"],
+    [64, "vis-grid-box"]
+  ],
+  "(method 15 city-level-info)": [
+    [32, "vis-grid-box"],
+    [16, "vis-grid-box"]
+  ],
+  "(method 14 traffic-level-data)": [
+    [32, "vector"],
+    [48, "vector"],
+    [64, "vector"]
+  ],
+  "(method 18 city-level-info)": [[16, "bounding-box"]],
+  "(method 13 traffic-suppressor)": [
+    [16, "bounding-box"],
+    [48, "vector4w"]
+  ],
+  "(method 22 traffic-tracker)": [[16, "traffic-engine-stack-var0"]],
+  "(method 22 traffic-engine)": [[16, "traffic-danger-info"]],
+  "(method 43 traffic-engine)": [[16, ["array", "collide-shape", 64]]],
+  "(method 52 traffic-engine)": [[16, "vector"]],
+  "(method 54 traffic-engine)": [
+    [16, ["inline-array", "traffic-suppression-box", 2]],
+    [80, ["array", "collide-shape", 40]]
+  ],
+  "(method 53 traffic-engine)": [[16, ["array", "collide-shape", 40]]],
+  "(trans active formation-object)": [[48, "traffic-danger-info"]],
+  "(method 37 formation-object)": [[96, "vector"]],
+  "(method 38 formation-object)": [[16, "vector"]],
+  "(method 30 formation-object)": [[16, "msg-get-num-inactive"]],
+  "(method 204 citizen)": [
+    [16, "vector"],
+    [32, "iter-seg"]
+  ],
+  "(method 207 citizen)": [
+    [80, "vector"],
+    [32, "iter-seg"]
+  ],
+  "(method 202 citizen)": [
+    [64, "vector"],
+    [80, "vector"]
+  ],
+  "(method 160 citizen)": [[16, "vector"]],
+  "(method 203 citizen)": [
+    [80, "vector"],
+    [96, "vector"]
+  ],
+  "(method 45 formation-object)": [[16, "msg-get-inactive-objects"]],
+  "(trans move-to-vehicle civilian)": [
+    [32, "vector"],
+    [128, "vector"]
+  ],
+  "civilian-flee-post": [
+    [48, "vector"],
+    [80, "vector"]
+  ],
+  "civilian-avoid-danger-post": [[112, "vector"]],
+  "civilian-clear-path-post": [[96, "vector"]],
+  "(method 239 wlander)": [[48, "vector"]],
+  "(method 242 wlander)": [[32, "vector"]],
+  "(method 238 wlander)": [
+    [32, ["array", "collide-shape", 64]],
+    [288, "vector"]
+  ],
+  "(method 241 wlander)": [
+    [32, ["array", "collide-shape", 64]],
+    [288, "vector"]
+  ],
+  "(method 13 flee-info)": [
+    [48, "vector"],
+    [64, "vector"]
+  ],
+  "(method 15 rope-system)": [
+    [64, "vector"],
+    [80, "vector"],
+    [96, "vector"]
+  ],
+  "(enter rescue-start task-manager-desert-rescue)": [
+    [16, "desert-rescue-transport-init-params"]
+  ],
+  "(code board-vehicle wland-passenger)": [
+    [128, "vector"],
+    [176, "vector"]
+  ],
+  "(trans boarding wland-passenger)": [[32, "vector"]],
+  "(enter approaching-transport wland-passenger)": [[16, "vector"]],
+  "(method 231 wland-passenger)": [[16, "matrix"]],
+  "spawn-wlander": [[16, "wland-passenger-init-by-other-params"]],
+  "update-nav-sphere": [[64, "vector"]],
+  "compute-transport-approach-pt": [[48, "vector"]],
+  "(method 19 rope-prim-system)": [
+    [48, "matrix"],
+    [16, "vector"],
+    [32, "vector"],
+    [208, "rgbaf"],
+    [224, "rgbaf"]
+  ],
+  "(method 211 neo-sat)": [[16, "vector"]],
+  "(method 232 neo-sat)": [[32, "vector"]],
+  "(method 220 neo-sat)": [[16, "vector"]],
+  "(method 221 neo-sat)": [[32, "vector"]],
+  "(method 206 neo-sat)": [
+    [32, "vector"],
+    [48, "collide-query"]
+  ],
+  "(method 227 neo-sat)": [[16, "collide-query"]],
+  "spawn-neo-sat": [[16, "neo-sat-init-params"]],
+  "desert-rescue-transport-init-by-other": [
+    [16, "light-trail-tracker-spawn-params"]
+  ],
+  "(method 254 crimson-guard)": [
+    [16, "vector"],
+    [96, "vector"]
+  ],
+  "(method 255 crimson-guard)": [[64, "vector"]],
+  "(enter grenade-throw-post-reloading crimson-guard)": [
+    [16, "reload-grenade-spawn-params"]
+  ],
+  "(method 46 ff-squad-control)": [[112, "ff-squad-control-stack-var0"]],
+  "(method 16 ff-squad-control)": [[192, ["array", "float", 16]]],
+  "(method 45 ff-squad-control)": [[16, "vector"]],
+  "(method 258 crimson-guard)": [[16, "collide-query"]],
+  "(method 259 crimson-guard)": [
+    [96, "vector"],
+    [32, "vector"],
+    [48, "vector"],
+    [160, "vector"],
+    [176, "vector"]
+  ],
+  "(method 212 crimson-guard)": [[16, "shield-sphere-spawn-params"]],
+  "(method 266 crimson-guard)": [
+    [32, "vector"],
+    [656, "vector"],
+    [720, "vector"]
+  ],
+  "(code sitting vehicle-grenade)": [[16, "traffic-danger-info"]],
+  "(method 44 vehicle-grenade)": [[16, "traffic-danger-info"]],
+  "(method 31 vehicle-grenade)": [[16, "vector"]],
+  "(method 264 crimson-guard)": [[32, "vector"]],
+  "(method 78 h-bike-base)": [[16, "bike-stack-var0"]],
+  "(method 217 citizen-enemy)": [[32, ["array", "collide-shape", 64]]],
+  "(method 225 metalhead-predator)": [
+    [160, "vector"],
+    [176, "vector"],
+    [208, "vector"]
+  ],
+  "(trans close-attack roboguard-city)": [
+    [32, ["array", "collide-shape", 384]]
+  ],
+  "(code roll-chase roboguard-city)": [[16, "vector"]],
+  "spawn-roboguard-projectile": [[16, "projectile-init-by-other-params"]],
+  "(method 237 roboguard-city)": [[16, "vector"]],
+  "(code flip spydroid)": [[16, "vector"]],
+  "(trans fire flying-turret)": [[16, "ft-gun-info"]],
+  "(method 161 flying-turret)": [[16, "vector"]],
+  "(method 31 h-kg-pickup)": [[48, "vector"]],
+  "(method 11 flitter-spawner)": [[32, "traffic-object-spawn-params"]],
+  "(method 21 flitter-spawner)": [[16, "traffic-object-spawn-params"]],
+  "(code active task-manager-ctywide-bbush-get-to)": [
+    [160, "vector"],
+    [176, "matrix"]
+  ],
+  "(trans talking burning-bush)": [
+    [32, "vector"],
+    [64, "vector"]
+  ],
+  "(method 32 propa)": [[16, ["array", "collide-shape", 64]]],
+  "(event idle propa)": [
+    [16, "vector"],
+    [32, "vector"]
+  ],
+  "(method 23 parking-spot)": [
+    [16, "cquery-with-vec"],
+    [560, ["inline-array", "vector", 1]]
+  ],
+  "(method 24 parking-spot)": [[16, "cquery-with-vec"]],
+  "(method 11 security-wall)": [[32, "vector"]],
+  "(method 24 security-wall)": [[16, ["inline-array", "vector", 1]]],
+  "(trans menu burning-bush)": [[128, "vector"]],
+  "check-drop-level-ctyport-drop-userdata": [[16, "vector"]],
+  "(method 16 boat-manager)": [[16, "matrix"]],
+  "(method 31 boat-base)": [[16, "matrix"]],
+  "(method 30 boat-base)": [[96, ["inline-array", "vector", 6]]],
+  "(trans idle krimson-wall-break)": [[16, "matrix"]],
+  "(trans explode krimson-wall-break)": [[16, "vector"]],
+  "(method 35 htorpedo)": [[16, "light-trail-tracker-spawn-params"]],
+  "(enter start-mission ctyport-attack-manager)": [
+    [16, ["array", "symbol", 10]]
+  ],
+  "(method 49 ctyport-attack-manager)": [
+    [64, ["array", "collide-shape", 384]],
+    [32, "vector"]
+  ],
+  "(method 44 ctyport-attack-manager)": [
+    [16, "vector"],
+    [32, "vector"]
+  ],
+  "(method 45 ctyport-attack-manager)": [[16, "vector"]],
+  "(method 79 v-catapult)": [[16, ["inline-array", "quaternion", 3]]],
+  "(method 33 desert-jump-manager)": [[16, "vector"]],
+  "(code active desert-jump-manager)": [[96, ["array", "symbol", 10]]],
+  "(method 32 desert-jump-manager)": [
+    [16, "mystery-traffic-object-spawn-params0"]
+  ],
+  "(method 23 beam-generator)": [[16, "matrix"]],
+  "(trans active player-controller)": [
+    [32, "vector"],
+    [48, "vector"]
+  ],
+  "(code active desert-chase-ambush-manager)": [
+    [16, ["inline-array", "traffic-object-spawn-params", 1]]
+  ],
+  "(method 34 desert-chase-ambush-manager)": [
+    [16, "projectile-init-by-other-params"]
+  ],
+  "(method 33 desert-chase-ambush-manager)": [
+    [16, "mystery-traffic-object-spawn-params0"]
+  ],
+  "(method 32 desert-chase-ambush-manager)": [
+    [16, "marauder-init-by-other-params"]
+  ],
+  "(method 22 deschase-artifact)": [[16, "matrix"]],
+  "(method 12 deschase-vehicle-control)": [
+    [64, ["array", "collide-shape", 128]],
+    [16, "matrix"]
+  ],
+  "(method 32 desert-chase-chase-manager)": [
+    [16, "traffic-object-spawn-params"]
+  ],
+  "desert-chase-chase-intro-sequence": [[16, "player-controller-init-params"]],
+  "(method 32 bombbot-bomb)": [[592, "vector"]],
+  "bombbot-player-init-by-other": [[96, "vector"]],
+  "(trans hostile bombbot-player)": [[16, "vector"]],
+  "(post hostile bombbot-player)": [
+    [576, "vector"],
+    [16, "vector"],
+    [592, "vector"]
+  ],
+  "(method 197 bombbot-player)": [
+    [80, "matrix"],
+    [144, "vector"],
+    [720, "vector"],
+    [736, "vector"]
+  ],
+  "(method 196 bombbot-player)": [
+    [16, "vector"],
+    [32, "vector"],
+    [48, "collide-query"],
+    [592, ["array", "collide-shape", 64]],
+    [848, "vector"]
+  ],
+  "(method 139 bombbot-player)": [
+    [560, "vector"],
+    [576, "vector"],
+    [592, "vector"],
+    [608, "vector"],
+    [1168, "vector"],
+    [1216, "collide-query"]
+  ],
+  "(post hostile bomb-bot)": [[16, "vector"]],
+  "(trans hostile bomb-bot)": [
+    [16, "vector"],
+    [32, "vector"]
+  ],
+  "(method 196 bomb-bot)": [[16, ["array", "collide-shape", 64]]],
+  "(method 198 bombbot)": [
+    [48, "vector"],
+    [64, "vector"],
+    [144, "vector"]
+  ],
+  "bombbot-gun-callback": [
+    [32, "vector"],
+    [80, "vector"]
+  ],
+  "bombbot-gun-swivel-callback": [
+    [16, "vector"],
+    [80, "vector"]
+  ],
+  "bombbot-callback": [
+    [16, "vector"],
+    [32, "quaternion"],
+    [48, "vector"],
+    [64, "vector"]
+  ],
+  "(enter explode bombbot)": [
+    [112, "vector"],
+    [128, "vector"],
+    [144, "vector"],
+    [256, "vector"]
+  ],
+  "(trans drop-bombs bombbot)": [[32, "vector"]],
+  "(trans hostile bombbot)": [[32, "vector"]],
+  "(method 160 bombbot)": [[32, "vector"]],
+  "(method 197 bombbot)": [
+    [80, "matrix"],
+    [144, "vector"],
+    [720, "vector"],
+    [736, "vector"]
+  ],
+  "(method 203 bombbot)": [
+    [80, "vector"],
+    [176, "vector"],
+    [400, "vector"]
+  ],
+  "(method 204 bombbot)": [[80, "vector"]],
+  "(method 194 bombbot)": [
+    [608, "vector"],
+    [592, "vector"],
+    [624, "vector"],
+    [672, "vector"],
+    [720, "vector"]
+  ],
+  "(method 196 bombbot)": [[16, ["array", "collide-shape", 64]]],
+  "(method 25 bombbot-shot)": [[32, "vector"]],
+  "(enter blow-wall bombbot)": [[16, ["array", "symbol", 10]]],
+  "check-drop-level-ctysluma-drop-userdata": [[16, "vector"]],
+  "check-drop-level-ctyslumb-drop-userdata": [[16, "vector"]],
+  "(method 36 cty-missile-lure)": [[16, "vector"]],
+  "(method 36 cty-hijack-manager)": [[16, "cty-hijack-missile-init-params"]],
+  "(enter player-riding cty-hijack-manager)": [
+    [96, "cty-missile-lure-init-params"],
+    [144, "cty-missile-lure-init-params"]
+  ],
+  "(method 38 cty-hijack-manager)": [
+    [64, "vector"],
+    [80, "vector"]
+  ],
+  "(code wait ctyport-attack-manager-bbush)": [[96, ["array", "symbol", 10]]],
+  "check-drop-level-palroof-drop-userdata": [[16, "vector"]],
+  "check-drop-level-ctyfarmb-drop-userdata": [[16, "vector"]],
+  "check-drop-level-ctyfarma-drop-userdata": [[16, "vector"]],
+  "(method 11 com-power-box)": [
+    [16, "vector"],
+    [48, "vector"]
+  ],
+  "(enter active task-manager-city-destroy-grid)": [
+    [224, "enemy-init-by-other-params"]
+  ],
+  "(code exit-vehicle jinx)": [
+    [32, "vector"],
+    [64, "vector"]
+  ],
+  "(code board-vehicle jinx)": [[16, "vector"]],
+  "(trans move-to-vehicle jinx)": [
+    [32, "vector"],
+    [128, "vector"]
+  ],
+  "(post moribund cty-sniper-button)": [[16, "vector"]],
+  "(method 11 cty-sniper-turret)": [[32, "vector"]],
+  "sniper-turret-post": [[112, "vector"]],
+  "citysniper-draw-2d-hud": [
+    [16, ["inline-array", "vector4w", 4]],
+    [80, "vector4w"]
+  ],
+  "(enter lock cty-sniper-turret-reticle)": [[16, "vector"]],
+  "(enter impact cty-sniper-turret-shot)": [[16, "traffic-danger-info"]],
+  "(method 25 cty-sniper-turret-shot)": [[32, "vector"]],
+  "cty-sniper-turret-handler": [[96, "vector"]],
+  "king-pilot-trans": [[16, "matrix"]],
+  "(method 32 rubble-attack-manager)": [
+    [16, "vector"],
+    [592, "traj3d-params"],
+    [672, "vector"]
+  ],
+  "(method 30 power-game)": [[32, "vector"]],
+  "(method 44 power-game)": [[32, "vector"]],
+  "spt-func-ground-spark-bounce": [[16, "vector"]],
+  "gun-dummy-gun-aim-jmod-func": [[16, "vector"]],
+  "(method 38 gun-dummy-gun)": [[16, "primary-target-pos-vel"]],
+  "(method 29 gungame-manager)": [[16, "primary-target-pos-vel"]],
+  "(method 30 gungame-manager)": [
+    [16, ["inline-array", "primary-target-pos-vel", 2]]
+  ],
+  "(method 11 blow-tower-path-cursor)": [[16, "vector"]],
+  "(method 44 tower-flyer)": [[16, "flyer-projectile-params"]],
+  "(method 49 tower-flyer)": [
+    [48, "collide-query"],
+    [16, "vector"]
+  ],
+  "(method 41 bt-barrel)": [[32, ["array", "collide-shape", 384]]],
+  "(method 40 bt-barrel)": [[32, ["array", "collide-shape", 384]]],
+  "setup-bt-barrels": [
+    [32, "vector"],
+    [64, "vector"]
+  ],
+  "bt-barrel-ground-probe": [[16, "collide-query"]],
+  "(method 51 bt-mh-flyer)": [
+    [16, "vector"],
+    [112, "vector"]
+  ],
+  "(method 40 bt-missile)": [[112, "vector"]],
+  "(anon-function 16 blow-tower-obs2)": [[112, "quaternion"]],
+  "kg-bombbot-find-nearest-nav-mesh": [[16, "nav-poly"]],
+  "(method 47 bt-grunt)": [[16, "vector"]],
+  "(method 22 bt-gun-manager)": [[16, "vector"]],
+  "(method 36 bt-hellcat)": [
+    [16, "matrix"],
+    [80, "vector"],
+    [96, "vector"],
+    [112, "vector"],
+    [128, "vector"]
+  ],
+  "(method 43 bt-pickup)": [[16, "vector"]],
+  "(method 38 task-manager-blow-tower)": [[480, "vector"]],
+  "(method 18 bt-gun-manager)": [
+    [16, "vector"],
+    [32, "vector"],
+    [48, "vector"],
+    [64, "vector"],
+    [208, "vector"]
+  ],
+  "(method 32 bt-vehicle)": [
+    [16, "vector"],
+    [32, "vector"]
+  ],
+  "(method 47 tower-flyer)": [
+    [32, "vector"],
+    [48, "vector"],
+    [144, "vector"]
+  ],
+  "(method 20 bt-gun-manager)": [
+    [48, "vector"],
+    [64, "vector"],
+    [96, "vector"],
+    [144, "vector"],
+    [160, "vector"],
+    [176, "vector"],
+    [192, "vector"],
+    [800, "vector"],
+    [816, "vector"],
+    [848, "vector"],
+    [896, "vector"],
+    [912, "vector"],
+    [928, "vector"],
+    [944, "vector"],
+    [1552, "vector"],
+    [1568, "vector"],
+    [1600, "vector"],
+    [1648, "vector"],
+    [1664, "vector"],
+    [1680, "vector"],
+    [1696, "vector"]
+  ],
+  "(method 19 bt-gun-manager)": [
+    [80, "vector"],
+    [32, "vector"],
+    [48, "vector"]
+  ],
+  "(method 31 bt-pickup)": [[128, "vector"]],
+  "(method 31 bt-hellcat)": [
+    [176, "vector"],
+    [192, "vector"],
+    [240, "vector"],
+    [320, "vector"],
+    [336, "vector"]
+  ],
+  "(method 50 bt-hellcat)": [
+    [16, "vector"],
+    [32, "quaternion"]
+  ],
+  "(method 47 bt-hellcat)": [
+    [80, "vector"],
+    [32, "vector"],
+    [96, "matrix"]
+  ],
+  "(method 48 bt-hellcat)": [
+    [32, "vector"],
+    [48, "vector"]
+  ],
+  "(method 29 bt-hellcat)": [[112, "quaternion"]],
+  "(method 21 bt-gun-manager)": [
+    [320, "collide-query"],
+    [160, "vector"]
+  ],
+  "draw-path-blow-curve": [[16, "matrix"]],
+  "(method 22 trail-graph)": [[16, "trail-vis-work"]],
+  "(method 13 trail-graph)": [[16, ["inline-array", "vector", 3]]],
+  "(method 16 trail-graph)": [[16, ["inline-array", "vector", 2]]],
+  "(method 17 trail-graph)": [[16, "trail-conn-search"]],
+  "(trans explode gunship-missile)": [[32, ["array", "collide-shape", 384]]],
+  "(trans spinning-laser protect-gunship)": [
+    [32, "vector"],
+    [64, "vector"]
+  ],
+  "find-reposition-pt": [
+    [16, "vector"],
+    [96, "vector"],
+    [112, "vector"]
+  ],
+  "(method 50 protect-gunship)": [[16, "vector"]],
+  "find-nearest-nav-mesh-protect": [[16, "nav-poly"]],
+  "(method 37 protect-gunship)": [[752, "vector"]],
+  "(method 36 protect-gunship)": [[16, "gunship-missile-init-params"]],
+  "(method 39 protect-gunship)": [[16, "protect-gunship-enemy-spawn-params"]],
+  "(method 55 protect-gunship)": [[16, "collide-query"]],
+  "(method 21 prot-crate)": [[16, "vector"]],
+  "(method 36 task-manager-protect-hq)": [[16, "protect-gunship-init-params"]],
+  "spawn-protect-enemy": [[16, "traffic-object-spawn-params"]],
+  "(method 35 task-manager-protect-hq)": [[16, "traffic-object-spawn-params"]],
+  "(trans hostile assault-bombbot)": [[32, "vector"]],
+  "(enter explode assault-bombbot)": [[112, ["array", "collide-shape", 384]]],
+  "(code roll-initial assault-roboguard)": [[16, "vector"]],
+  "(method 31 assault-bombbot-shot)": [
+    [16, "light-trail-tracker-spawn-params"]
+  ],
+  "(code active task-manager-city-port-assault)": [
+    [16, "player-controller-init-params"]
+  ],
+  "(method 40 task-manager-city-port-assault)": [[16, ["array", "symbol", 10]]],
+  "(enter combat task-manager-city-port-assault)": [
+    [16, "player-controller-init-params"]
+  ],
+  "(code player-entrance task-manager-city-port-assault)": [
+    [16, "player-controller-init-params"]
+  ],
+  "(enter transition task-manager-city-port-assault)": [
+    [16, "player-controller-init-params"]
+  ],
+  "(method 47 task-manager-city-port-assault)": [[64, "vector"]],
+  "(method 48 task-manager-city-port-assault)": [[64, "vector"]],
+  "(method 41 task-manager-city-port-assault)": [[48, "vector"]],
+  "(trans idle precur-generator)": [[112, "vector"]],
+  "(method 23 precur-generator-d)": [[16, "vector"]],
+  "(event idle precur-door-d)": [[16, "attack-info"]],
+  "(method 23 precur-door-c)": [[112, "vector"]],
+  "(method 11 precur-path)": [[16, "vector"]],
+  "precur-bomb-handler": [[16, "attack-info"]],
+  "(event idle precur-bomb-spawner)": [[16, "attack-info"]],
+  "(method 21 precur-bomb-spawner)": [[16, "vector"]],
+  "precur-bomb-init-by-other": [[16, "vector"]],
+  "(method 25 precur-laser-beam)": [[16, "collide-query"]],
+  "(method 24 precur-laser-beam)": [[16, "collide-query"]],
+  "process-drawable-shock-wave-effect": [
+    [16, "matrix"],
+    [80, "collide-query"]
+  ],
+  "(method 22 precur-path)": [[16, "collide-query"]],
+  "precur-bridge-path-2nd-trigger": [[16, "vector"]],
+  "precur-bridge-blocks-trigger": [[16, "vector"]],
+  "precur-bridge-path-trigger": [[16, "vector"]],
+  "(post idle precur-bridge-blocks-break)": [
+    [16, "vector"],
+    [32, "vector"]
+  ],
+  "texture-anim-layer-draw": [
+    [16, "matrix"],
+    [80, "matrix"],
+    [144, "matrix"],
+    [208, "matrix"]
   ]
 }

decompiler/config/jak3/ntsc_v1/var_names.jsonc

@@ -2074,5 +2074,66 @@
     "vars": {
       "s4-0": ["sname", "sound-name"]
     }
+  },
+  "symbol->city-target-type": {
+    "vars": {
+      "v0-0": ["type", "city-target-type"]
+    }
+  },
+  "city-target-type->traffic-object-type": {
+    "vars": {
+      "v0-0": ["type", "traffic-type"]
+    }
+  },
+  "city-target-type->cty-faction": {
+    "vars": {
+      "v0-0": ["type", "cty-faction"]
+    }
+  },
+  "map-symbol-to-nav-territory-type": {
+    "vars": {
+      "v0-0": ["type", "nav-territory-type"]
+    }
+  },
+  "guard-type->city-target-type": {
+    "vars": {
+      "v0-0": ["type", "city-target-type"]
+    }
+  },
+  "(code die neo-sat)": {
+    "vars": {
+      "gp-0": ["hand", "handle"]
+    }
+  },
+  "(method 48 task-manager-desert-rescue)": {
+    "vars": {
+      "f30-0": ["f30-0", "float"]
+    }
+  },
+  "(method 31 gun-dummy)": {
+    "vars": {
+      "f0-0": "total-time",
+      "v1-0": "curr-frame"
+    }
+  },
+  "symbol->gun-pickup-type": {
+    "vars": {
+      "v0-0": ["type", "pickup-type"]
+    }
+  },
+  "symbol->faction-type": {
+    "vars": {
+      "v0-0": ["type", "assault-cmd-faction-type"]
+    }
+  },
+  "symbol->speech-type": {
+    "vars": {
+      "v0-0": ["type", "assault-cmd-speech-type"]
+    }
+  },
+  "symbol->assault-target-type": {
+    "vars": {
+      "v0-0": ["type", "assault-target-type"]
+    }
   }
 }

decompiler/data/tpage.cpp

@@ -515,8 +515,44 @@ TPageResultStats process_tpage(ObjectFileData& data,
           // currently not needed.
           break;
         case int(PSM::PSMT8):
-          ASSERT(tex.clutpsm == int(CPSM::PSMCT32));
-          {
+          if (tex.clutpsm == (int)CPSM::PSMCT16) {
+            // will store output pixels, rgba (8888)
+            std::vector<u8> index_out;
+
+            // width is like the TEX0 register, in 64 texel units.
+            // not sure what the other widths are yet.
+            int read_width = 64 * tex.width[0];
+
+            // loop over pixels in output texture image
+            for (int y = 0; y < tex.h; y++) {
+              for (int x = 0; x < tex.w; x++) {
+                // read as the PSMT8 type. The dest field tells us a block offset.
+                auto addr8 = psmt8_addr(x, y, read_width) + tex.dest[0] * 256;
+                u8 value = *(u8*)(vram.data() + addr8);
+                index_out.push_back(value);
+              }
+            }
+            std::array<math::Vector4<u8>, 256> unscrambled_clut{};
+            for (int i = 0; i < 256; i++) {
+              u32 clut_chunk = i / 16;
+              u32 off_in_chunk = i % 16;
+              u8 clx = 0, cly = 0;
+              if (clut_chunk & 1) {
+                clx = 8;
+              }
+              cly = (clut_chunk >> 1) * 2;
+              if (off_in_chunk >= 8) {
+                off_in_chunk -= 8;
+                cly++;
+              }
+              clx += off_in_chunk;
+              u32 clut_addr = psmct16_addr(clx, cly, 64) + tex.clutdest * 256;
+              memcpy(&unscrambled_clut[i], vram.data() + clut_addr, 4);
+            }
+            texture_db.add_index_texture(texture_page.id, tex_id, index_out, unscrambled_clut,
+                                         tex.w, tex.h, tex.name, texture_page.name, level_names);
+            stats.successful_textures++;
+          } else if (tex.clutpsm == (int)PSM::PSMCT32) {
             // will store output pixels, index (u8)
             std::vector<u8> index_out;
 
@@ -555,10 +591,12 @@ TPageResultStats process_tpage(ObjectFileData& data,
             texture_db.add_index_texture(texture_page.id, tex_id, index_out, unscrambled_clut,
                                          tex.w, tex.h, tex.name, texture_page.name, level_names);
             stats.successful_textures++;
+          } else {
+            ASSERT_NOT_REACHED();
           }
           break;
         default:
-          lg::die("Animated texture {} format {}\n", tex.name, tex.psm);
+          lg::die("Animated texture {} format {} clut {}\n", tex.name, tex.psm, tex.clutpsm);
       }
     }
 

decompiler/decompilation_process.cpp

@@ -0,0 +1,284 @@
+#include "decompilation_process.h"
+
+#include <string>
+#include <vector>
+
+#include "config.h"
+
+#include "common/log/log.h"
+#include "common/util/Timer.h"
+#include "common/util/os.h"
+
+#include "Disasm/OpcodeInfo.h"
+#include "ObjectFile/ObjectFileDB.h"
+#include "data/streamed_audio.h"
+#include "level_extractor/extract_level.h"
+
+int run_decompilation_process(decompiler::Config config,
+                              const fs::path& in_folder,
+                              const fs::path& out_folder,
+                              const bool minimal_for_extractor) {
+  using namespace decompiler;
+  Timer decomp_timer;
+
+  lg::info("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024));
+
+  init_opcode_info();
+
+  lg::info("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024));
+
+  std::vector<fs::path> dgos, objs, strs, tex_strs, art_strs;
+  if (minimal_for_extractor) {
+    // TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already
+    // happen?)
+    for (const auto& dgo_name : config.dgo_names) {
+      std::string common_name = "GAME.CGO";
+      if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
+        // ends in DGO, it's a level
+        dgos.push_back(in_folder / dgo_name);
+      } else if (dgo_name.length() >= common_name.length() &&
+                 dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
+        // it's COMMON.CGO, we need that too.
+        dgos.push_back(in_folder / dgo_name);
+      }
+    }
+  } else {
+    for (const auto& dgo_name : config.dgo_names) {
+      dgos.push_back(in_folder / dgo_name);
+    }
+  }
+
+  if (minimal_for_extractor) {
+    // TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already
+    // happen?)
+    for (const auto& obj_name : config.object_file_names) {
+      if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
+        // ends in TXT
+        objs.push_back(in_folder / obj_name);
+      }
+    }
+  } else {
+    for (const auto& obj_name : config.object_file_names) {
+      objs.push_back(in_folder / obj_name);
+    }
+  }
+
+  if (!minimal_for_extractor) {
+    for (const auto& str_name : config.str_file_names) {
+      strs.push_back(in_folder / str_name);
+    }
+  }
+
+  for (const auto& str_name : config.str_texture_file_names) {
+    tex_strs.push_back(in_folder / str_name);
+  }
+
+  for (const auto& str_name : config.str_art_file_names) {
+    art_strs.push_back(in_folder / str_name);
+  }
+
+  lg::info("[Mem] After config read: {} MB", get_peak_rss() / (1024 * 1024));
+
+  // build file database
+  lg::info("Setting up object file DB...");
+  ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs,
+                  config);
+
+  // Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB
+  // as this is another silent error that can be confusing
+  if (!config.allowed_objects.empty()) {
+    for (const auto& expected_obj : config.allowed_objects) {
+      if (db.obj_files_by_name.count(expected_obj) == 0) {
+        // TODO - this is wrong for jak1, fix eventually as this is now done in 3 places
+        lg::error(
+            "Expected to find '{}' in the ObjectFileDB but did not. Check "
+            "./decompiler/config/{}/inputs.jsonc",
+            expected_obj, config.game_name);
+        return 1;
+      }
+    }
+  }
+
+  lg::info("[Mem] After DB setup: {} MB", get_peak_rss() / (1024 * 1024));
+
+  // write out DGO file info
+  file_util::create_dir_if_needed(out_folder);
+  file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing());
+  // write out object file map (used for future decompilations, if desired)
+  file_util::write_text_file(out_folder / "obj.txt",
+                             db.generate_obj_listing(config.merged_objects));
+
+  // dump raw objs
+  if (config.dump_objs) {
+    auto path = out_folder / "raw_obj";
+    file_util::create_dir_if_needed(path);
+    db.dump_raw_objects(path);
+  }
+
+  // process files (required for all analysis)
+  db.process_link_data(config);
+  lg::info("[Mem] After link data: {} MB", get_peak_rss() / (1024 * 1024));
+  db.find_code(config);
+  db.process_labels();
+  lg::info("[Mem] After code: {} MB", get_peak_rss() / (1024 * 1024));
+
+  // top level decompile (do this before printing asm so we get function names)
+  if (config.find_functions) {
+    db.ir2_top_level_pass(config);
+  }
+
+  // print disassembly
+  if (config.disassemble_code || config.disassemble_data) {
+    db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code,
+                         config.write_hex_near_instructions);
+  }
+
+  if (config.process_art_groups) {
+    db.extract_art_info();
+    // dump art info to json if requested
+    if (config.dump_art_group_info) {
+      auto ag_file_name = out_folder / "dump" / "art-group-info.min.json";
+      nlohmann::json ag_json = db.dts.art_group_info;
+      file_util::create_dir_if_needed_for_file(ag_file_name);
+      file_util::write_text_file(ag_file_name, ag_json.dump(-1));
+      lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string());
+    }
+    if (config.dump_joint_geo_info) {
+      auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json";
+      nlohmann::json jg_json = db.dts.jg_info;
+      file_util::create_dir_if_needed_for_file(jg_file_name);
+      file_util::write_text_file(jg_file_name, jg_json.dump(-1));
+      lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string());
+    }
+  } else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) {
+    // process art groups (used in decompilation)
+    // - if the config has a path to the art info dump, just use that
+    // - otherwise (or if we want to dump it fresh) extract it
+    if (!config.art_group_info_dump.empty()) {
+      db.dts.art_group_info = config.art_group_info_dump;
+    }
+    if (!config.jg_info_dump.empty()) {
+      db.dts.jg_info = config.jg_info_dump;
+    }
+  } else {
+    lg::error("`process_art_groups` was false and no art-group-info dump was provided!");
+    return 1;
+  }
+
+  if (config.process_tpages && !config.texture_info_dump.empty()) {
+    db.dts.textures = config.texture_info_dump;
+  }
+
+  // main decompile.
+  if (config.decompile_code) {
+    db.analyze_functions_ir2(out_folder, config, {}, {}, {});
+  }
+
+  if (config.generate_all_types) {
+    ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types");
+    db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file,
+                             config.hacks.types_with_bad_inspect_methods);
+  }
+
+  lg::info("[Mem] After decomp: {} MB", get_peak_rss() / (1024 * 1024));
+
+  // write out all symbols
+  file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types());
+
+  // write art groups
+  if (config.process_art_groups) {
+    db.dump_art_info(out_folder);
+  }
+
+  if (config.hexdump_code || config.hexdump_data) {
+    db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code);
+  }
+
+  // data stuff
+  if (config.write_scripts) {
+    db.find_and_write_scripts(out_folder);
+  }
+
+  // ensure asset dir exists
+  file_util::create_dir_if_needed(out_folder / "assets");
+
+  if (config.process_game_text) {
+    auto result = db.process_game_text_files(config);
+    if (!result.empty()) {
+      file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
+    }
+  }
+
+  lg::info("[Mem] After text: {} MB", get_peak_rss() / (1024 * 1024));
+
+  if (config.process_subtitle_text || config.process_subtitle_images) {
+    auto result = db.process_all_spool_subtitles(
+        config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : "");
+    if (!result.empty()) {
+      file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result);
+    }
+  }
+
+  lg::info("[Mem] After spool handling: {} MB", get_peak_rss() / (1024 * 1024));
+
+  TextureDB tex_db;
+  if (config.process_tpages || config.levels_extract) {
+    auto textures_out = out_folder / "textures";
+    auto dump_out = out_folder / "import";
+    file_util::create_dir_if_needed(textures_out);
+    auto result = db.process_tpages(tex_db, textures_out, config, dump_out);
+    if (!result.empty() && config.process_tpages) {
+      file_util::write_text_file(textures_out / "tpage-dir.txt", result);
+      file_util::write_text_file(textures_out / "tex-remap.txt",
+                                 tex_db.generate_texture_dest_adjustment_table());
+    }
+    if (config.dump_tex_info) {
+      auto texture_file_name = out_folder / "dump" / "tex-info.min.json";
+      nlohmann::json texture_json = db.dts.textures;
+      file_util::create_dir_if_needed_for_file(texture_file_name);
+      file_util::write_text_file(texture_file_name, texture_json.dump(-1));
+      lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string());
+    }
+  }
+
+  lg::info("[Mem] After textures: {} MB", get_peak_rss() / (1024 * 1024));
+
+  // Merge textures before replacing them, in other words, replacements take priority
+  auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" /
+                            game_version_names[config.game_version] / "texture_merges";
+  if (fs::exists(texture_merge_path)) {
+    tex_db.merge_textures(texture_merge_path);
+  }
+
+  auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" /
+                           game_version_names[config.game_version] / "texture_replacements";
+  if (fs::exists(replacements_path)) {
+    tex_db.replace_textures(replacements_path);
+  }
+
+  if (config.process_game_count) {
+    auto result = db.process_game_count_file();
+    if (!result.empty()) {
+      file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
+    }
+  }
+
+  if (config.levels_extract) {
+    auto level_out_path =
+        file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
+    file_util::create_dir_if_needed(level_out_path);
+    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
+  }
+
+  lg::info("[Mem] After extraction: {} MB", get_peak_rss() / (1024 * 1024));
+
+  if (config.rip_streamed_audio) {
+    auto streaming_audio_out = out_folder / "audio";
+    file_util::create_dir_if_needed(streaming_audio_out);
+    process_streamed_audio(config, streaming_audio_out, in_folder,
+                           config.streamed_audio_file_names);
+  }
+
+  lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds());
+  return 0;
+}

decompiler/decompilation_process.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "config.h"
+
+int run_decompilation_process(decompiler::Config config,
+                              const fs::path& in_folder,
+                              const fs::path& out_folder,
+                              const bool minimal_for_extractor);

decompiler/extractor/main.cpp

@@ -5,10 +5,9 @@
 #include "common/util/term_util.h"
 #include "common/util/unicode_util.h"
 
-#include "decompiler/ObjectFile/ObjectFileDB.h"
 #include "decompiler/config.h"
+#include "decompiler/decompilation_process.h"
 #include "decompiler/extractor/extractor_util.h"
-#include "decompiler/level_extractor/extract_level.h"
 #include "goalc/compiler/Compiler.h"
 
 #include "third-party/CLI11.hpp"
@@ -102,115 +101,24 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
   };
 }
 
-// TODO - remove code duplication, most of this is copying what happens in decompiler's `main.cpp`
-void decompile(const fs::path& iso_data_path,
-               const std::string& data_subfolder,
-               const std::string& config_override) {
-  using namespace decompiler;
-
+ExtractorErrorCode decompile(const fs::path& in_folder,
+                             const std::string& data_subfolder,
+                             const std::string& config_override) {
   // Determine which config to use from the database
-  const auto version_info = get_version_info_or_default(iso_data_path);
+  const auto version_info = get_version_info_or_default(in_folder);
 
-  Config config = read_config_file(file_util::get_jak_project_dir() / "decompiler" / "config" /
-                                       version_info.game_name /
-                                       fmt::format("{}_config.jsonc", version_info.game_name),
-                                   version_info.decomp_config_version, config_override);
+  decompiler::Config config = decompiler::read_config_file(
+      file_util::get_jak_project_dir() / "decompiler" / "config" / version_info.game_name /
+          fmt::format("{}_config.jsonc", version_info.game_name),
+      version_info.decomp_config_version, config_override);
 
-  std::vector<fs::path> dgos, objs, tex_strs, art_strs;
-
-  // grab all DGOS we need (level + common)
-  // TODO - Jak 2 - jak 1 specific code?
-  for (const auto& dgo_name : config.dgo_names) {
-    std::string common_name = "GAME.CGO";
-    if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
-      // ends in DGO, it's a level
-      dgos.push_back(iso_data_path / dgo_name);
-    } else if (dgo_name.length() >= common_name.length() &&
-               dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
-      // it's COMMON.CGO, we need that too.
-      dgos.push_back(iso_data_path / dgo_name);
-    }
-  }
-
-  // grab all the object files we need (just text)
-  for (const auto& obj_name : config.object_file_names) {
-    if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
-      // ends in TXT
-      objs.push_back(iso_data_path / obj_name);
-    }
-  }
-
-  for (const auto& str_name : config.str_texture_file_names) {
-    tex_strs.push_back(iso_data_path / str_name);
-  }
-
-  for (const auto& str_name : config.str_art_file_names) {
-    art_strs.push_back(iso_data_path / str_name);
-  }
-
-  // set up objects
-  ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, tex_strs, art_strs,
-                  config);
-
-  // save object files
   auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / data_subfolder;
-  auto raw_obj_folder = out_folder / "raw_obj";
-  file_util::create_dir_if_needed(raw_obj_folder);
-  db.dump_raw_objects(raw_obj_folder);
 
-  // analyze object file link data
-  db.process_link_data(config);
-  db.find_code(config);
-  db.process_labels();
-
-  // ensure asset dir exists
-  file_util::create_dir_if_needed(out_folder / "assets");
-
-  // text files
-  {
-    auto result = db.process_game_text_files(config);
-    if (!result.empty()) {
-      file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
-    }
-  }
-
-  // textures
-  decompiler::TextureDB tex_db;
-  auto textures_out = out_folder / "textures";
-  auto dump_out = out_folder / "import";
-  file_util::create_dir_if_needed(textures_out);
-  file_util::write_text_file(textures_out / "tpage-dir.txt",
-                             db.process_tpages(tex_db, textures_out, config, dump_out));
-
-  // texture merges
-  // TODO - put all this stuff in somewhere common
-  auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" /
-                            game_version_names[config.game_version] / "texture_merges";
-  if (fs::exists(texture_merge_path)) {
-    tex_db.merge_textures(texture_merge_path);
-  }
-
-  // texture replacements
-  auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements";
-  if (fs::exists(replacements_path)) {
-    tex_db.replace_textures(replacements_path);
-  }
-
-  // game count
-  {
-    auto result = db.process_game_count_file();
-    if (!result.empty()) {
-      file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
-    }
-  }
-
-  // levels
-  {
-    auto level_out_path =
-        file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
-    file_util::create_dir_if_needed(level_out_path);
-    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
+  const auto result = run_decompilation_process(config, in_folder, out_folder, true);
+  if (result != 0) {
+    return ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR;
   }
+  return ExtractorErrorCode::SUCCESS;
 }
 
 const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
@@ -314,7 +222,6 @@ int main(int argc, char** argv) {
   }
 
   // - SETUP
-  decompiler::init_opcode_info();
   if (!project_path_override.empty()) {
     if (!fs::exists(project_path_override)) {
       lg::error("Error: project path override '{}' does not exist", project_path_override.string());
@@ -425,7 +332,11 @@ int main(int argc, char** argv) {
       // Get hash and file count
       const auto [hash, file_count] = calculate_extraction_hash(iso_data_path);
       // Validate
-      auto [version_info, code] = validate(iso_data_path, hash, file_count);
+      auto [version_info, validate_code] = validate(iso_data_path, hash, file_count);
+      if (validate_code == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION ||
+          (flag_fail_on_validation && validate_code != ExtractorErrorCode::SUCCESS)) {
+        return static_cast<int>(validate_code);
+      }
     }
 
     // write out a json file with some metadata for the game
@@ -454,7 +365,10 @@ int main(int argc, char** argv) {
 
   if (flag_decompile) {
     try {
-      decompile(iso_data_path, data_subfolder, decomp_config_override);
+      const auto status_code = decompile(iso_data_path, data_subfolder, decomp_config_override);
+      if (status_code != ExtractorErrorCode::SUCCESS) {
+        return static_cast<int>(status_code);
+      }
     } catch (std::exception& e) {
       lg::error("Error during decompile: {}", e.what());
       return static_cast<int>(ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR);
@@ -462,7 +376,10 @@ int main(int argc, char** argv) {
   }
 
   if (flag_compile) {
-    compile(iso_data_path, data_subfolder);
+    const auto status_code = compile(iso_data_path, data_subfolder);
+    if (status_code != ExtractorErrorCode::SUCCESS) {
+      return static_cast<int>(status_code);
+    }
   }
 
   if (flag_play) {

decompiler/level_extractor/BspHeader.cpp

@@ -11,16 +11,6 @@
 
 namespace level_tools {
 
-std::string DrawStats::print() const {
-  std::string result;
-  result += fmt::format("tfrag tris: {}\n", total_tfrag_tris);
-  result += fmt::format("tie prototype tris: {}\n", total_tie_prototype_tris);
-  result += fmt::format("actors: {}\n", total_actors);
-  result += fmt::format("instances: {}\n", total_tie_instances);
-  result += fmt::format("total tfragments: {}\n", total_tfragments);
-  return result;
-}
-
 void Vector::read_from_file(Ref ref) {
   if ((ref.byte_offset % 16) != 0) {
     throw Error("misaligned vector");
@@ -47,6 +37,17 @@ void Matrix4h::read_from_file(Ref ref) {
   }
 }
 
+void TimeOfDayPalette::read_from_file(Ref ref) {
+  width = deref_u32(ref, 0);
+  ASSERT(width == 8);
+  height = deref_u32(ref, 1);
+  pad = deref_u32(ref, 2);
+  ASSERT(pad == 0);
+  for (int i = 0; i < int(8 * height); i++) {
+    colors.push_back(deref_u32(ref, 3 + i));
+  }
+}
+
 std::string Vector::print(int indent) const {
   std::string is(indent, ' ');
   std::string result;
@@ -95,7 +96,6 @@ std::string FileInfo::print(int indent) const {
 
 void DrawableTreeUnknown::read_from_file(TypedRef ref,
                                          const decompiler::DecompilerTypeSystem& /*dts*/,
-                                         DrawStats* /*stats*/,
                                          GameVersion /*version*/) {
   type_name = ref.type->get_name();
 }
@@ -113,7 +113,6 @@ std::string DrawableTreeUnknown::my_type() const {
 
 void DrawableInlineArrayUnknown::read_from_file(TypedRef ref,
                                                 const decompiler::DecompilerTypeSystem& /*dts*/,
-                                                DrawStats* /*stats*/,
                                                 GameVersion /*version*/) {
   type_name = ref.type->get_name();
 }
@@ -131,27 +130,26 @@ std::string DrawableInlineArrayUnknown::my_type() const {
 
 std::unique_ptr<Drawable> make_draw_node_child(TypedRef ref,
                                                const decompiler::DecompilerTypeSystem& dts,
-                                               DrawStats* stats,
                                                GameVersion version) {
   if (ref.type->get_name() == "draw-node") {
     auto result = std::make_unique<DrawNode>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   } else if (ref.type->get_name() == "tfragment") {
     auto result = std::make_unique<TFragment>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   } else if (ref.type->get_name() == "instance-tie") {
     auto result = std::make_unique<InstanceTie>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   } else if (ref.type->get_name() == "drawable-actor") {
     auto result = std::make_unique<DrawableActor>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   } else if (ref.type->get_name() == "instance-shrubbery") {
     auto result = std::make_unique<shrub_types::InstanceShrubbery>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   } else {
     throw Error("Unknown child of draw node: {}\n", ref.type->get_name());
@@ -174,9 +172,7 @@ int get_child_stride(const std::string& type) {
   }
 }
 
-void TFragmentDebugData::read_from_file(Ref ref,
-                                        const decompiler::DecompilerTypeSystem& /*dts*/,
-                                        DrawStats* stats) {
+void TFragmentDebugData::read_from_file(Ref ref, const decompiler::DecompilerTypeSystem& /*dts*/) {
   u32 data[4];
   auto& words = ref.data->words_by_seg.at(ref.seg);
   for (int i = 0; i < 4; i++) {
@@ -194,7 +190,6 @@ void TFragmentDebugData::read_from_file(Ref ref,
   for (auto num_tri : num_tris) {
     tris = std::max(tris, (u32)num_tri);
   }
-  stats->total_tfrag_tris += tris;
 
   auto& debug_word = words.at(4 + (ref.byte_offset / 4));
   if (debug_word.kind() != decompiler::LinkedWord::PLAIN_DATA || debug_word.data != 0) {
@@ -319,12 +314,10 @@ std::vector<u8> read_dma_chain(Ref& start, u32 qwc) {
 
 void TFragment::read_from_file(TypedRef ref,
                                const decompiler::DecompilerTypeSystem& dts,
-                               DrawStats* stats,
                                GameVersion version) {
-  stats->total_tfragments++;
   id = read_plain_data_field<s16>(ref, "id", dts);
   color_index = read_plain_data_field<s16>(ref, "color-index", dts);
-  debug_data.read_from_file(deref_label(get_field_ref(ref, "debug-data", dts)), dts, stats);
+  debug_data.read_from_file(deref_label(get_field_ref(ref, "debug-data", dts)), dts);
   // todo color_indices
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
 
@@ -350,7 +343,7 @@ void TFragment::read_from_file(TypedRef ref,
     dma_slot.byte_offset += 4;
   }
 
-  if (stats->debug_print_dma_data) {
+  if (false) {
     // first, common
     lg::info("DMA COMMON {}, {} qwc:", dmas[0].label_name, dma_qwc[0]);
     tfrag_debug_print_unpack(dmas[0].ref, dma_qwc[0]);
@@ -457,7 +450,6 @@ void memcpy_plain_data(u8* dst, const Ref& ref, size_t size_bytes) {
 
 void TieFragment::read_from_file(TypedRef ref,
                                  const decompiler::DecompilerTypeSystem& dts,
-                                 DrawStats* stats,
                                  GameVersion version) {
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
   switch (version) {
@@ -502,8 +494,6 @@ void TieFragment::read_from_file(TypedRef ref,
     memcpy(point_ref.data() + (i * 4), &word.data, 4);
   }
 
-  stats->total_tie_prototype_tris += num_tris;
-
   if (version > GameVersion::Jak1) {
     u16 normals_qwc = read_plain_data_field<u16>(ref, "normal-count", dts);
     if (normals_qwc) {
@@ -539,7 +529,6 @@ std::string DrawableActor::print(const PrintSettings& /*settings*/, int indent)
 
 void InstanceTie::read_from_file(TypedRef ref,
                                  const decompiler::DecompilerTypeSystem& dts,
-                                 DrawStats* stats,
                                  GameVersion /*version*/) {
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
   bucket_index = read_plain_data_field<u16>(ref, "bucket-index", dts);
@@ -549,7 +538,6 @@ void InstanceTie::read_from_file(TypedRef ref,
   origin.read_from_file(get_field_ref(ref, "origin", dts));
   wind_index = read_plain_data_field<u16>(ref, "wind-index", dts);
   color_indices = deref_label(get_field_ref(ref, "color-indices", dts));
-  stats->total_tie_instances++;
 }
 
 std::string InstanceTie::print(const PrintSettings& /*settings*/, int indent) const {
@@ -562,7 +550,6 @@ std::string InstanceTie::print(const PrintSettings& /*settings*/, int indent) co
 
 void DrawNode::read_from_file(TypedRef ref,
                               const decompiler::DecompilerTypeSystem& dts,
-                              DrawStats* stats,
                               GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);             // 4
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));  // 16
@@ -575,7 +562,7 @@ void DrawNode::read_from_file(TypedRef ref,
 
   for (int i = 0; i < child_count; i++) {
     children.push_back(
-        make_draw_node_child(typed_ref_from_basic(first_child_obj, dts), dts, stats, version));
+        make_draw_node_child(typed_ref_from_basic(first_child_obj, dts), dts, version));
     first_child_obj.byte_offset += get_child_stride(get_type_of_basic(first_child_obj));
   }
 
@@ -607,7 +594,6 @@ std::string DrawNode::my_type() const {
 
 void DrawableInlineArrayNode::read_from_file(TypedRef ref,
                                              const decompiler::DecompilerTypeSystem& dts,
-                                             DrawStats* stats,
                                              GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -622,7 +608,7 @@ void DrawableInlineArrayNode::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     draw_nodes.emplace_back();
-    draw_nodes.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    draw_nodes.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -650,7 +636,6 @@ std::string DrawableInlineArrayNode::my_type() const {
 
 void DrawableInlineArrayTFrag::read_from_file(TypedRef ref,
                                               const decompiler::DecompilerTypeSystem& dts,
-                                              DrawStats* stats,
                                               GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -665,7 +650,7 @@ void DrawableInlineArrayTFrag::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     tfragments.emplace_back();
-    tfragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    tfragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -693,7 +678,6 @@ std::string DrawableInlineArrayTFrag::my_type() const {
 
 void DrawableInlineArrayInstanceTie::read_from_file(TypedRef ref,
                                                     const decompiler::DecompilerTypeSystem& dts,
-                                                    DrawStats* stats,
                                                     GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -708,7 +692,7 @@ void DrawableInlineArrayInstanceTie::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     instances.emplace_back();
-    instances.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    instances.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -740,7 +724,6 @@ std::string PrototypeTie::my_type() const {
 
 void PrototypeTie::read_from_file(TypedRef ref,
                                   const decompiler::DecompilerTypeSystem& dts,
-                                  DrawStats* stats,
                                   GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -755,7 +738,7 @@ void PrototypeTie::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     tie_fragments.emplace_back();
-    tie_fragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    tie_fragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -780,58 +763,56 @@ std::string PrototypeTie::print(const PrintSettings& settings, int indent) const
 std::unique_ptr<DrawableInlineArray> make_drawable_inline_array(
     TypedRef ref,
     const decompiler::DecompilerTypeSystem& dts,
-    DrawStats* stats,
     GameVersion version) {
   if (ref.type->get_name() == "drawable-inline-array-node") {
     auto result = std::make_unique<DrawableInlineArrayNode>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-tfrag") {
     auto result = std::make_unique<DrawableInlineArrayTFrag>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-trans-tfrag") {
     auto result = std::make_unique<DrawableInlineArrayTransTFrag>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-tfrag-trans") {
     auto result = std::make_unique<DrawableInlineArrayTFragTrans>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-tfrag-water") {
     auto result = std::make_unique<DrawableInlineArrayTFragWater>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-instance-tie") {
     auto result = std::make_unique<DrawableInlineArrayInstanceTie>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   if (ref.type->get_name() == "drawable-inline-array-instance-shrub") {
     auto result = std::make_unique<shrub_types::DrawableInlineArrayInstanceShrub>();
-    result->read_from_file(ref, dts, stats, version);
+    result->read_from_file(ref, dts, version);
     return result;
   }
 
   auto result = std::make_unique<DrawableInlineArrayUnknown>();
-  result->read_from_file(ref, dts, stats, version);
+  result->read_from_file(ref, dts, version);
   return result;
 }
 
 void DrawableTreeTfrag::read_from_file(TypedRef ref,
                                        const decompiler::DecompilerTypeSystem& dts,
-                                       DrawStats* stats,
                                        GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -842,16 +823,7 @@ void DrawableTreeTfrag::read_from_file(TypedRef ref,
     throw Error("misaligned data array");
   }
 
-  auto palette = deref_label(get_field_ref(ref, "time-of-day-pal", dts));
-  time_of_day.width = deref_u32(palette, 0);
-
-  ASSERT(time_of_day.width == 8);
-  time_of_day.height = deref_u32(palette, 1);
-  time_of_day.pad = deref_u32(palette, 2);
-  ASSERT(time_of_day.pad == 0);
-  for (int i = 0; i < int(8 * time_of_day.height); i++) {
-    time_of_day.colors.push_back(deref_u32(palette, 3 + i));
-  }
+  time_of_day.read_from_file(deref_label(get_field_ref(ref, "time-of-day-pal", dts)));
 
   for (int idx = 0; idx < length; idx++) {
     Ref array_slot_ref = data_ref;
@@ -861,7 +833,7 @@ void DrawableTreeTfrag::read_from_file(TypedRef ref,
     object_ref.byte_offset -= 4;
 
     arrays.push_back(
-        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
   }
 }
 
@@ -894,7 +866,6 @@ std::string DrawableTreeTfrag::my_type() const {
 
 void DrawableTreeActor::read_from_file(TypedRef ref,
                                        const decompiler::DecompilerTypeSystem& dts,
-                                       DrawStats* stats,
                                        GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -913,7 +884,7 @@ void DrawableTreeActor::read_from_file(TypedRef ref,
     object_ref.byte_offset -= 4;
 
     arrays.push_back(
-        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
   }
 }
 
@@ -942,7 +913,6 @@ std::string DrawableTreeActor::my_type() const {
 
 void PrototypeBucketTie::read_from_file(TypedRef ref,
                                         const decompiler::DecompilerTypeSystem& dts,
-                                        DrawStats* stats,
                                         GameVersion version) {
   name = read_string_field(ref, "name", dts, true);
   switch (version) {
@@ -970,7 +940,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
       if (word.kind() == decompiler::LinkedWord::PTR) {
         auto p = deref_label(fr);
         p.byte_offset -= 4;
-        collide_frag.read_from_file(typed_ref_from_basic(p, dts), dts, stats, version);
+        collide_frag.read_from_file(typed_ref_from_basic(p, dts), dts, version);
       }
     }
   } else {
@@ -1024,7 +994,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
     if (get_type_of_basic(geom) != "prototype-tie") {
       throw Error("bad type in prototype-bucket-tie: {}", get_type_of_basic(geom));
     }
-    geometry[i].read_from_file(typed_ref_from_basic(geom, dts), dts, stats, version);
+    geometry[i].read_from_file(typed_ref_from_basic(geom, dts), dts, version);
     geom_start.byte_offset += 4;
   }
 
@@ -1060,16 +1030,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
   }
 
   // get the colors
-  auto palette = deref_label(get_field_ref(ref, "tie-colors", dts));
-  time_of_day.width = deref_u32(palette, 0);
-
-  ASSERT(time_of_day.width == 8);
-  time_of_day.height = deref_u32(palette, 1);
-  time_of_day.pad = deref_u32(palette, 2);
-  ASSERT(time_of_day.pad == 0);
-  for (int i = 0; i < int(8 * time_of_day.height); i++) {
-    time_of_day.colors.push_back(deref_u32(palette, 3 + i));
-  }
+  time_of_day.read_from_file(deref_label(get_field_ref(ref, "tie-colors", dts)));
 
   auto fr = get_field_ref(ref, "envmap-shader", dts);
   const auto& word = fr.data->words_by_seg.at(fr.seg).at(fr.byte_offset / 4);
@@ -1129,7 +1090,6 @@ std::string PrototypeBucketTie::print(const PrintSettings& settings, int indent)
 
 void PrototypeArrayTie::read_from_file(TypedRef ref,
                                        const decompiler::DecompilerTypeSystem& dts,
-                                       DrawStats* stats,
                                        GameVersion version) {
   length = read_plain_data_field<u32>(ref, "length", dts);
   allocated_length = read_plain_data_field<u32>(ref, "allocated-length", dts);
@@ -1146,7 +1106,7 @@ void PrototypeArrayTie::read_from_file(TypedRef ref,
       throw Error("bad type in PrototypeArrayTie data: {}\n", type);
     }
     data.emplace_back();
-    data.back().read_from_file(typed_ref_from_basic(thing, dts), dts, stats, version);
+    data.back().read_from_file(typed_ref_from_basic(thing, dts), dts, version);
   }
 }
 
@@ -1167,11 +1127,10 @@ std::string PrototypeArrayTie::print(const PrintSettings& settings, int indent)
 
 void ProxyPrototypeArrayTie::read_from_file(TypedRef ref,
                                             const decompiler::DecompilerTypeSystem& dts,
-                                            DrawStats* stats,
                                             GameVersion version) {
   prototype_array_tie.read_from_file(
       get_and_check_ref_to_basic(ref, "prototype-array-tie", "prototype-array-tie", dts), dts,
-      stats, version);
+      version);
   wind_vectors = deref_label(get_field_ref(ref, "wind-vectors", dts));
 }
 
@@ -1182,7 +1141,6 @@ std::string ProxyPrototypeArrayTie::print(const PrintSettings& settings, int ind
 
 void DrawableTreeInstanceTie::read_from_file(TypedRef ref,
                                              const decompiler::DecompilerTypeSystem& dts,
-                                             DrawStats* stats,
                                              GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -1191,7 +1149,7 @@ void DrawableTreeInstanceTie::read_from_file(TypedRef ref,
   auto pt = deref_label(get_field_ref(ref, "prototypes", dts));
   pt.byte_offset -= 4;
 
-  prototypes.read_from_file(typed_ref_from_basic(pt, dts), dts, stats, version);
+  prototypes.read_from_file(typed_ref_from_basic(pt, dts), dts, version);
 
   auto data_ref = get_field_ref(ref, "data", dts);
   if ((data_ref.byte_offset % 4) != 0) {
@@ -1205,7 +1163,7 @@ void DrawableTreeInstanceTie::read_from_file(TypedRef ref,
     object_ref.byte_offset -= 4;
 
     arrays.push_back(
-        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
   }
 }
 
@@ -1241,7 +1199,6 @@ std::string DrawableTreeInstanceTie::my_type() const {
 
 void DrawableTreeCollideFragment::read_from_file(TypedRef ref,
                                                  const decompiler::DecompilerTypeSystem& dts,
-                                                 DrawStats* stats,
                                                  GameVersion version) {
   s16 length = read_plain_data_field<s16>(ref, "length", dts);
   auto data_ref = get_field_ref(ref, "data", dts);
@@ -1254,7 +1211,7 @@ void DrawableTreeCollideFragment::read_from_file(TypedRef ref,
 
   Ref object_ref = deref_label(array_slot_ref);
   object_ref.byte_offset -= 4;
-  last_array.read_from_file(typed_ref_from_basic(object_ref, dts), dts, stats, version);
+  last_array.read_from_file(typed_ref_from_basic(object_ref, dts), dts, version);
 }
 
 std::string DrawableTreeCollideFragment::print(const PrintSettings& settings, int indent) const {
@@ -1267,7 +1224,6 @@ std::string DrawableTreeCollideFragment::my_type() const {
 
 void DrawableInlineArrayCollideFragment::read_from_file(TypedRef ref,
                                                         const decompiler::DecompilerTypeSystem& dts,
-                                                        DrawStats* stats,
                                                         GameVersion /*version*/) {
   ASSERT(ref.type->get_name() == "drawable-inline-array-collide-fragment");
   id = read_plain_data_field<s16>(ref, "id", dts);
@@ -1283,7 +1239,7 @@ void DrawableInlineArrayCollideFragment::read_from_file(TypedRef ref,
       throw Error("bad collide fragment type: {}", type);
     }
     collide_fragments.emplace_back();
-    collide_fragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats);
+    collide_fragments.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts);
   }
 }
 
@@ -1309,13 +1265,11 @@ std::string DrawableInlineArrayCollideFragment::my_type() const {
   return "drawable-inline-array-collide-fragment";
 }
 
-void CollideFragment::read_from_file(TypedRef ref,
-                                     const decompiler::DecompilerTypeSystem& dts,
-                                     DrawStats* stats) {
+void CollideFragment::read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts) {
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
   auto r = deref_label(get_field_ref(ref, "mesh", dts));
   r.byte_offset -= 4;
-  mesh.read_from_file(typed_ref_from_basic(r, dts), dts, stats);
+  mesh.read_from_file(typed_ref_from_basic(r, dts), dts);
 }
 
 std::string CollideFragment::print(const PrintSettings& settings, int indent) const {
@@ -1327,9 +1281,7 @@ std::string CollideFragment::print(const PrintSettings& settings, int indent) co
   return result;
 }
 
-void CollideFragMesh::read_from_file(TypedRef ref,
-                                     const decompiler::DecompilerTypeSystem& dts,
-                                     DrawStats* /*stats*/) {
+void CollideFragMesh::read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts) {
   strip_data_len = read_plain_data_field<u16>(ref, "strip-data-len", dts);
   poly_count = read_plain_data_field<u16>(ref, "poly-count", dts);
   vertex_count = read_plain_data_field<u8>(ref, "vertex-count", dts);
@@ -1363,7 +1315,6 @@ namespace shrub_types {
 
 void DrawableTreeInstanceShrub::read_from_file(TypedRef ref,
                                                const decompiler::DecompilerTypeSystem& dts,
-                                               level_tools::DrawStats* stats,
                                                GameVersion version) {
   // the usual drawable stuff
   id = read_plain_data_field<s16>(ref, "id", dts);
@@ -1385,7 +1336,7 @@ void DrawableTreeInstanceShrub::read_from_file(TypedRef ref,
     object_ref.byte_offset -= 4;
 
     arrays.push_back(
-        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+        make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
   }
   // confirm that we have the weird shrub pattern and only found one array.
   ASSERT(length == 1);
@@ -1399,7 +1350,7 @@ void DrawableTreeInstanceShrub::read_from_file(TypedRef ref,
   Ref object_ref = deref_label(data_ref);
   object_ref.byte_offset -= 4;
   discovered_arrays.push_back(
-      make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+      make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
 
   bool done = false;
   object_ref.byte_offset += 16;
@@ -1408,10 +1359,10 @@ void DrawableTreeInstanceShrub::read_from_file(TypedRef ref,
     if (word.kind() == decompiler::LinkedWord::TYPE_PTR) {
       if (word.symbol_name() == "drawable-inline-array-node") {
         discovered_arrays.push_back(
-            make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+            make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
       } else if (word.symbol_name() == "drawable-inline-array-instance-shrub") {
         discovered_arrays.push_back(
-            make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+            make_drawable_inline_array(typed_ref_from_basic(object_ref, dts), dts, version));
       } else if (word.symbol_name() == "time-of-day-palette" ||
                  word.symbol_name() == "light-hash" ||
                  word.symbol_name() == "collide-hash-fragment") {
@@ -1426,18 +1377,10 @@ void DrawableTreeInstanceShrub::read_from_file(TypedRef ref,
   // this "info" thing holds all the prototypes
   auto pt = deref_label(get_field_ref(ref, "info", dts));
   pt.byte_offset -= 4;
-  info.read_from_file(typed_ref_from_basic(pt, dts), dts, stats, version);
+  info.read_from_file(typed_ref_from_basic(pt, dts), dts, version);
 
   // time of day palette. we'll want these colors in the FR3 file.
-  auto palette = deref_label(get_field_ref(ref, "colors-added", dts));
-  time_of_day.width = deref_u32(palette, 0);
-  ASSERT(time_of_day.width == 8);
-  time_of_day.height = deref_u32(palette, 1);
-  time_of_day.pad = deref_u32(palette, 2);
-  ASSERT(time_of_day.pad == 0);
-  for (int i = 0; i < int(8 * time_of_day.height); i++) {
-    time_of_day.colors.push_back(deref_u32(palette, 3 + i));
-  }
+  time_of_day.read_from_file(deref_label(get_field_ref(ref, "colors-added", dts)));
 }
 
 std::string DrawableTreeInstanceShrub::my_type() const {
@@ -1471,7 +1414,6 @@ std::string DrawableTreeInstanceShrub::print(const level_tools::PrintSettings& s
 
 void InstanceShrubbery::read_from_file(TypedRef ref,
                                        const decompiler::DecompilerTypeSystem& dts,
-                                       level_tools::DrawStats* /*stats*/,
                                        GameVersion /*version*/) {
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
   bucket_index = read_plain_data_field<u16>(ref, "bucket-index", dts);
@@ -1496,7 +1438,6 @@ std::string InstanceShrubbery::print(const level_tools::PrintSettings& /*setting
 
 void DrawableInlineArrayInstanceShrub::read_from_file(TypedRef ref,
                                                       const decompiler::DecompilerTypeSystem& dts,
-                                                      DrawStats* stats,
                                                       GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -1511,7 +1452,7 @@ void DrawableInlineArrayInstanceShrub::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     instances.emplace_back();
-    instances.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    instances.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -1536,12 +1477,11 @@ std::string DrawableInlineArrayInstanceShrub::print(const PrintSettings& setting
 
 void PrototypeArrayShrubInfo::read_from_file(TypedRef ref,
                                              const decompiler::DecompilerTypeSystem& dts,
-                                             level_tools::DrawStats* stats,
                                              GameVersion version) {
   prototype_inline_array_shrub.read_from_file(
       get_and_check_ref_to_basic(ref, "prototype-inline-array-shrub",
                                  "prototype-inline-array-shrub", dts),
-      dts, stats, version);
+      dts, version);
   wind_vectors = deref_label(get_field_ref(ref, "wind-vectors", dts));
 }
 
@@ -1552,7 +1492,6 @@ std::string PrototypeArrayShrubInfo::print(const level_tools::PrintSettings& set
 
 void PrototypeInlineArrayShrub::read_from_file(TypedRef ref,
                                                const decompiler::DecompilerTypeSystem& dts,
-                                               level_tools::DrawStats* stats,
                                                GameVersion version) {
   length = read_plain_data_field<s16>(ref, "length", dts);
   auto data_ref = get_field_ref(ref, "data", dts);
@@ -1566,7 +1505,7 @@ void PrototypeInlineArrayShrub::read_from_file(TypedRef ref,
       throw Error("bad type in PrototypeInlineArrayShrub data: {}\n", type);
     }
     data.emplace_back();
-    data.back().read_from_file(typed_ref_from_basic(thing, dts), dts, stats, version);
+    data.back().read_from_file(typed_ref_from_basic(thing, dts), dts, version);
   }
 }
 
@@ -1586,7 +1525,6 @@ std::string PrototypeGenericShrub::print(const level_tools::PrintSettings& setti
 
 void PrototypeGenericShrub::read_from_file(TypedRef ref,
                                            const decompiler::DecompilerTypeSystem& dts,
-                                           level_tools::DrawStats* stats,
                                            GameVersion version) {
   length = read_plain_data_field<s16>(ref, "length", dts);
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
@@ -1602,7 +1540,7 @@ void PrototypeGenericShrub::read_from_file(TypedRef ref,
       throw Error("bad type in PrototypeGenericShrub data: {}\n", type);
     }
     shrubs.emplace_back();
-    shrubs.back().read_from_file(typed_ref_from_basic(thing, dts), dts, stats, version);
+    shrubs.back().read_from_file(typed_ref_from_basic(thing, dts), dts, version);
   }
 }
 
@@ -1622,7 +1560,6 @@ std::string PrototypeInlineArrayShrub::print(const level_tools::PrintSettings& s
 
 void PrototypeBucketShrub::read_from_file(TypedRef ref,
                                           const decompiler::DecompilerTypeSystem& dts,
-                                          level_tools::DrawStats* stats,
                                           GameVersion version) {
   name = read_string_field(ref, "name", dts, true);
   switch (version) {
@@ -1667,10 +1604,10 @@ void PrototypeBucketShrub::read_from_file(TypedRef ref,
   if (get_type_of_basic(normal_geom) != "prototype-shrubbery") {
     throw Error("bad normal shrub type: {}", get_type_of_basic(normal_geom));
   }
-  shrubbery_geom.read_from_file(typed_ref_from_basic(normal_geom, dts), dts, stats, version);
+  shrubbery_geom.read_from_file(typed_ref_from_basic(normal_geom, dts), dts, version);
   geom_start.byte_offset += 4;
 
-  generic_geom.read_from_file(typed_ref_from_basic(generic_geom_l, dts), dts, stats, version);
+  generic_geom.read_from_file(typed_ref_from_basic(generic_geom_l, dts), dts, version);
 
   // todo transparent version
   // todo billboard version.
@@ -1694,7 +1631,6 @@ std::string PrototypeBucketShrub::print(const level_tools::PrintSettings& settin
 
 void PrototypeShrubbery::read_from_file(TypedRef ref,
                                         const decompiler::DecompilerTypeSystem& dts,
-                                        level_tools::DrawStats* stats,
                                         GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -1709,7 +1645,7 @@ void PrototypeShrubbery::read_from_file(TypedRef ref,
       throw Error("bad draw node type: {}", type);
     }
     shrubs.emplace_back();
-    shrubs.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    shrubs.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
@@ -1740,7 +1676,6 @@ void copy_dma_to_vector(std::vector<u8>* out, Ref data_start, int qwc) {
 
 void Shrubbery::read_from_file(TypedRef ref,
                                const decompiler::DecompilerTypeSystem& dts,
-                               level_tools::DrawStats* /*stats*/,
                                GameVersion /*version*/) {
   // read the easy ones.
   obj_qwc = read_plain_data_field<u8>(ref, "obj-qwc", dts);
@@ -1779,7 +1714,6 @@ std::string GenericShrubFragment::print(const level_tools::PrintSettings& /*sett
 
 void GenericShrubFragment::read_from_file(TypedRef ref,
                                           const decompiler::DecompilerTypeSystem& dts,
-                                          level_tools::DrawStats* /*stats*/,
                                           GameVersion /*version*/) {
   cnt_qwc = read_plain_data_field<u8>(ref, "cnt-qwc", dts);
   vtx_qwc = read_plain_data_field<u8>(ref, "vtx-qwc", dts);
@@ -1797,82 +1731,80 @@ void GenericShrubFragment::read_from_file(TypedRef ref,
 
 std::unique_ptr<DrawableTree> make_drawable_tree(TypedRef ref,
                                                  const decompiler::DecompilerTypeSystem& dts,
-                                                 DrawStats* stats,
                                                  GameVersion version) {
   if (ref.type->get_name() == "drawable-tree-tfrag") {
     auto tree = std::make_unique<DrawableTreeTfrag>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-trans-tfrag") {
     auto tree = std::make_unique<DrawableTreeTransTfrag>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-lowres-tfrag") {
     auto tree = std::make_unique<DrawableTreeLowresTfrag>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-dirt-tfrag") {
     auto tree = std::make_unique<DrawableTreeDirtTfrag>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-ice-tfrag") {
     auto tree = std::make_unique<DrawableTreeIceTfrag>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-instance-tie") {
     auto tree = std::make_unique<DrawableTreeInstanceTie>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-tfrag-trans") {
     auto tree = std::make_unique<DrawableTreeTfragTrans>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-tfrag-water") {
     auto tree = std::make_unique<DrawableTreeTfragWater>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-actor") {
     auto tree = std::make_unique<DrawableTreeActor>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-instance-shrub") {
     auto tree = std::make_unique<shrub_types::DrawableTreeInstanceShrub>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   if (ref.type->get_name() == "drawable-tree-collide-fragment") {
     auto tree = std::make_unique<DrawableTreeCollideFragment>();
-    tree->read_from_file(ref, dts, stats, version);
+    tree->read_from_file(ref, dts, version);
     return tree;
   }
 
   auto tree = std::make_unique<DrawableTreeUnknown>();
-  tree->read_from_file(ref, dts, stats, version);
+  tree->read_from_file(ref, dts, version);
   return tree;
 }
 
 void DrawableTreeArray::read_from_file(TypedRef ref,
                                        const decompiler::DecompilerTypeSystem& dts,
-                                       DrawStats* stats,
                                        GameVersion version) {
   id = read_plain_data_field<s16>(ref, "id", dts);
   length = read_plain_data_field<s16>(ref, "length", dts);
@@ -1889,7 +1821,7 @@ void DrawableTreeArray::read_from_file(TypedRef ref,
     Ref object_ref = deref_label(array_slot_ref);
     object_ref.byte_offset -= 4;
 
-    trees.push_back(make_drawable_tree(typed_ref_from_basic(object_ref, dts), dts, stats, version));
+    trees.push_back(make_drawable_tree(typed_ref_from_basic(object_ref, dts), dts, version));
   }
 }
 
@@ -1916,7 +1848,6 @@ void fill_res_with_value_types(Res& res_tag, const Ref& data) {
 
 void EntityActor::read_from_file(TypedRef ref,
                                  const decompiler::DecompilerTypeSystem& dts,
-                                 level_tools::DrawStats* /*stats*/,
                                  GameVersion /*version*/) {
   trans.read_from_file(get_field_ref(ref, "trans", dts));
   aid = read_plain_data_field<u32>(ref, "aid", dts);
@@ -1998,17 +1929,13 @@ void EntityActor::read_from_file(TypedRef ref,
 
 void DrawableActor::read_from_file(TypedRef ref,
                                    const decompiler::DecompilerTypeSystem& dts,
-                                   DrawStats* stats,
                                    GameVersion version) {
   bsphere.read_from_file(get_field_ref(ref, "bsphere", dts));
-  actor.read_from_file(get_and_check_ref_to_basic(ref, "actor", "entity-actor", dts), dts, stats,
-                       version);
-  stats->total_actors++;
+  actor.read_from_file(get_and_check_ref_to_basic(ref, "actor", "entity-actor", dts), dts, version);
 }
 
 void DrawableInlineArrayActor::read_from_file(TypedRef ref,
                                               const decompiler::DecompilerTypeSystem& dts,
-                                              DrawStats* stats,
                                               GameVersion version) {
   int num_actors = read_plain_data_field<int16_t>(ref, "length", dts);
   auto data_ref = get_field_ref(ref, "data", dts);
@@ -2020,21 +1947,48 @@ void DrawableInlineArrayActor::read_from_file(TypedRef ref,
       throw Error("bad drawable-actor type: {}", type);
     }
     drawable_actors.emplace_back();
-    drawable_actors.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, stats, version);
+    drawable_actors.back().read_from_file(typed_ref_from_basic(obj_ref, dts), dts, version);
   }
 }
 
 void CollideHash::read_from_file(TypedRef ref,
                                  const decompiler::DecompilerTypeSystem& dts,
-                                 level_tools::DrawStats* /*stats*/,
                                  GameVersion /*version*/) {
   num_items = read_plain_data_field<uint32_t>(ref, "num-items", dts);
   item_array = deref_label(get_field_ref(ref, "item-array", dts));
 }
 
+void HFragment::read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts) {
+  start_corner.read_from_file(get_field_ref(ref, "start-corner", dts));
+  spheres.resize(kNumCorners);
+  auto spheres_ptr = deref_label(get_field_ref(ref, "spheres", dts));
+  for (int i = 0; i < kNumCorners; i++) {
+    spheres[i].read_from_file(spheres_ptr);
+    spheres_ptr.byte_offset += 16;
+  }
+
+  vis_ids.resize(kNumCorners);
+  memcpy_plain_data((u8*)vis_ids.data(), deref_label(get_field_ref(ref, "visids", dts)),
+                    sizeof(s16) * kNumCorners);
+  memcpy_plain_data((u8*)shaders, deref_label(get_field_ref(ref, "shaders", dts)), 80 * 3);
+  colors.read_from_file(deref_label(get_field_ref(ref, "colors", dts)));
+  // note: using hard-coded size here
+  memcpy_plain_data((u8*)montage, deref_label(get_field_ref(ref, "montage", dts)),
+                    sizeof(u16) * 16 * 17);
+  // bucket
+  verts.resize(kNumVerts);
+  memcpy_plain_data((u8*)verts.data(), deref_label(get_field_ref(ref, "verts", dts)),
+                    sizeof(u32) * kNumVerts);
+  // pat-array
+  // pat-len
+  num_buckets_far = read_plain_data_field<u16>(ref, "num-buckets-far", dts);
+  num_buckets_mid = read_plain_data_field<u16>(ref, "num-buckets-mid", dts);
+  num_buckets_near = read_plain_data_field<u16>(ref, "num-buckets-near", dts);
+  size = read_plain_data_field<u32>(ref, "size", dts);
+}
+
 void BspHeader::read_from_file(const decompiler::LinkedObjectFile& file,
                                const decompiler::DecompilerTypeSystem& dts,
-                               DrawStats* stats,
                                GameVersion version,
                                bool only_read_texture_remap) {
   TypedRef ref;
@@ -2072,14 +2026,14 @@ void BspHeader::read_from_file(const decompiler::LinkedObjectFile& file,
     case GameVersion::Jak2:
     case GameVersion::Jak3:
       visible_list_length = read_plain_data_field<s16>(ref, "visible-list-length", dts);
+      extra_vis_list_length = read_plain_data_field<s16>(ref, "extra-vis-list-length", dts);
       break;
     default:
       ASSERT(false);
   }
 
   drawable_tree_array.read_from_file(
-      get_and_check_ref_to_basic(ref, "drawable-trees", "drawable-tree-array", dts), dts, stats,
-      version);
+      get_and_check_ref_to_basic(ref, "drawable-trees", "drawable-tree-array", dts), dts, version);
 
   if (version > GameVersion::Jak1) {
     auto ff = get_field_ref(ref, "texture-flags", dts);
@@ -2088,14 +2042,22 @@ void BspHeader::read_from_file(const decompiler::LinkedObjectFile& file,
 
   if (get_word_kind_for_field(ref, "actors", dts) == decompiler::LinkedWord::PTR) {
     actors.read_from_file(
-        get_and_check_ref_to_basic(ref, "actors", "drawable-inline-array-actor", dts), dts, stats,
+        get_and_check_ref_to_basic(ref, "actors", "drawable-inline-array-actor", dts), dts,
         version);
   }
 
   if (version > GameVersion::Jak1 &&
       get_word_kind_for_field(ref, "collide-hash", dts) == decompiler::LinkedWord::PTR) {
     collide_hash.read_from_file(
-        get_and_check_ref_to_basic(ref, "collide-hash", "collide-hash", dts), dts, stats, version);
+        get_and_check_ref_to_basic(ref, "collide-hash", "collide-hash", dts), dts, version);
+  }
+
+  if (version == GameVersion::Jak3) {
+    if (get_word_kind_for_field(ref, "hfrag-drawable", dts) == decompiler::LinkedWord::PTR) {
+      hfrag.emplace();
+      hfrag->read_from_file(get_and_check_ref_to_basic(ref, "hfrag-drawable", "hfragment", dts),
+                            dts);
+    }
   }
 }
 

decompiler/level_extractor/BspHeader.h

@@ -32,18 +32,6 @@ struct PrintSettings {
   bool expand_collide = false;
 };
 
-struct DrawStats {
-  int total_tfrag_tris = 0;
-  int total_tie_prototype_tris = 0;
-  int total_actors = 0;
-  int total_tie_instances = 0;
-  int total_tfragments = 0;
-
-  bool debug_print_dma_data = false;
-
-  std::string print() const;
-};
-
 /////////////////////
 // Common Types
 /////////////////////
@@ -73,6 +61,7 @@ struct TimeOfDayPalette {
   u32 height;
   u32 pad;
   std::vector<u32> colors;
+  void read_from_file(Ref ref);
 };
 
 /////////////////////
@@ -87,7 +76,6 @@ struct TimeOfDayPalette {
 struct Drawable {
   virtual void read_from_file(TypedRef ref,
                               const decompiler::DecompilerTypeSystem& dts,
-                              DrawStats* stats,
                               GameVersion version) = 0;
   virtual std::string print(const PrintSettings& settings, int indent) const = 0;
   virtual std::string my_type() const = 0;
@@ -100,7 +88,6 @@ struct Drawable {
 struct DrawNode : public Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -126,7 +113,6 @@ struct DrawableInlineArrayNode : public DrawableInlineArray {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -140,7 +126,6 @@ struct DrawableTree : public Drawable {};
 struct DrawableTreeUnknown : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -150,7 +135,6 @@ struct DrawableTreeUnknown : public DrawableTree {
 struct DrawableInlineArrayUnknown : public DrawableInlineArray {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -190,7 +174,6 @@ struct EntityActor {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
 };
 
@@ -201,7 +184,6 @@ struct DrawableActor : public Drawable {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "drawable-actor"; }
@@ -210,7 +192,6 @@ struct DrawableActor : public Drawable {
 struct DrawableTreeActor : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -241,7 +222,7 @@ struct CollideFragMesh {
    (unused          uint8          :offset 31)
    )
    */
-  void read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts, DrawStats* stats);
+  void read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts);
   std::string print(const PrintSettings& settings, int indent) const;
 
   u16 strip_data_len;
@@ -257,7 +238,7 @@ struct CollideFragMesh {
 };
 
 struct CollideFragment {
-  void read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts, DrawStats* stats);
+  void read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts);
   std::string print(const PrintSettings& settings, int indent) const;
   Vector bsphere;
   CollideFragMesh mesh;
@@ -266,7 +247,6 @@ struct CollideFragment {
 struct DrawableInlineArrayCollideFragment : public DrawableInlineArray {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -279,7 +259,6 @@ struct DrawableInlineArrayCollideFragment : public DrawableInlineArray {
 struct DrawableTreeCollideFragment : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -297,14 +276,13 @@ struct TFragmentDebugData {
   bool has_debug_lines;
 
   std::string print(int indent) const;
-  void read_from_file(Ref ref, const decompiler::DecompilerTypeSystem& dts, DrawStats* stats);
+  void read_from_file(Ref ref, const decompiler::DecompilerTypeSystem& dts);
 };
 
 // the "fragment" is just a collection of data that fits into the VU memory.
 struct TFragment : public Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "tfragment"; }
@@ -346,7 +324,6 @@ struct DrawableInlineArrayTFrag : public DrawableInlineArray {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -357,7 +334,6 @@ struct DrawableInlineArrayTFrag : public DrawableInlineArray {
 struct DrawableTreeTfrag : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -417,7 +393,6 @@ struct DrawableTreeTfragWater : public DrawableTreeTfrag {
 struct TieFragment : public Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "tie-fragment"; }
@@ -448,7 +423,6 @@ struct TieFragment : public Drawable {
 struct InstanceTie : public Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "instance-tie"; }
@@ -470,7 +444,6 @@ struct InstanceTie : public Drawable {
 struct PrototypeTie : public DrawableInlineArray {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -524,7 +497,6 @@ struct PrototypeBucketTie {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
   std::string print(const PrintSettings& settings, int indent) const;
 };
@@ -539,7 +511,6 @@ struct PrototypeArrayTie {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
   std::string print(const PrintSettings& settings, int indent) const;
 };
@@ -549,7 +520,6 @@ struct PrototypeArrayTie {
 struct ProxyPrototypeArrayTie {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
   std::string print(const PrintSettings& settings, int indent) const;
 
@@ -567,7 +537,6 @@ struct DrawableInlineArrayInstanceTie : public DrawableInlineArray {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -577,7 +546,6 @@ struct DrawableInlineArrayInstanceTie : public DrawableInlineArray {
 struct DrawableTreeInstanceTie : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version) override;
   std::string print(const PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -599,7 +567,6 @@ namespace shrub_types {
 struct Shrubbery : public level_tools::Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "shrubbery"; }
@@ -622,7 +589,6 @@ struct Shrubbery : public level_tools::Drawable {
 struct PrototypeShrubbery : public level_tools::DrawableInlineArray {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "prototype-shrubbery"; }
@@ -638,7 +604,6 @@ struct Billboard {};
 struct GenericShrubFragment : public level_tools::Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "generic-shrub-fragment"; }
@@ -670,7 +635,6 @@ struct GenericShrubFragment : public level_tools::Drawable {
 struct PrototypeGenericShrub : public level_tools::Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "prototype-generic-shrub"; }
@@ -709,7 +673,6 @@ struct PrototypeBucketShrub {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version);
   std::string print(const level_tools::PrintSettings& settings, int indent) const;
 };
@@ -722,7 +685,6 @@ struct PrototypeInlineArrayShrub {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version);
   std::string print(const level_tools::PrintSettings& settings, int indent) const;
 };
@@ -730,7 +692,6 @@ struct PrototypeInlineArrayShrub {
 struct PrototypeArrayShrubInfo {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version);
   std::string print(const level_tools::PrintSettings& settings, int indent) const;
 
@@ -741,7 +702,6 @@ struct PrototypeArrayShrubInfo {
 struct InstanceShrubbery : public level_tools::Drawable {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "instance-shrubbery"; }
@@ -766,7 +726,6 @@ struct DrawableInlineArrayInstanceShrub : public level_tools::DrawableInlineArra
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override { return "drawable-inline-array-instance-shrub"; }
@@ -775,7 +734,6 @@ struct DrawableInlineArrayInstanceShrub : public level_tools::DrawableInlineArra
 struct DrawableTreeInstanceShrub : public level_tools::DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      level_tools::DrawStats* stats,
                       GameVersion version) override;
   std::string print(const level_tools::PrintSettings& settings, int indent) const override;
   std::string my_type() const override;
@@ -804,7 +762,6 @@ struct DrawableInlineArrayActor {
   std::vector<DrawableActor> drawable_actors;
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
 };
 
@@ -814,10 +771,57 @@ struct CollideHash {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
 };
 
+/*
+((start-corner     vector :inline :offset-assert 32)
+ (spheres          (inline-array vector)         :offset-assert 48)
+ (visids           (pointer int16)         :offset-assert 52)
+ (shaders          (inline-array adgif-shader) :offset-assert 56)
+ (colors           time-of-day-palette       :offset-assert 60)
+ (montage          uint32         :offset-assert 64)
+ (buckets-far      (inline-array hfrag-bucket)   :offset-assert 68)
+ (buckets-mid      (inline-array hfrag-bucket)         :offset-assert 72)
+ (buckets-near     (inline-array hfrag-bucket)         :offset-assert 76)
+ (verts            (inline-array hfrag-vertex) :offset-assert 80) ;; wrong type? 8288
+ (pat-array        (pointer pat-surface) :offset-assert 84)
+ (pat-length       uint16         :offset-assert 88)
+ (num-buckets-far  uint16         :offset-assert 90)
+ (num-buckets-mid  uint16         :offset-assert 92)
+ (num-buckets-near uint16         :offset-assert 94)
+ (size             uint32         :offset 44 :score 1)
+ */
+
+struct HFragmentMontage {
+  u16 table[16];
+};
+
+struct HFragment {
+  void read_from_file(TypedRef ref, const decompiler::DecompilerTypeSystem& dts);
+
+  static constexpr int kCornersPerEdge = 32;
+  static constexpr int kNumCorners = kCornersPerEdge * kCornersPerEdge;
+
+  static constexpr int kVertsPerEdge = 512;
+  static constexpr int kNumVerts = 512 * 512;
+
+  level_tools::Vector start_corner;          // location of corner (0, 0)
+  std::vector<level_tools::Vector> spheres;  // array of bspheres for each "corner"
+  std::vector<s16> vis_ids;                  // precomputed vis id for each "corner"
+  AdGifData shaders[3];
+  TimeOfDayPalette colors;
+  HFragmentMontage montage[17];
+  // buckets??
+  std::vector<u32> verts;
+  // pat-array
+  // pat-length
+  u16 num_buckets_far = 0;
+  u16 num_buckets_mid = 0;
+  u16 num_buckets_near = 0;
+  u32 size = 0;
+};
+
 ////////////////////////////////
 // Main Level Type (bsp-header)
 ////////////////////////////////
@@ -829,7 +833,6 @@ struct DrawableTreeArray {
 
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version);
 
   std::string print(const PrintSettings& settings, int indent) const;
@@ -864,6 +867,7 @@ struct BspHeader {
 
   //  (visible-list-length int32 :offset-assert 36)
   s32 visible_list_length = -1;
+  s32 extra_vis_list_length = -1;  // jak 2+ only
 
   //  (drawable-trees drawable-tree-array :offset-assert 40)
   DrawableTreeArray drawable_tree_array;
@@ -910,9 +914,11 @@ struct BspHeader {
   // jak 2 only
   CollideHash collide_hash;
 
+  // jak 3 only
+  std::optional<HFragment> hfrag;
+
   void read_from_file(const decompiler::LinkedObjectFile& file,
                       const decompiler::DecompilerTypeSystem& dts,
-                      DrawStats* stats,
                       GameVersion version,
                       bool only_read_texture_remap = false);