Extract stage map into individual files (#1316)

This took me a lot of manual work. But I think I confirmed a pattern
that will help me to automate all of this for all the next stage
overlays that will be imported in the repo.

I noticed some stages with only one room having more than two layers or
more than two tile definitions, it might be either debugging or unused
content? I did not have time to explore any of that.

I modified the `tiledef` splat extension to greatly minimise the set-up
and noise.

.gitignore

@@ -29,3 +29,5 @@ config/saturn/zero_syms.txt
 
 sotn_calltree.txt
 function_calls
+
+gmon.out

config/splat.us.stcen.yaml

@@ -34,6 +34,8 @@ segments:
     subalign: 4
     subsegments:
       - [0x0, data]
+      - [0xDC, layers, rooms]
+      - [0x134, data]
       - [0x7E4, .data, collision]
       - [0xBA4, .data, e_red_door]
       - [0xBBC, data]
@@ -41,6 +43,12 @@ segments:
       - [0x1110, data]
       - [0x12D4, roomdef, g_Rooms]
       - [0x1300, data]
+      - [0x69EC, tilelayout, D_801869EC]
+      - [0x6DEC, tilelayout, D_80186DEC]
+      - [0x7FEC, tilelayout, D_80187FEC]
+      - [0x81EC, tiledef, t_801884EC, D_801884EC]
+      - [0x85FC, tiledef, t_8018C5FC, D_8018C5FC]
+      - [0xC60C, data]
       - [0xD42C, .rodata, holyglassescutscene]
       - [0xD4B0, .rodata, F890]
       - [0xD4D8, .rodata, e_red_door] # EntityRedDoor

config/splat.us.stdre.yaml

@@ -34,6 +34,8 @@ segments:
     subalign: 4
     subsegments:
       - [0x0, data]
+      - [0xE8, layers, rooms]
+      - [0x128, data]
       - [0x590, data]
       - [0x610, data]
       - [0x630, data]
@@ -51,6 +53,11 @@ segments:
       - [0x1498, roomdef, g_Rooms]
       - [0x14AC, data]
       - [0x16C0, data]
+      - [0xB548, tilelayout, D_8018B548]
+      - [0xB948, tilelayout, D_8018B948]
+      - [0xBD48, tiledef, t_8018C148, D_8018C148]
+      - [0xC158, tiledef, t_80190158, D_80190158]
+      - [0x10168, data]
       - [0x1171C, .rodata, 11A64]
       - [0x11728, .rodata, succubus] # EntitySuccubus
       - [0x11808, .rodata, succubuscutscene]

config/splat.us.stno3.yaml

@@ -34,6 +34,8 @@ segments:
     subalign: 4
     subsegments:
       - [0x0, data]
+      - [0x1C4, layers, rooms]
+      - [0x5A4, data]
       - [0x920, data]
       - [0xAAC, data]
       - [0xDC0, data]
@@ -58,6 +60,53 @@ segments:
       - [0x3CC4, roomdef, g_Rooms]
       - [0x3DC8, data]
       - [0x4CE0, data]
+      - [0x18838, tilelayout, D_80198838]
+      - [0x19438, tilelayout, D_80199438]
+      - [0x19638, tilelayout, D_80199638]
+      - [0x19A38, tilelayout, D_80199A38]
+      - [0x19E38, tilelayout, D_80199E38]
+      - [0x1AC38, tilelayout, D_8019AC38]
+      - [0x1AE38, tilelayout, D_8019AE38]
+      - [0x1B038, tilelayout, D_8019B038]
+      - [0x1B238, tilelayout, D_8019B238]
+      - [0x1B638, tilelayout, D_8019B638]
+      - [0x1BA38, tilelayout, D_8019BA38]
+      - [0x1C238, tilelayout, D_8019C238]
+      - [0x1CA38, tilelayout, D_8019CA38]
+      - [0x1CC38, tilelayout, D_8019CC38]
+      - [0x1CE38, tilelayout, D_8019CE38]
+      - [0x1DA38, tilelayout, D_8019DA38]
+      - [0x1E638, tilelayout, D_8019E638]
+      - [0x1E838, tilelayout, D_8019E838]
+      - [0x1EA38, tilelayout, D_8019EA38]
+      - [0x1F638, tilelayout, D_8019F638]
+      - [0x1F838, tilelayout, D_8019F838]
+      - [0x1FA38, tilelayout, D_8019FA38]
+      - [0x1FC38, tilelayout, D_8019FC38]
+      - [0x20038, tilelayout, D_801A0038]
+      - [0x20238, tilelayout, D_801A0238]
+      - [0x20838, tilelayout, D_801A0838]
+      - [0x20E38, tilelayout, D_801A0E38]
+      - [0x21038, tilelayout, D_801A1038]
+      - [0x21238, tilelayout, D_801A1238]
+      - [0x21438, tilelayout, D_801A1438]
+      - [0x21638, tilelayout, D_801A1638]
+      - [0x22238, tilelayout, D_801A2238]
+      - [0x22E38, tilelayout, D_801A2E38]
+      - [0x23038, tilelayout, D_801A3038]
+      - [0x23238, tilelayout, D_801A3238]
+      - [0x23438, tilelayout, D_801A3438]
+      - [0x23638, tilelayout, D_801A3638]
+      - [0x25A38, tilelayout, D_801A5A38]
+      - [0x25C38, tilelayout, D_801A5C38]
+      - [0x25E38, tilelayout, D_801A5E38]
+      - [0x26038, tilelayout, D_801A6038]
+      - [0x26238, tilelayout, D_801A6238]
+      - [0x26438, tilelayout, D_801A6438]
+      - [0x26638, tiledef, t_801A6A38, D_801A6A38]
+      - [0x26A48, tiledef, t_801AAA48, D_801AAA48]
+      - [0x2AA58, tiledef, t_801AEA58, D_801AEA58]
+      - [0x2EA68, data]
       - [0x373E0, data]
       - [0x373E8, .rodata, 377D4] # EntityCastleDoor
       - [0x37400, .rodata, 377D4] # EntityStairwayPiece

config/splat.us.stnp3.yaml

@@ -34,6 +34,8 @@ segments:
     subalign: 4
     subsegments:
       - [0x0, data]
+      - [0x1D0, layers, rooms]
+      - [0x558, data]
       - [0xE50, data]
       - [0x10C0, data]
       - [0x1300, data]
@@ -60,6 +62,49 @@ segments:
       - [0x3760, data]
       - [0x3A7C, roomdef, g_Rooms]
       - [0x3B68, data]
+      - [0x168F4, tilelayout, D_801968F4]
+      - [0x174F4, tilelayout, D_801974F4]
+      - [0x176F4, tilelayout, D_801976F4]
+      - [0x17AF4, tilelayout, D_80197AF4]
+      - [0x17EF4, tilelayout, D_80197EF4]
+      - [0x18CF4, tilelayout, D_80198CF4]
+      - [0x18EF4, tilelayout, D_80198EF4]
+      - [0x190F4, tilelayout, D_801990F4]
+      - [0x192F4, tilelayout, D_801992F4]
+      - [0x196F4, tilelayout, D_801996F4]
+      - [0x19AF4, tilelayout, D_80199AF4]
+      - [0x1A2F4, tilelayout, D_8019A2F4]
+      - [0x1AAF4, tilelayout, D_8019AAF4]
+      - [0x1ACF4, tilelayout, D_8019ACF4]
+      - [0x1AEF4, tilelayout, D_8019AEF4]
+      - [0x1BAF4, tilelayout, D_8019BAF4]
+      - [0x1C6F4, tilelayout, D_8019C6F4]
+      - [0x1C8F4, tilelayout, D_8019C8F4]
+      - [0x1CAF4, tilelayout, D_8019CAF4]
+      - [0x1D6F4, tilelayout, D_8019D6F4]
+      - [0x1D8F4, tilelayout, D_8019D8F4]
+      - [0x1DAF4, tilelayout, D_8019DAF4]
+      - [0x1DCF4, tilelayout, D_8019DCF4]
+      - [0x1E0F4, tilelayout, D_8019E0F4]
+      - [0x1E2F4, tilelayout, D_8019E2F4]
+      - [0x1E8F4, tilelayout, D_8019E8F4]
+      - [0x1EEF4, tilelayout, D_8019EEF4]
+      - [0x1F0F4, tilelayout, D_8019F0F4]
+      - [0x1F2F4, tilelayout, D_8019F2F4]
+      - [0x1F4F4, tilelayout, D_8019F4F4]
+      - [0x1F6F4, tilelayout, D_8019F6F4]
+      - [0x202F4, tilelayout, D_801A02F4]
+      - [0x20EF4, tilelayout, D_801A0EF4]
+      - [0x210F4, tilelayout, D_801A10F4]
+      - [0x212F4, tilelayout, D_801A12F4]
+      - [0x214F4, tilelayout, D_801A14F4]
+      - [0x216F4, tilelayout, D_801A16F4]
+      - [0x218F4, tilelayout, D_801A18F4]
+      - [0x21AF4, tilelayout, D_801A1AF4]
+      - [0x21CF4, tiledef, t_801A20F4, D_801A20F4]
+      - [0x22104, tiledef, t_801A6104, D_801A6104]
+      - [0x26114, tiledef, t_801AA114, D_801AA114]
+      - [0x2A124, data]
       - [0x31EA0, data]
       - [0x31EA8, .rodata, 3246C] # EntityStairwayPiece
       - [0x31EBC, .rodata, 365FC] # func_801B65FC

config/splat.us.stnz0.yaml

@@ -85,7 +85,10 @@ segments:
       - [0x2045C, tilelayout, D_801A045C]
       - [0x2065C, tilelayout, D_801A065C]
       - [0x2085C, tilelayout, D_801A085C]
-      - [0x20A5C, data]
+      - [0x20A5C, tiledef, t_801A0E5C, D_801A0E5C]
+      - [0x20E6C, tiledef, t_801A4E6C, D_801A4E6C]
+      - [0x24E7C, tiledef, t_801A6E7C, D_801A6E7C]
+      - [0x26E8C, data]
       - [0x305A4, .rodata, 30958] # func_801B3C38
       - [0x305B8, .rodata, bossfight] # EntityBossFightManager
       - [0x305E8, .rodata, bossfight] # EntityBossRoomBlock

config/splat.us.stst0.yaml

@@ -34,6 +34,8 @@ segments:
     subalign: 4
     subsegments:
       - [0x0, data]
+      - [0x124, layers, rooms]
+      - [0x1A4, data]
       - [0x580, data]
       - [0x770, data]
       - [0x828, .data, draculacutscene] # "Richter, Dracula"
@@ -51,6 +53,13 @@ segments:
       - [0x187BC, data]
       #- [0x1A40C, cmp]
       - [0x1A750, data]
+      - [0x1B2D0, tilelayout, D_8019B2D0]
+      - [0x1C6D0, tilelayout, D_8019C6D0]
+      - [0x1DAD0, tilelayout, D_8019DAD0]
+      - [0x1E0D0, tilelayout, D_8019E0D0]
+      - [0x1E2D0, tilelayout, D_8019E2D0]
+      - [0x1E6D0, tiledef, t_801A26D0, D_801A26D0]
+      - [0x226E0, data]
       - [0x27984, .rodata, 27D64]
       - [0x279E8, .rodata, draculacutscene] # EntityDraculaCutscene
       - [0x27A74, .rodata, 2A218] # EntityCutscene

config/splat.us.stwrp.yaml

@@ -72,16 +72,8 @@ segments:
       - [0x2368, tilelayout, D_80182368]
       - [0x2568, tilelayout, D_80182568]
       - [0x2768, tilelayout, D_80182768]
-      - [0x2968, raw, D_80182968]
+      - [0x2968, tiledef, t_80182D68, D_80182D68]
-      - [0x2A68, raw, D_80182A68]
+      - [0x2D78, tiledef, t_80186D78, D_80186D78]
-      - [0x2B68, raw, D_80182B68]
-      - [0x2C68, raw, D_80182C68]
-      - [0x2D68, tiledef, D_80182D68]
-      - [0x2D78, raw, D_80182D78]
-      - [0x3D78, raw, D_80183D78]
-      - [0x4D78, raw, D_80184D78]
-      - [0x5D78, raw, D_80185D78]
-      - [0x6D78, tiledef, D_80186D78]
       - [0x6D88, animset, D_80186D88]
       - [0x6E30, .rodata, 6FD0]
       - [0x6E70, .rodata, warp] # EntityWarpRoom

config/symbols.us.stwrp.txt

@@ -22,15 +22,7 @@ D_80182168 = 0x80182168;
 D_80182368 = 0x80182368;
 D_80182568 = 0x80182568;
 D_80182768 = 0x80182768;
-D_80182968 = 0x80182968;
-D_80182A68 = 0x80182A68;
-D_80182B68 = 0x80182B68;
-D_80182C68 = 0x80182C68;
 D_80182D68 = 0x80182D68;
-D_80182D78 = 0x80182D78;
-D_80183D78 = 0x80183D78;
-D_80184D78 = 0x80184D78;
-D_80185D78 = 0x80185D78;
 D_80186D78 = 0x80186D78;
 D_80186D88 = 0x80186D88;
 EntityWarpRoom = 0x801873A0;

src/pc/stages/stage_loader.c

@@ -25,7 +25,7 @@ static bool LoadRoomTileDef(FileStringified* file) {
 
     FILE* f;
     char buf[0x100];
-    snprintf(buf, sizeof(buf), "%s/%s.bin", desc->assetPath,
+    snprintf(buf, sizeof(buf), "%s/%s", desc->assetPath,
              JITEM("gfxPage")->valuestring);
     f = fopen(buf, "rb");
     if (f) {
@@ -35,7 +35,7 @@ static bool LoadRoomTileDef(FileStringified* file) {
         ERRORF("unable to load '%s'", buf);
         return false;
     }
-    snprintf(buf, sizeof(buf), "%s/%s.bin", desc->assetPath,
+    snprintf(buf, sizeof(buf), "%s/%s", desc->assetPath,
              JITEM("gfxIndex")->valuestring);
     f = fopen(buf, "rb");
     if (f) {
@@ -45,8 +45,8 @@ static bool LoadRoomTileDef(FileStringified* file) {
         ERRORF("unable to load '%s'", buf);
         return false;
     }
-    snprintf(buf, sizeof(buf), "%s/%s.bin", desc->assetPath,
+    snprintf(
-             JITEM("clut")->valuestring);
+        buf, sizeof(buf), "%s/%s", desc->assetPath, JITEM("clut")->valuestring);
     f = fopen(buf, "rb");
     if (f) {
         fread(g_TileDefToLoad->clut, 0x1000, 1, f);
@@ -55,7 +55,7 @@ static bool LoadRoomTileDef(FileStringified* file) {
         ERRORF("unable to load '%s'", buf);
         return false;
     }
-    snprintf(buf, sizeof(buf), "%s/%s.bin", desc->assetPath,
+    snprintf(buf, sizeof(buf), "%s/%s", desc->assetPath,
              JITEM("collision")->valuestring);
     f = fopen(buf, "rb");
     if (f) {
@@ -115,8 +115,11 @@ static bool LoadRoomLayerDef(LayerDef* l, cJSON* jitem, RoomLoadDesc* desc) {
     l->tileDef->clut = g_TileDefDataPool[g_TileDefIndex][2];
     l->tileDef->collision = g_TileDefDataPool[g_TileDefIndex][3];
     g_TileDefIndex++;
-    snprintf(buf, sizeof(buf), "%s/%s.tiledef.json", desc->assetPath,
+
-             JITEM("tiledef")->valuestring);
+    // hack to load t_8018xxxx.tiledef instead of D_8018xxxx.tiledef
+    snprintf(buf, sizeof(buf), "%s/t_%s.tiledef.json", desc->assetPath,
+             JITEM("tiledef")->valuestring + 2);
+
     g_TileDefToLoad = l->tileDef;
 
     return FileStringify(LoadRoomTileDef, buf, desc);

tools/splat_ext/layers.py

@@ -66,9 +66,10 @@ def generate_assembly_layers(writer: io.BufferedWriter, name: str, content: str)
                 | get_int(layer, "top", 0, 0x3F) << 6
                 | get_int(layer, "right", 0, 0x3F) << 12
                 | get_int(layer, "bottom", 0, 0x3F) << 18
-                | get_int(layer, "scrollMode", 0, 15) << 24
+                | get_int(layer, "scrollMode", 0, 0x1F) << 24
                 | (1 if get_bool(layer, "isSaveRoom") else 0) << 29
                 | (1 if get_bool(layer, "isLoadingRoom") else 0) << 30
+                | (1 if get_bool(layer, "unusedFlag") else 0) << 31
             )
             zPriority = get_int(layer, "zPriority", 0, 32768)  # s16 or u16?
             unkE = get_int(layer, "unkE", 0, 255)
@@ -150,7 +151,7 @@ class PSXSegLayers(N64Segment):
             if fgPtr < ram_start or bgPtr < ram_start:
                 min_start = min(fgPtr, bgPtr)
                 log.error(
-                    f"data for '{self.name}' needs to start at least from {min_start}"
+                    f"data for '{self.name}' needs to start at least from 0x{min_start:X}"
                 )
                 raise err
             layerDefOffsets.add(fgPtr)
@@ -164,7 +165,7 @@ class PSXSegLayers(N64Segment):
             roomOff -= 8
             if roomOff <= 0:
                 log.error(
-                    f"data for '{self.name}' needs to start at least from {min_start}"
+                    f"data for '{self.name}' needs to start at least from 0x{min_start:X}"
                 )
                 raise err
 
@@ -187,6 +188,7 @@ class PSXSegLayers(N64Segment):
                     "scrollMode": (flags >> 24) & 0x1F,
                     "isSaveRoom": ((flags >> 24) & 0x20) != 0,
                     "isLoadingRoom": ((flags >> 24) & 0x40) != 0,
+                    "unusedFlag": ((flags >> 24) & 0x80) != 0,
                     "zPriority": utils.to_u16(layerDefData[12:]),
                     "unkE": utils.to_u8(layerDefData[14:]),
                     "unkF": utils.to_u8(layerDefData[15:]),

tools/splat_ext/tiledef.py

@@ -16,17 +16,34 @@ from splat.util.symbols import spim_context
 import utils
 
 
-def generate_assembly_tiledef(writer: io.BufferedWriter, name: str, content: str):
+def generate_assembly_tiledef(
+    writer: io.BufferedWriter, base_dir: str, name: str, content: str
+):
+    def incbin(writer: io.BufferedWriter, path: str):
+        with open(path, "rb") as f:
+            for ch in f.read():
+                writer.write(f".byte {int(ch)}\n")
+
     obj = json.loads(content)
+    symbol_name = obj["name"]
     gfxPage = obj["gfxPage"]
     gfxIndex = obj["gfxIndex"]
     clut = obj["clut"]
     collision = obj["collision"]
 
     writer.write(".section .data\n")
-    writer.write(f".global {name}\n")
+    writer.write(f"gfxPage:\n")
-    writer.write(f"{name}:\n")
+    incbin(writer, f"{base_dir}/{gfxPage}")
-    writer.write(f".word {gfxPage}, {gfxIndex}, {clut}, {collision}\n")
+    writer.write(f"gfxIndex:\n")
+    incbin(writer, f"{base_dir}/{gfxIndex}")
+    writer.write(f"clut:\n")
+    incbin(writer, f"{base_dir}/{clut}")
+    writer.write(f"collision:\n")
+    incbin(writer, f"{base_dir}/{collision}")
+
+    writer.write(f".global {symbol_name}\n")
+    writer.write(f"{symbol_name}:\n")
+    writer.write(f".word gfxPage, gfxIndex, clut, collision\n")
 
 
 class PSXSegTiledef(N64Segment):
@@ -34,27 +51,64 @@ class PSXSegTiledef(N64Segment):
         super().__init__(rom_start, rom_end, type, name, vram_start, args, yaml),
 
     def out_path(self) -> Optional[Path]:
-        return options.opts.asset_path / self.dir / self.name
+        return options.opts.asset_path / self.dir / f"{self.name}"
 
     def src_path(self) -> Optional[Path]:
         return options.opts.asset_path / self.dir / f"{self.name}.tiledef.json"
 
     def split(self, rom_bytes):
-        path = self.src_path()
+        if len(self.args) != 1:
-        path.parent.mkdir(parents=True, exist_ok=True)
+            utils.log_fatal(f"unable to determine the real symbol name for {self.name}")
+        symbol_name = self.args[0]
 
-        data = self.parse_tiledef(rom_bytes[self.rom_start : self.rom_end])
+        out_file_name = self.src_path()
-        with open(path, "w") as f:
+        out_file_name.parent.mkdir(parents=True, exist_ok=True)
-            f.write(json.dumps(data, indent=4))
 
-    def parse_tiledef(self, data: bytearray):
+        header_data = rom_bytes[self.rom_end - 0x10 : self.rom_end]
-        return {
+        gfxPage = utils.to_u32(header_data[0:]) - self.vram_start
-            "gfxPage": self.get_symbol(utils.to_u32(data[0:])).given_name,
+        gfxIndex = utils.to_u32(header_data[4:]) - self.vram_start
-            "gfxIndex": self.get_symbol(utils.to_u32(data[4:])).given_name,
+        clut = utils.to_u32(header_data[8:]) - self.vram_start
-            "clut": self.get_symbol(utils.to_u32(data[8:])).given_name,
+        collision = utils.to_u32(header_data[12:]) - self.vram_start
-            "collision": self.get_symbol(utils.to_u32(data[12:])).given_name,
+        header_start = self.rom_end - 0x10 - self.rom_start
+
+        expected_start = min(gfxPage, min(gfxIndex, min(clut, collision)))
+        if expected_start != 0:
+            expected_start += self.rom_start
+            actual_start = self.rom_start
+            utils.log_fatal(
+                f"{self.name} should start from 0x{expected_start:X} but got 0x{actual_start:X}\n"
+                f"- [0x{expected_start:X}, tiledef, {self.name}]"
+            )
+
+        descriptor = {
+            "name": symbol_name,
+            "gfxPage": f"{self.name}.page.bin",
+            "gfxIndex": f"{self.name}.tile.bin",
+            "clut": f"{self.name}.clut.bin",
+            "collision": f"{self.name}.col.bin",
         }
 
+        file_sizes = [
+            gfxIndex - gfxPage,
+            clut - gfxIndex,
+            collision - clut,
+            header_start - collision,
+        ]
+
+        raw_data = rom_bytes[self.rom_start : self.rom_end]
+        out_dir = options.opts.asset_path / self.dir
+        with open(out_dir / descriptor["gfxPage"], "wb") as f:
+            f.write(raw_data[gfxPage:][: file_sizes[0]])
+        with open(out_dir / descriptor["gfxIndex"], "wb") as f:
+            f.write(raw_data[gfxIndex:][: file_sizes[1]])
+        with open(out_dir / descriptor["clut"], "wb") as f:
+            f.write(raw_data[clut:][: file_sizes[2]])
+        with open(out_dir / descriptor["collision"], "wb") as f:
+            f.write(raw_data[collision:][: file_sizes[3]])
+
+        with open(out_file_name, "w") as f:
+            f.write(json.dumps(descriptor, indent=4))
+
 
 if __name__ == "__main__":
 
@@ -69,6 +123,7 @@ if __name__ == "__main__":
     output_file_name = sys.argv[2]
 
     with open(input_file_name, "r") as f_in:
+        base_path = os.path.dirname(input_file_name)
         name = get_file_name(input_file_name)
         with open(output_file_name, "w") as f_out:
-            generate_assembly_tiledef(f_out, name, f_in.read())
+            generate_assembly_tiledef(f_out, base_path, name, f_in.read())