Rework extraction of equipment and accessory (#664)

When extracting equipment and accessories, we use scripts in
tools/splat_ext called accessory.py and equipment.py, each called with a
different makefile rule. These files have a lot of "data in code", with
function calls matching the structure of the equipment and accessory
structs. They also involve repeating the structure both in the
extracting from the game, and the rebuilding into the compiled
executable.

This PR takes the equipment and accessories, and turns them into a
single generic makefile rule, which calls a new script called
`assets.py`. We change the splat yaml to match. To control the
extraction, we add new _config.json files in the splat_ext. This means
we only need to define the structure of each of these in a single,
localized place.

If we like this new approach (which should be more flexible), I will see
whether it can work for more of the assets in that same area of the
makefile. I will then see about adding extraction of enemies (in
g_EnemyDefs) next.

assets.py is adapted from the old equipment.py with a lot of changes to
make it work more flexibly. I'm hoping it will be a nicer path forward
into the future.

Makefile

@@ -428,11 +428,8 @@ $(BUILD_DIR)/$(ASSETS_DIR)/%.spritesheet.json.o: $(ASSETS_DIR)/%.spritesheet.jso
 $(BUILD_DIR)/$(ASSETS_DIR)/%.animset.json.o: $(ASSETS_DIR)/%.animset.json
 	./tools/splat_ext/animset.py gen-asm $< $(BUILD_DIR)/$(ASSETS_DIR)/$*.s
 	$(AS) $(AS_FLAGS) -o $(BUILD_DIR)/$(ASSETS_DIR)/$*.o $(BUILD_DIR)/$(ASSETS_DIR)/$*.s
-$(BUILD_DIR)/$(ASSETS_DIR)/%.equipment.json.o: $(ASSETS_DIR)/%.equipment.json
+$(BUILD_DIR)/$(ASSETS_DIR)/%.json.o: $(ASSETS_DIR)/%.json
-	./tools/splat_ext/equipment.py $< $(BUILD_DIR)/$(ASSETS_DIR)/$*.bin
+	./tools/splat_ext/assets.py $< $(BUILD_DIR)/$(ASSETS_DIR)/$*.bin
-	$(LD) -r -b binary -o $(BUILD_DIR)/$(ASSETS_DIR)/$*.o $(BUILD_DIR)/$(ASSETS_DIR)/$*.bin
-$(BUILD_DIR)/$(ASSETS_DIR)/%.accessory.json.o: $(ASSETS_DIR)/%.accessory.json
-	./tools/splat_ext/accessory.py $< $(BUILD_DIR)/$(ASSETS_DIR)/$*.bin
 	$(LD) -r -b binary -o $(BUILD_DIR)/$(ASSETS_DIR)/$*.o $(BUILD_DIR)/$(ASSETS_DIR)/$*.bin
 $(BUILD_DIR)/$(ASSETS_DIR)/%.tilelayout.bin.o: $(ASSETS_DIR)/%.tilelayout.bin
 	$(LD) -r -b binary -o $(BUILD_DIR)/$(ASSETS_DIR)/$*.o $(ASSETS_DIR)/$*.tilelayout.bin

config/splat.us.dra.yaml

@@ -38,8 +38,8 @@ segments:
       - [0x2EE8, data]
       - [0x3C40, data]
       - [0x4A00, data]
-      - [0x4B04, equipment, equipments]
+      - [0x4B04, assets, equipment]
-      - [0x7718, accessory, accessory_data]
+      - [0x7718, assets, accessory]
       - [0x8258, data]
       - [0x8900, data]
       - [0xCEB0, data]

tools/splat_ext/accessory_config.json

@@ -0,0 +1,15 @@
+{
+    "name_addr": "str_ptr",
+    "desc_addr": "str_ptr",
+    "attBonus": "s16",
+    "defBonus": "s16",
+    "strBonus": "s8",
+    "conBonus": "s8",
+    "intBonus": "s8",
+    "lckBonus": "s8",
+    "unk10": "u32",
+    "unk14": "u32",
+    "icon": "s16",
+    "iconPalette": "s16",
+    "unk1C": "u32"
+}

tools/splat_ext/assets.py

@@ -1,10 +1,5 @@
 #!/usr/bin/python3
 
-# Note: This file was created by bismurphy. It is effectively the same as
-# equipment.py, just with the current best-known state of the accessory
-# struct subbed into the two locations where it comes up.
-# Longer term, would be cool if equipment and accessories could be parsed
-# by the same Python script, rather than tons of duplicate code.
 import json
 import os
 import sys
@@ -17,36 +12,62 @@ from util import options, log
 from segtypes.n64.segment import N64Segment
 import utils
 
-item_size = 0x20  # sizeof(Accessory)
+
+def get_serializer(dataType: str):
+    match dataType:
+        case "str_ptr":
+            return utils.from_ptr_str
+        case "bool":
+            return utils.from_bool
+        case "s8":
+            return utils.from_s8
+        case "u8":
+            return utils.from_u8
+        case "s16":
+            return utils.from_16
+        case "u16":
+            return utils.from_16
+        case "u32":
+            return utils.from_u32
+        case _:
+            print(f"Failed to find serializer for {dataType}")
 
 
-def serialize_accessory(content: str) -> bytearray:
+def get_parser_and_size(dataType: str):
+    match dataType:
+        case "str_ptr":
+            return (utils.to_ptr_str, 4)
+        case "bool":
+            return (utils.to_bool, 1)
+        case "s8":
+            return (utils.to_s8, 1)
+        case "u8":
+            return (utils.to_u8, 1)
+        case "s16":
+            return (utils.to_s16, 2)
+        case "u16":
+            return (utils.to_u16, 2)
+        case "u32":
+            return (utils.to_u32, 4)
+        case _:
+            print(f"Failed to find parser for {dataType}")
+
+
+def serialize_asset(content: str, asset_config: str) -> bytearray:
     obj = json.loads(content)
+    config = json.loads(asset_config)
     item_count = len(obj)
     serialized_data = bytearray()
     for i in range(0, item_count):
         item = obj[i]
-        serialized_data += utils.from_ptr_str(item["name_addr"])
+        for entry, entryType in config.items():
-        serialized_data += utils.from_ptr_str(item["desc_addr"])
+            serializer = get_serializer(entryType)
-        serialized_data += utils.from_16(item["attBonus"])
+            serialized_data += serializer(item[entry])
-        serialized_data += utils.from_16(item["defBonus"])
-        serialized_data += utils.from_s8(item["strBonus"])
-        serialized_data += utils.from_s8(item["conBonus"])
-        serialized_data += utils.from_s8(item["intBonus"])
-        serialized_data += utils.from_s8(item["lckBonus"])
-        serialized_data += utils.from_u32(item["unk10"])
-        serialized_data += utils.from_u32(item["unk14"])
-        serialized_data += utils.from_16(item["icon"])
-        serialized_data += utils.from_16(item["iconPalette"])
-        serialized_data += utils.from_u32(item["unk1C"])
-
-    expected_data_size = item_count * item_size
-    assert len(serialized_data) == expected_data_size
 
     return serialized_data
 
 
-class PSXSegAccessory(N64Segment):
+class PSXSegAssets(N64Segment):
     def __init__(self, rom_start, rom_end, type, name, vram_start, args, yaml):
         super().__init__(rom_start, rom_end, type, name, vram_start, args, yaml),
 
@@ -54,24 +75,31 @@ class PSXSegAccessory(N64Segment):
         return options.opts.asset_path / self.dir / self.name
 
     def src_path(self) -> Optional[Path]:
-        return options.opts.asset_path / self.dir / f"{self.name}.accessory.json"
+        return options.opts.asset_path / self.dir / f"{self.name}.json"
 
     def split(self, rom_bytes):
         path = self.src_path()
         path.parent.mkdir(parents=True, exist_ok=True)
 
-        data = self.parse_accessory(rom_bytes[self.rom_start : self.rom_end], rom_bytes)
+        data = self.parse_asset(rom_bytes[self.rom_start : self.rom_end], rom_bytes)
         with open(path, "w") as f:
             f.write(json.dumps(data, indent=4))
 
-    def parse_accessory(self, data: bytearray, rom: bytearray) -> list:
+    def parse_asset(self, data: bytearray, rom: bytearray) -> list:
         def get_ptr_data(src_ptr_data):
             return rom[
                 utils.to_u32(src_ptr_data) - (self.vram_start - self.rom_start) :
             ]
 
+        config_file_name = f"tools/splat_ext/{self.name}_config.json"
+        with open(config_file_name, "r") as config_in:
+            config_json = config_in.read()
+        config = json.loads(config_json)
+
+        item_size = sum(get_parser_and_size(x)[1] for x in config.values())
         count = int(len(data) / item_size)
         expected_data_size = count * item_size
+
         if len(data) != expected_data_size:
             log.write(
                 f"data for '{self.name}' is {expected_data_size - len(data)} too long. Data might look incorrect.",
@@ -92,21 +120,12 @@ class PSXSegAccessory(N64Segment):
                 "desc_resolved": utils.sotn_menu_desc_to_str(
                     get_ptr_data(item_data[0x04:])
                 ),
-                # debugging stuff ends
-                "name_addr": utils.to_ptr_str(item_data[0x00:]),
-                "desc_addr": utils.to_ptr_str(item_data[0x04:]),
-                "attBonus": utils.to_s16(item_data[0x08:]),
-                "defBonus": utils.to_s16(item_data[0x0A:]),
-                "strBonus": utils.to_s8(item_data[0x0C:]),
-                "conBonus": utils.to_s8(item_data[0x0D:]),
-                "intBonus": utils.to_s8(item_data[0x0E:]),
-                "lckBonus": utils.to_s8(item_data[0x0F:]),
-                "unk10": utils.to_u32(item_data[0x10:]),
-                "unk14": utils.to_u32(item_data[0x14:]),
-                "icon": utils.to_u16(item_data[0x18:]),
-                "iconPalette": utils.to_u16(item_data[0x1A:]),
-                "unk1C": utils.to_u32(item_data[0x1C:]),
             }
+            data_pointer = 0
+            for entry, entryType in config.items():
+                parser, dataSizeBytes = get_parser_and_size(entryType)
+                item[entry] = parser(item_data[data_pointer:])
+                data_pointer += dataSizeBytes
             items.append(item)
         return items
 
@@ -114,8 +133,12 @@ class PSXSegAccessory(N64Segment):
 if __name__ == "__main__":
     input_file_name = sys.argv[1]
     output_file_name = sys.argv[2]
+    config_file_name = input_file_name.replace(".json", "_config.json")
+    config_file_name = config_file_name.replace("assets/dra", "tools/splat_ext")
+    with open(config_file_name, "r") as config_in:
+        config_json = config_in.read()
 
     with open(input_file_name, "r") as f_in:
-        data = serialize_accessory(f_in.read())
+        data = serialize_asset(f_in.read(), config_json)
         with open(output_file_name, "wb") as f_out:
             f_out.write(data)

tools/splat_ext/equipment.py

@@ -1,148 +0,0 @@
-#!/usr/bin/python3
-
-import json
-import os
-import sys
-from typing import Optional
-from pathlib import Path
-
-sys.path.append(f"{os.getcwd()}/tools/n64splat")
-sys.path.append(f"{os.getcwd()}/tools/splat_ext")
-from util import options, log
-from segtypes.n64.segment import N64Segment
-import utils
-
-item_size = 0x34  # sizeof(Equipment)
-
-
-def serialize_equipment(content: str) -> bytearray:
-    obj = json.loads(content)
-    item_count = len(obj)
-    serialized_data = bytearray()
-    for i in range(0, item_count):
-        item = obj[i]
-        serialized_data += utils.from_ptr_str(item["name_addr"])
-        serialized_data += utils.from_ptr_str(item["desc_addr"])
-        serialized_data += utils.from_16(item["attack"])
-        serialized_data += utils.from_16(item["defense"])
-        serialized_data += utils.from_16(item["element"])
-        serialized_data += utils.from_u8(item["itemCategory"])
-        serialized_data += utils.from_u8(item["weaponId"])
-        serialized_data += utils.from_u8(item["palette"])
-        serialized_data += utils.from_u8(item["unk11"])
-        serialized_data += utils.from_u8(item["playerAnim"])
-        serialized_data += utils.from_u8(item["unk13"])
-        serialized_data += utils.from_u8(item["unk14"])
-        serialized_data += utils.from_u8(item["lockDuration"])
-        serialized_data += utils.from_u8(item["chainLimit"])
-        serialized_data += utils.from_u8(item["unk17"])
-        serialized_data += utils.from_u8(item["specialMove"])
-        serialized_data += utils.from_bool(item["isConsumable"])
-        serialized_data += utils.from_u8(item["enemyInvincibilityFrames"])
-        serialized_data += utils.from_u8(item["unk1B"])
-        serialized_data += utils.from_u32(item["unk1C"])
-        serialized_data += utils.from_u32(item["unk20"])
-        serialized_data += utils.from_16(item["mpUsage"])
-        serialized_data += utils.from_16(item["stunFrames"])
-        serialized_data += utils.from_16(item["hitType"])
-        serialized_data += utils.from_16(item["hitEffect"])
-        serialized_data += utils.from_16(item["icon"])
-        serialized_data += utils.from_16(item["iconPalette"])
-        serialized_data += utils.from_16(item["criticalRate"])
-        serialized_data += utils.from_16(item["unk32"])
-
-    expected_data_size = item_count * item_size
-    assert len(serialized_data) == expected_data_size
-
-    return serialized_data
-
-
-class PSXSegEquipment(N64Segment):
-    def __init__(self, rom_start, rom_end, type, name, vram_start, args, yaml):
-        super().__init__(rom_start, rom_end, type, name, vram_start, args, yaml),
-
-    def out_path(self) -> Optional[Path]:
-        return options.opts.asset_path / self.dir / self.name
-
-    def src_path(self) -> Optional[Path]:
-        return options.opts.asset_path / self.dir / f"{self.name}.equipment.json"
-
-    def split(self, rom_bytes):
-        path = self.src_path()
-        path.parent.mkdir(parents=True, exist_ok=True)
-
-        data = self.parse_equipment(rom_bytes[self.rom_start : self.rom_end], rom_bytes)
-        with open(path, "w") as f:
-            f.write(json.dumps(data, indent=4))
-
-    def parse_equipment(self, data: bytearray, rom: bytearray) -> list:
-        def get_ptr_data(src_ptr_data):
-            return rom[
-                utils.to_u32(src_ptr_data) - (self.vram_start - self.rom_start) :
-            ]
-
-        count = int(len(data) / item_size)
-        expected_data_size = count * item_size
-        if len(data) != expected_data_size:
-            log.write(
-                f"data for '{self.name}' is {expected_data_size - len(data)} too long. Data might look incorrect.",
-                status="warn",
-            )
-
-        items = []
-        for i in range(0, count):
-            item_data = data[i * item_size :][:item_size]
-            item = {
-                # debugging stuff
-                "id": i,
-                "id_hex": hex(i)[2:].upper(),
-                "ram_addr": hex(self.vram_start + i * item_size)[2:].upper(),
-                "name_resolved": utils.sotn_menu_name_to_str(
-                    get_ptr_data(item_data[0x00:])
-                ),
-                "desc_resolved": utils.sotn_menu_desc_to_str(
-                    get_ptr_data(item_data[0x04:])
-                ),
-                # debugging stuff ends
-                "name_addr": utils.to_ptr_str(item_data[0x00:]),
-                "desc_addr": utils.to_ptr_str(item_data[0x04:]),
-                "attack": utils.to_s16(item_data[0x08:]),
-                "defense": utils.to_s16(item_data[0x0A:]),
-                "element": utils.to_u16(item_data[0x0C:]),
-                "itemCategory": utils.to_u8(item_data[0x0E:]),
-                "weaponId": utils.to_u8(item_data[0x0F:]),
-                "palette": utils.to_u8(item_data[0x10:]),
-                "unk11": utils.to_u8(item_data[0x11:]),
-                "playerAnim": utils.to_u8(item_data[0x12:]),
-                "unk13": utils.to_u8(item_data[0x13:]),
-                "unk14": utils.to_u8(item_data[0x14:]),
-                "lockDuration": utils.to_u8(item_data[0x15:]),
-                "chainLimit": utils.to_u8(item_data[0x16:]),
-                "unk17": utils.to_u8(item_data[0x17:]),
-                "specialMove": utils.to_u8(item_data[0x18:]),
-                "isConsumable": utils.to_bool(item_data[0x19:]),
-                "enemyInvincibilityFrames": utils.to_u8(item_data[0x1A:]),
-                "unk1B": utils.to_u8(item_data[0x1B:]),
-                "unk1C": utils.to_u32(item_data[0x1C:]),
-                "unk20": utils.to_u32(item_data[0x20:]),
-                "mpUsage": utils.to_u16(item_data[0x24:]),
-                "stunFrames": utils.to_u16(item_data[0x26:]),
-                "hitType": utils.to_u16(item_data[0x28:]),
-                "hitEffect": utils.to_u16(item_data[0x2A:]),
-                "icon": utils.to_u16(item_data[0x2C:]),
-                "iconPalette": utils.to_u16(item_data[0x2E:]),
-                "criticalRate": utils.to_u16(item_data[0x30:]),
-                "unk32": utils.to_u16(item_data[0x32:]),
-            }
-            items.append(item)
-        return items
-
-
-if __name__ == "__main__":
-    input_file_name = sys.argv[1]
-    output_file_name = sys.argv[2]
-
-    with open(input_file_name, "r") as f_in:
-        data = serialize_equipment(f_in.read())
-        with open(output_file_name, "wb") as f_out:
-            f_out.write(data)

tools/splat_ext/equipment_config.json

@@ -0,0 +1,31 @@
+    {
+        "name_addr": "str_ptr",
+        "desc_addr": "str_ptr",
+        "attack": "s16",
+        "defense": "s16",
+        "element": "u16",
+        "itemCategory": "u8",
+        "weaponId": "u8",
+        "palette": "u8",
+        "unk11": "u8",
+        "playerAnim": "u8",
+        "unk13": "u8",
+        "unk14": "u8",
+        "lockDuration": "u8",
+        "chainLimit": "u8",
+        "unk17": "u8",
+        "specialMove": "u8",
+        "isConsumable": "bool",
+        "enemyInvincibilityFrames": "u8",
+        "unk1B": "u8",
+        "unk1C": "u32",
+        "unk20": "u32",
+        "mpUsage": "s16",
+        "stunFrames": "s16",
+        "hitType": "s16",
+        "hitEffect": "s16",
+        "icon": "s16",
+        "iconPalette": "s16",
+        "criticalRate": "s16",
+        "unk32": "s16"
+    }