Use decompiled overlays when building ROM (#121)

* Restore padding in ObjVisiblock struct

* Copy overlays out of compiled `code.elf` to put into the ROM

This is just a "tempory fix"

It seems like the built code/assets in `code.elf` should not be copied
into `build/baserom/...` (over the original ROM's files) but instead
into a `build/decomp/...` tree or similar.

`dmadata_table.txt` would also need to be updated to read from the
correct location.

* Use dmadata_table.txt to generate build rules

`makerom_files.txt` & `makerom_uncompressed_files.txt` contained a lot
of the same data in `dmadata_table.txt`, so I added a small python
script to generate this information into `build/`

Segments are no longer dumped out of `code.elf` into `build/baserom/``,
instead they are put in `build/binary/`.

`linker_scripts/dmadata_script.txt` was checked in, but generated by
`dmadata.py`. I deleted it / moved it to `build/dmadata_script.txt.pre`.

I also introduced some sentinel files (`dep`). I ended up needing these
to make incremental builds work smoothly? (Without them, there were a
lot of steps that were getting re-triggered on every build.) If this
style isn't welcome, I can try to fiddle with the Makefile more to try
to avoid having them?

* Restore padding in BgLbfshot struct

* Touch sentinel file before command; rm on failure

* Restore padding in ObjKepnKoya struct

* Ensure asm/ directories exist before disasm steps

* Clean up Makefile rules

* Set default goal; silent objcopy; fix code_script path

* Fix ovl_En_Ginko_Man, ovl_Obj_Lightswitch merge

ovl_En_Encount2 still needs work to bring back to matching

* Fix ovl_En_Encount2 merge

Makefile

@@ -76,6 +76,7 @@ ELF := $(MM_ROM_NAME).elf
 SRC_DIRS := $(shell find src -type d)
 BASEROM_DIRS := $(shell find baserom -type d 2>/dev/null)
 COMP_DIRS := $(BASEROM_DIRS:baserom%=comp%)
+BINARY_DIRS := $(BASEROM_DIRS:baserom%=binary%)
 ASSET_XML_DIRS := $(shell find assets/xml* -type d)
 ASSET_SRC_DIRS := $(patsubst assets/xml%,assets/src%,$(ASSET_XML_DIRS))
 ASSET_FILES_XML := $(foreach dir,$(ASSET_XML_DIRS),$(wildcard $(dir)/*.xml))
@@ -89,18 +90,19 @@ C_O_FILES := $(C_FILES:%.c=build/%.o)
 S_O_FILES := $(S_FILES:asm/%.asm=build/asm/%.o)
 ASSET_O_FILES := $(ASSET_FILES_OUT:%.c=build/%.o)
 O_FILES := $(C_O_FILES) $(S_O_FILES) $(ASSET_O_FILES)
-ROM_FILES := $(shell cat ./tables/makerom_files.txt)
-UNCOMPRESSED_ROM_FILES := $(shell cat ./tables/makerom_uncompressed_files.txt)
-
-# Exclude code and scene files, they will be extracted from the file instead
-DMADATA_FILES := $(subst build/baserom/boot ,,$(UNCOMPRESSED_ROM_FILES))
-DMADATA_FILES := $(subst build/baserom/code ,,$(DMADATA_FILES))
-DMADATA_FILES := $(DMADATA_FILES:build/baserom/overlays/%=)
-DMADATA_FILES := $(DMADATA_FILES:build/baserom/assets/scenes/%=)
-DMADATA_FILES := $(DMADATA_FILES:build/uncompressed_dmadata=)
 
 # create build directories
-$(shell mkdir -p build/linker_scripts build/asm build/asm/boot build/asm/code build/asm/overlays $(foreach dir,$(BASEROM_DIRS) $(COMP_DIRS) $(SRC_DIRS) $(ASSET_SRC_DIRS),$(shell mkdir -p build/$(dir))))
+$(shell mkdir -p build/linker_scripts build/asm build/asm/boot build/asm/code build/asm/overlays $(foreach dir,$(BASEROM_DIRS) $(COMP_DIRS) $(BINARY_DIRS) $(SRC_DIRS) $(ASSET_SRC_DIRS),$(shell mkdir -p build/$(dir))))
+
+# This file defines `ROM_FILES`, `UNCOMPRESSED_ROM_FILES`, and rules for generating `.yaz0` files
+ifneq ($(MAKECMDGOALS), clean)
+ifneq ($(MAKECMDGOALS), distclean)
+$(shell tools/dmadata_dependencies.py \
+	--dmadata-table=tables/dmadata_table.txt \
+	--output-deps=build/rom_dependencies.d)
+-include build/rom_dependencies.d
+endif
+endif
 
 build/src/libultra/os/%: OPTFLAGS := -O1
 build/src/libultra/voice/%: OPTFLAGS := -O2
@@ -123,10 +125,9 @@ build/src/libultra/voice/%: CC := ./tools/preprocess.py $(CC_OLD) -- $(AS) $(ASF
 CC := ./tools/preprocess.py $(CC) -- $(AS) $(ASFLAGS) --
 
 .PHONY: all clean setup diff-init init
-# disasm is not a file so we must tell make not to check it when evaluating timestamps
-.INTERMEDIATE: disasm
 # make will delete any generated assembly files that are not a prerequisite for anything, so keep it from doing so
 .PRECIOUS: asm/%.asm $(ASSET_FILES_OUT)
+.DEFAULT_GOAL := $(UNCOMPRESSED_ROM)
 
 $(UNCOMPRESSED_ROM): $(UNCOMPRESSED_ROM_FILES)
 	./tools/makerom.py ./tables/dmadata_table.txt $@
@@ -144,14 +145,14 @@ endif
 
 all: $(UNCOMPRESSED_ROM) $(ROM) ;
 
-build/code.elf: $(O_FILES) build/linker_scripts/code_script.txt undef.txt build/linker_scripts/object_script.txt build/linker_scripts/dmadata_script.txt
-	$(LD) -T build/linker_scripts/code_script.txt -T undef.txt -T build/linker_scripts/object_script.txt -T build/linker_scripts/dmadata_script.txt --no-check-sections --accept-unknown-input-arch -Map build/mm.map -N -o $@
+build/code.elf: $(O_FILES) build/linker_scripts/code_script.ld undef.txt build/linker_scripts/object_script.ld build/dmadata_script.ld
+	$(LD) -T build/linker_scripts/code_script.ld -T undef.txt -T build/linker_scripts/object_script.ld -T build/dmadata_script.ld --no-check-sections --accept-unknown-input-arch -Map build/mm.map -N -o $@
 
-build/code_pre_dmadata.elf: $(O_FILES) build/linker_scripts/code_script.txt undef.txt build/linker_scripts/object_script.txt
-	$(LD) -r -T build/linker_scripts/code_script.txt -T undef.txt -T build/linker_scripts/object_script.txt --no-check-sections --accept-unknown-input-arch -N -o $@
+build/code_pre_dmadata.elf: $(O_FILES) build/linker_scripts/code_script.ld undef.txt build/linker_scripts/object_script.ld
+	$(LD) -r -T build/linker_scripts/code_script.ld -T undef.txt -T build/linker_scripts/object_script.ld --no-check-sections --accept-unknown-input-arch -N -o $@
 
-linker_scripts/dmadata_script.txt: $(DMADATA_FILES) build/code_pre_dmadata.elf
-	./tools/dmadata.py ./tables/dmadata_table.txt /dev/null -u -l linker_scripts/dmadata_script.txt -e build/code_pre_dmadata.elf
+build/dmadata_script.txt: tables/dmadata_table.txt build/code_pre_dmadata.elf
+	./tools/dmadata.py ./tables/dmadata_table.txt /dev/null -u -l $@ -e build/code_pre_dmadata.elf
 
 build/dmadata: $(ROM_FILES:build/dmadata=)
 	./tools/dmadata.py ./tables/dmadata_table.txt $@
@@ -159,19 +160,30 @@ build/dmadata: $(ROM_FILES:build/dmadata=)
 build/uncompressed_dmadata: $(UNCOMPRESSED_ROM_FILES:build/uncompressed_dmadata=)
 	./tools/dmadata.py ./tables/dmadata_table.txt $@ -u
 
-build/baserom/boot build/baserom/code: build/code.elf
+build/binary/boot build/binary/code: build/code.elf
 	@$(OBJCOPY) --dump-section $(notdir $@)=$@ $< /dev/null
 
-build/baserom/assets/scenes/%: build/code.elf
+build/binary/assets/scenes/%: build/code.elf
 	@$(OBJCOPY) --dump-section $*=$@ $< /dev/null
 
-asm/non_matchings/%: asm/%.asm
-	@./tools/split_asm.py $< $@
+build/binary/overlays/%: build/code.elf
+	@$(OBJCOPY) --dump-section $*=$@ $< /dev/null
 
-asm/%.asm: disasm ;
+# Use an empty sentinel file (dep) to track the directory as a dependency, and
+# emulate GNU Make's order-only dependency.
+# The `touch $@; action || rm $@` pattern ensures that the `dep` file is older
+# than the output files from `action`, and only exists if `action` succeeds.
+asm/non_matchings/%/dep: asm/%.asm
+	@mkdir -p $(dir $@)
+	@touch $@
+	./tools/split_asm.py $< asm/non_matchings/$* || rm $@
 
-disasm: tables/files.txt tables/functions.txt tables/objects.txt tables/variables.txt tables/vrom_variables.txt
-	./tools/disasm.py -d ./asm -l ./tables/files.txt -f ./tables/functions.txt -o ./tables/objects.txt -v ./tables/variables.txt -v ./tables/vrom_variables.txt
+asm/%.asm: asm/disasm.dep ;
+
+asm/disasm.dep: tables/files.txt tables/functions.txt tables/objects.txt tables/variables.txt tables/vrom_variables.txt
+	@mkdir -p asm
+	@touch $@
+	./tools/disasm.py -d ./asm -l ./tables/files.txt -f ./tables/functions.txt -o ./tables/objects.txt -v ./tables/variables.txt -v ./tables/vrom_variables.txt || rm $@
 
 clean:
 	rm -rf $(ROM) $(UNCOMPRESSED_ROM) build asm
@@ -230,14 +242,29 @@ build/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
 	@./tools/set_o32abi_bit.py $@
 
-build/comp/%.yaz0: build/baserom/%
+build/comp/%.yaz0: build/binary/%
+	./tools/yaz0 $< $@
+
+build/comp/assets/audio/%.yaz0: build/baserom/assets/audio/%
+	./tools/yaz0 $< $@
+
+build/comp/assets/misc/%.yaz0: build/baserom/assets/misc/%
+	./tools/yaz0 $< $@
+
+build/comp/assets/objects/%.yaz0: build/baserom/assets/objects/%
+	./tools/yaz0 $< $@
+
+build/comp/assets/textures/%.yaz0: build/baserom/assets/textures/%
 	./tools/yaz0 $< $@
 
 build/%.d: %.c
 	@./tools/depend.py $< $@
 	@$(GCC) $< -Iinclude -I./ -MM -MT 'build/$*.o' >> $@
 
-build/linker_scripts/%: linker_scripts/%
+build/dmadata_script.ld: build/dmadata_script.txt
+	@$(GCC) -E -CC -x c -Iinclude $< | grep -v '^#' > $@
+
+build/linker_scripts/%.ld: linker_scripts/%.txt
 	@$(GCC) -E -CC -x c -Iinclude $< | grep -v '^#' > $@
 
 build/assets/%.d: assets/%.c

src/overlays/actors/ovl_Bg_Lbfshot/z_bg_lbfshot.h

@@ -7,6 +7,7 @@ struct BgLbfshot;
 
 typedef struct BgLbfshot {
     /* 0x000 */ DynaPolyActor dyna;
+    /* 0x15C */ UNK_TYPE1 pad15C[0x4];
 } BgLbfshot; // size = 0x160
 
 extern const ActorInit Bg_Lbfshot_InitVars;

src/overlays/actors/ovl_En_Encount2/z_en_encount2.c

@@ -67,7 +67,7 @@ void EnEncount2_Init(Actor* thisx, GlobalContext* globalCtx) {
         this->switchFlag = -1;
     }
 
-    if ((this->switchFlag >= 0) && (Actor_GetSwitchFlag(globalCtx, this->switchFlag))) {
+    if ((this->switchFlag >= 0) && (Flags_GetSwitch(globalCtx, this->switchFlag))) {
         Actor_MarkForDeath(&this->dynaActor.actor);
         return;
     }

src/overlays/actors/ovl_En_Encount2/z_en_encount2.h

@@ -8,6 +8,7 @@ typedef void (*EnEncount2ActionFunc)(struct EnEncount2*, GlobalContext*);
 typedef struct EnEncount2Particle{
    /* 0x00 */ u8 enabled;
    /* 0x04 */ Vec3f pos;
+   /* 0x10 */ UNK_TYPE4 unk10;
    /* 0x14 */ s16 alpha;
    /* 0x16 */ s16 alphaFadeDelay; // frame count before alpha fade starts
    /* 0x18 */ Vec3f vel;

src/overlays/actors/ovl_En_Ginko_Man/z_en_ginko_man.c

@@ -665,13 +665,13 @@ void EnGinkoMan_SwitchAnimation(EnGinkoMan *this, GlobalContext *globalCtx) {
         if (this->animTimer == 0) {
             if (this->skelAnime.animCurrentSeg != object_ginko_amazed_anim) {
                 this->animTimer = 0x28;
-                func_800BDC5C(&this->skelAnime, animations, GINKO_SITTING);
+                func_800BDC5C(&this->skelAnime, animations, GINKO_ADVERTISING);
             }
         }
 
     } else if ((this->animTimer == 0) && (this->skelAnime.animCurrentSeg != object_ginko_advertising_anim)) {
         this->animTimer = 0x28;
-        func_800BDC5C(&this->skelAnime, animations, GINKO_REACHING);
+        func_800BDC5C(&this->skelAnime, animations, GINKO_AMAZED);
     }
 
     DECR(this->animTimer);

src/overlays/actors/ovl_Obj_Kepn_Koya/z_obj_kepn_koya.h

@@ -7,6 +7,7 @@ struct ObjKepnKoya;
 
 typedef struct ObjKepnKoya {
     /* 0x000 */ DynaPolyActor dyna;
+    /* 0x15C */ UNK_TYPE1 unk15C[0x4];
 } ObjKepnKoya; // size = 0x160
 
 extern const ActorInit Obj_Kepn_Koya_InitVars;

src/overlays/actors/ovl_Obj_Lightswitch/z_obj_lightswitch.c

@@ -154,7 +154,7 @@ void ObjLightswitch_Init(Actor* thisx, GlobalContext* globalCtx) {
     u32 switchFlagResult;
     s32 isTriggered;
 
-    switchFlagResult = Actor_GetSwitchFlag(globalCtx, GET_LIGHTSWITCH_SWITCHFLAG(this));
+    switchFlagResult = Flags_GetSwitch(globalCtx, GET_LIGHTSWITCH_SWITCHFLAG(this));
     isTriggered = 0;
     Actor_ProcessInitChain(&this->actor, &sInitChain);
     Actor_SetHeight(&this->actor, 0.0f);
@@ -268,7 +268,7 @@ void ObjLightSwitch_Enabled(ObjLightswitch* this, GlobalContext* globalCtx) {
 
     if (actorType == LIGHTSWITCH_TYPE_REGULAR) {
         // switch can be disabled outside of this actor by flag
-        if (Actor_GetSwitchFlag(globalCtx, GET_LIGHTSWITCH_SWITCHFLAG(this)) == 0) {
+        if (Flags_GetSwitch(globalCtx, GET_LIGHTSWITCH_SWITCHFLAG(this)) == 0) {
             ObjLightSwitch_SetupDisabled(this);
         }
     } else if (actorType == LIGHTSWITCH_TYPE_FLIP) {

src/overlays/actors/ovl_Obj_Visiblock/z_obj_visiblock.h

@@ -7,6 +7,7 @@ struct ObjVisiblock;
 
 typedef struct ObjVisiblock {
     /* 0x000 */ DynaPolyActor dyna;
+    /* 0x15C */ UNK_TYPE1 unk15C[0x4];
 } ObjVisiblock; // size = 0x160
 
 extern const ActorInit Obj_Visiblock_InitVars;

tools/depend.py

@@ -18,7 +18,7 @@ if __name__ == '__main__':
                 base_path = os.path.normpath(line.split('"')[1])
                 path = os.path.split(base_path)[0]
                 asm_file = path.replace('non_matchings/', '') + '.asm ' # base .asm file
-                asm_file += path # split function .asm file
+                asm_file += path + "/dep" # split function .asm directory (sentinel file)
                 break
 
     with open(args.output, 'w') as f:

tools/dmadata_dependencies.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Generate a .d file with Makefile variables from the dmadata_table.txt file
+
+import os, struct, sys, ast, argparse
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--yaz0-path", required=False, help="path to `yaz0` tool")
+    parser.add_argument(
+        "--dmadata-table",
+        type=argparse.FileType("r"),
+        help="dmadata_table.txt input file",
+    )
+    parser.add_argument(
+        "--output-deps", type=argparse.FileType("w"), help="romfiles.d output file"
+    )
+    args = parser.parse_args()
+
+    dmadata_table = ast.literal_eval(args.dmadata_table.read())
+    rom_files = []
+    for base_file, comp_file, _, _ in dmadata_table:
+        if base_file == "":
+            assert comp_file == ""
+            continue
+
+        # TODO: Unsure if it's better to emit these rules here, or use a set of
+        # prefix rules in the Makefile.
+        if args.yaz0_path is not None and comp_file.endswith(".yaz0"):
+            # Add a rule for generating the compressed `.yaz0` file
+            # from the uncompressed file
+            args.output_deps.write(f"{comp_file}: {base_file}\n")
+            args.output_deps.write(f"\t{args.yaz0_path} $< $@\n")
+            args.output_deps.write("\n")
+
+        if comp_file == "":
+            comp_file = base_file
+
+        rom_files.append((base_file, comp_file))
+    args.dmadata_table.close()
+
+    # Define lists of files for using in other rules
+    uncompressed_files, compressed_files = zip(*rom_files)
+    for var_name, filenames in (
+        ("UNCOMPRESSED_ROM_FILES", uncompressed_files),
+        ("ROM_FILES", compressed_files),
+    ):
+        args.output_deps.write(f"{var_name} := \\\n")
+        for filename in filenames:
+            args.output_deps.write(f"    {filename} \\\n")
+        args.output_deps.write("\n\n")
+    args.output_deps.close()
+
+
+if __name__ == "__main__":
+    main()