Build ROM with asset files

README.md

@@ -68,10 +68,10 @@ Before you do anything you need an existing ROM to extract assets from.
 The project can do the following:
 
 * Build individual ucode binaries (boot, library, setup, rarezip and game) which match the ones extracted from the base ROM.
-* Build a functioning ROM by splicing the C source into an existing ROM. Files in the "files" folder (eg. stage setup and lang) are not included yet. Additionally, the built ROM is not byte perfect yet, but is is functionally equivalent.
+* Build a functioning ROM by splicing the C source and assets into an existing ROM. The built ROM is not byte perfect yet, but is is functionally equivalent.
 
 * Run `make` to build the assets that will be included in the ROM. These files will be written to `build/ntsc-final` and are matching what's in the `extracted/ntsc-final` folder.
-* Run `make rom` to build a ROM from the C source. The ROM will be written to `build/ntsc-final/pd.z64`.
+* Run `make rom` to build the ROM. The ROM will be written to `build/ntsc-final/pd.z64`.
 
 ## How do I know the built files are matching?
 

src/include/setup/setup_020df0.h

@@ -203,7 +203,7 @@ extern u32 var80081018;
 extern u32 var80081058;
 extern struct smallsky smallskies[];
 extern u32 var80082050;
-extern u32 filetable[];
+extern void *filetable[];
 extern u32 var80084008;
 extern s32 var80084034;
 extern u32 var80084038;

tools/buildrom

@@ -19,7 +19,7 @@ def main():
     write_binary(fd, 0xcfbf30, get_seqctl())
     write_binary(fd, 0xd05f90, get_seqtbl())
     write_binary(fd, 0xe82000, get_midi())
-    write_binary(fd, 0xed83a0, get_files())
+    write_files(fd)
     write_binary(fd, 0x1d65f40, get_textures())
     fd.close()
 
@@ -83,8 +83,11 @@ def get_seqtbl():
 def get_midi():
     return getfilecontents('extracted/ntsc-final/audio/midi.bin')
 
-def get_files():
-    return getfrombaserom(0xed83a0, 0xe8d3a0)
+def write_files(fd):
+    start = getlinkervariable('_filesSegmentRomStart')
+    end = getlinkervariable('_filesSegmentRomEnd')
+
+    write_binary(fd, 0xed83a0, getfromstage1rom(start, end - start))
 
 def get_textures():
     return getfrombaserom(0x01d65f40, 0x29a0c0)
@@ -102,7 +105,20 @@ def getfrombaserom(offset, len):
     fd.close()
     return binary
 
+def getfromstage1rom(offset, len):
+    fd = open('build/ntsc-final/stage1.bin', 'rb')
+    fd.seek(offset)
+    binary = fd.read(len)
+    fd.close()
+    return binary
+
 def zip(filename):
     return subprocess.check_output(['tools/rarezip', filename])
 
+def getlinkervariable(varname):
+    objdump = subprocess.check_output(['mips64-elf-objdump', 'build/ntsc-final/stage1.elf', '-t']).decode('utf-8')
+
+    matches = re.findall(r'^([0-9a-f]+) .*? %s$' % varname, objdump, re.MULTILINE)
+    return int(matches[0], 16)
+
 main()

tools/extract

@@ -87,27 +87,36 @@ class Extractor:
                 return
             content = self.rom[offset:endoffset]
             name = names[index]
+
+            # If the content is zipped then the last byte might be padding. So
+            # unzip it to see.
+            unzipped = None
+            if content[0:2] == b'\x11\x73':
+                (unzipped, unused) = self.decompressandgetunused(content)
+                if len(unused):
+                    content = content[:-len(unused)]
+
             self.write('files/%s' % name, content)
 
             unzippedname = name[:-1] if name.endswith('Z') else name
 
             if name.endswith('tilesZ'):
-                self.write('files/bgdata/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/bgdata/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('C'):
-                self.write('files/chrs/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/chrs/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('G'):
-                self.write('files/guns/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/guns/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('L'):
-                self.write('files/lang/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/lang/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('P') and len(content):
-                self.write('files/props/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/props/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('U'):
-                self.write('files/setup/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/setup/%s.bin' % unzippedname, unzipped)
 
     def get_file_offsets(self):
         i = self.val('files')
@@ -193,6 +202,13 @@ class Extractor:
         assert(header == 0x1173)
         return zlib.decompress(buffer[5:], wbits=-15)
 
+    def decompressandgetunused(self, buffer):
+        header = int.from_bytes(buffer[0:2], 'big')
+        assert(header == 0x1173)
+        obj = zlib.decompressobj(wbits=-15)
+        bin = obj.decompress(buffer[5:])
+        return (bin, obj.unused_data)
+
     def write(self, filename, data):
         filename = 'extracted/%s/%s' % (self.romid, filename)