diff --git a/.gitignore b/.gitignore index b5d2c581f..d38785294 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ src/assets/*/fonts/*.bin src/assets/*/sequences/*.seq src/assets/*/textures/*.bin src/generated +tools/mkrom/mkrom diff --git a/Makefile b/Makefile index 7df615e14..9d516847c 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ export ROMID NTSC=0 PAL=0 JPN=0 +ZIPMAGIC=0x0000 ifeq ($(ROMID),ntsc-beta) NTSC=1 @@ -37,22 +38,27 @@ endif ifeq ($(ROMID),ntsc-1.0) NTSC=1 VERSION=1 + ZIPMAGIC=0xffff endif ifeq ($(ROMID),ntsc-final) NTSC=1 VERSION=2 + ZIPMAGIC=0xffff endif ifeq ($(ROMID),pal-beta) PAL=1 VERSION=3 + ZIPMAGIC=0x0c00 endif ifeq ($(ROMID),pal-final) PAL=1 VERSION=4 + ZIPMAGIC=0xaf00 endif ifeq ($(ROMID),jpn-final) JPN=1 VERSION=5 + ZIPMAGIC=0x0002 endif DEFINES := VERSION=$(VERSION) NTSC=$(NTSC) PAL=$(PAL) JPN=$(JPN) PIRACYCHECKS=$(PIRACYCHECKS) _FINALROM=1 @@ -233,7 +239,7 @@ ASSET_FILES := \ $(patsubst $(A_DIR)/files/guns/%.bin, $(B_DIR)/assets/files/G%Z, $(shell find $(A_DIR)/files/guns -name '*.bin')) \ $(patsubst $(A_DIR)/files/props/%.bin, $(B_DIR)/assets/files/P%Z, $(shell find $(A_DIR)/files/props -name '*.bin')) \ $(patsubst src/files/setup/%.c, $(B_DIR)/assets/files/U%Z, $(shell find src/files/setup -name '*.c')) \ - $(patsubst $(A_DIR)/files/setup/%.bin, $(B_DIR)/assets/files/U%Z, $(shell find $(A_DIR)/files/setup -name '*.bin')) \ + $(patsubst $(A_DIR)/files/setup/%.bin, $(B_DIR)/assets/files/U%Z, $(shell find $(A_DIR)/files -path '*/setup/*.bin')) \ $(patsubst $(A_DIR)/files/bgdata/%.seg, $(B_DIR)/assets/files/bgdata/%.seg, $(shell find $(A_DIR)/files/bgdata -name '*.seg')) \ $(patsubst src/files/bgdata/%_tiles.s, $(B_DIR)/assets/files/bgdata/%_tilesZ, $(shell find src/files/bgdata -name 'bg_*_tiles.s')) \ $(patsubst $(A_DIR)/files/bgdata/%_tiles.bin, $(B_DIR)/assets/files/bgdata/%_tilesZ, $(shell find $(A_DIR)/files/bgdata -name 'bg_*_tiles.bin')) \ @@ -266,6 +272,7 @@ O_FILES := \ $(B_DIR)/assets/fonts/ocramd.o \ $(B_DIR)/assets/fonts/tahoma.o \ $(B_DIR)/assets/fonts/zurich.o \ + $(B_DIR)/garbage.o \ $(B_DIR)/getitle.o \ $(B_DIR)/mpconfigs.o \ $(B_DIR)/assets/mpstrings/mpstringsE.o \ @@ -308,24 +315,14 @@ $(B_DIR)/stage1.elf: $(O_FILES) ld/pd.ld $(B_DIR)/stage1.bin: $(B_DIR)/stage1.elf $(TOOLCHAIN)-objcopy $< $@ -O binary -# Stage2 takes stage1 and patches the piracy checksums. -$(B_DIR)/stage2.bin: $(B_DIR)/stage1.bin - @cp $< $@.tmp - ROMID=$(ROMID) PIRACYCHECKS=$(PIRACYCHECKS) tools/patchpiracysums $@.tmp $(B_DIR)/pd.map && mv $@.tmp $@ +# Build the final ROM from stage1.bin using mkrom +# mkrom handles calculating the piracy checksums, zipping segments and +# calculating the ROM checksum. +$(B_DIR)/pd.z64: $(B_DIR)/stage1.bin tools/mkrom/mkrom + tools/mkrom/mkrom $(B_DIR)/stage1.bin $(B_DIR)/pd.map $(PIRACYCHECKS) $(ZIPMAGIC) $@ -# Stage3 takes stage2, compresses the game/lib/data segments, -# inserts them and truncates the ROM to 32MB. -$(B_DIR)/stage3.bin: $(B_DIR)/stage2.bin $(B_DIR)/segments/gamezips.bin - @cp $< $@.tmp - tools/packrom $@.tmp && mv $@.tmp $@ - -$(B_DIR)/segments/gamezips.bin: $(B_DIR)/segments/game.bin - ROMID=$(ROMID) tools/mkgamezips - -# The final ROM image takes stage3 and calculates the ROM CRC. -$(B_DIR)/pd.z64: $(B_DIR)/stage3.bin - @cp $< $@.tmp - tools/patchromcrc $@.tmp --write && mv $@.tmp $@ +tools/mkrom/mkrom: + $(MAKE) -C tools/mkrom ################################################################################ # Testing Related @@ -338,7 +335,7 @@ CHECK_FILES := $(shell awk '{print $$2}' checksums.$(ROMID).md5) test: $(CHECK_FILES) @md5sum --quiet -c checksums.$(ROMID).md5 -$(B_DIR)/segments/%.bin: $(B_DIR)/stage2.bin +$(B_DIR)/segments/%.bin: $(B_DIR)/pd.z64 @B_DIR=$(B_DIR) tools/extract-segment $* ################################################################################ @@ -348,6 +345,9 @@ $(B_DIR)/assets/fonts/%.o: $(A_DIR)/fonts/%.bin mkdir -p $(B_DIR)/assets/fonts TOOLCHAIN=$(TOOLCHAIN) ROMID=$(ROMID) tools/mkrawobject $< $@ +$(B_DIR)/garbage.o: $(E_DIR)/garbage.bin + TOOLCHAIN=$(TOOLCHAIN) ROMID=$(ROMID) tools/mkrawobject $< $@ + $(B_DIR)/getitle.o: $(E_DIR)/getitle.bin TOOLCHAIN=$(TOOLCHAIN) ROMID=$(ROMID) tools/mkrawobject $< $@ diff --git a/ld/pd.ld b/ld/pd.ld index 01a2d180b..08b454aa5 100644 --- a/ld/pd.ld +++ b/ld/pd.ld @@ -50,15 +50,13 @@ END_SEG(font##name) /** - * Placeholder segments are used to mark the locations where zipped content will - * go. It's really just here so it appears in the linker map which allows - * packrom to find it. We only care about the start address for this segment, - * so the romheader object is used as it's nice and short. + * Placeholder segments are used to mark the + * locations where zipped content will go. */ -#define PLACEHOLDER_SEGMENT(name) \ +#define PLACEHOLDER_SEGMENT(name, len) \ BEGIN_SEG(name) \ { \ - build/ROMID/romheader.o (.data); \ + . = . + len; \ } \ END_SEG(name) @@ -67,7 +65,7 @@ * ---------------------------------------------------------------------------- * The lib, data and game segments are compressed in the final ROM. To do this, * we build them uncompressed here but place them past the end of the ROM, then - * a later script compresses them and writes them into the ROM. + * mkrom compresses them and writes them into the ROM. * * These constants are defining how much space is reserved for the compressed * segments. If these segments are edited and grow to a point that their @@ -78,11 +76,15 @@ #if VERSION >= VERSION_PAL_FINAL #define ROMALLOCATION_LIB 0x038800 #define ROMALLOCATION_DATA 0x015000 -#define ROMALLOCATION_GAME 0x13d180 -#else +#define ROMALLOCATION_GAME 0x1306f0 +#elif VERSION >= VERSION_NTSC_1_0 #define ROMALLOCATION_LIB 0x038800 #define ROMALLOCATION_DATA 0x015000 -#define ROMALLOCATION_GAME 0x151980 +#define ROMALLOCATION_GAME 0x144ee0 +#else +#define ROMALLOCATION_LIB 0x02f800 +#define ROMALLOCATION_DATA 0x012000 +#define ROMALLOCATION_GAME 0x112080 #endif OUTPUT_ARCH (mips) @@ -203,7 +205,7 @@ SECTIONS __rompos = 0x00001050; - PLACEHOLDER_SEGMENT(libzip) + PLACEHOLDER_SEGMENT(libzip, ROMALLOCATION_LIB) __rompos = 0x02000000; __rampos = 0x70001050; @@ -227,7 +229,7 @@ SECTIONS __savedrompos = __rompos; __rompos = _libzipSegmentRomStart + ROMALLOCATION_LIB; - PLACEHOLDER_SEGMENT(datazip) + PLACEHOLDER_SEGMENT(datazip, ROMALLOCATION_DATA) __rompos = __savedrompos; __rampos = 0x80001050 + SIZEOF(.lib); @@ -244,8 +246,6 @@ SECTIONS } END_SEG(data) - _datazipSegmentRomEnd = _datazipSegmentRomStart + ROMALLOCATION_DATA; - rspbootTextStart = _dataSegmentStart; rspbootTextEnd = rspbootTextStart + 0xd0; gspTextStart = rspbootTextEnd; @@ -300,7 +300,8 @@ SECTIONS * ------------------------------------------------------------------------- */ - PLACEHOLDER_SEGMENT(gamezip) + PLACEHOLDER_SEGMENT(gamezip, ROMALLOCATION_GAME) + __rompos = __savedrompos; __rampos = 0x7f000000; @@ -311,6 +312,23 @@ SECTIONS } END_SEG(game) + __rompos = _inflateSegmentRomEnd + ROMALLOCATION_GAME; + + /*************************************************************************** + * garbage + * ------------------------------------------------------------------------- + * ROM range: 0x00194b30 - 0x001a15c0 + * RAM range: N/A + * ------------------------------------------------------------------------- + * On NTSC, this contains unused JPN fonts. On PAL, not sure what this is. + */ + + BEGIN_SEG(garbage) + { + build/ROMID/garbage.o (.data); + } + END_SEG(garbage) + /*************************************************************************** * animations * ------------------------------------------------------------------------- @@ -319,8 +337,6 @@ SECTIONS * ------------------------------------------------------------------------- */ - __rompos = _inflateSegmentRomEnd + ROMALLOCATION_GAME; - BEGIN_SEG(animations) { build/ROMID/assets/animations.o (.data); diff --git a/tools/extract b/tools/extract index fddc1f339..8ff872148 100755 --- a/tools/extract +++ b/tools/extract @@ -26,8 +26,7 @@ class Extractor: self.extract_firingrange() self.extract_fonts() self.extract_game() - self.extract_garbage1() - self.extract_garbage2() + self.extract_garbage() self.extract_getitle() self.extract_lib() self.extract_mpconfigs() @@ -188,15 +187,10 @@ class Extractor: def extract_inflate(self): self.write_extracted('inflate.bin', self.rom[0x4e850:0x4fc40]) - def extract_garbage1(self): - start = self.val('garbage1') - end = self.val('data') - self.write_extracted('garbage1.bin', self.rom[start:end]) - - def extract_garbage2(self): - start = self.val('garbage2') + def extract_garbage(self): + start = self.val('garbage') end = self.val('animations') - self.write_extracted('garbage2.bin', self.rom[start:end]) + self.write_extracted('garbage.bin', self.rom[start:end]) # In all versions, lib starts at 0x1050 and is compressed from 0x3050 onwards def extract_lib(self): @@ -461,11 +455,10 @@ class Extractor: vals = { # ntsc-beta ntsc-1.0 ntsc-final pal-beta pal-final jpn-final - 'game': [0x43c40, 0x4fc40, 0x4fc40, 0x4fc40, 0x4fc40, 0x4fc40, ], - 'garbage1': [0x0, 0x2ea22, 0x2ea6c, 0x0, 0x2eb21, 0x0, ], 'files': [0x29160, 0x28080, 0x28080, 0x29b90, 0x28910, 0x28800, ], 'data': [0x30850, 0x39850, 0x39850, 0x39850, 0x39850, 0x39850, ], - 'garbage2': [0x0, 0x1574a0, 0x157800, 0x0, 0x158038, 0x0, ], + 'game': [0x43c40, 0x4fc40, 0x4fc40, 0x4fc40, 0x4fc40, 0x4fc40, ], + 'garbage': [0x155cc0, 0x194b20, 0x194b20, 0x0, 0x180330, 0x0, ], 'animations': [0x155dc0, 0x1a15c0, 0x1a15c0, 0x18cdc0, 0x18cdc0, 0x190c50, ], 'mpconfigs': [0x785130, 0x7d0a40, 0x7d0a40, 0x7bc240, 0x7bc240, 0x7c00d0, ], 'firingrange': [0x79e410, 0x7e9d20, 0x7e9d20, 0x7d5520, 0x7d5520, 0x7d93b0, ], diff --git a/tools/extract-segment b/tools/extract-segment index e85432735..8a2d91a9d 100755 --- a/tools/extract-segment +++ b/tools/extract-segment @@ -1,12 +1,82 @@ -#!/bin/bash +#!/usr/bin/env python3 -# Extracts a segment from the binary that was produced by ld. +# Extracts a segment from the binary that was produced by ld +# and unzips it if necessary. Used for `make test`. -segment=$1 +import os +import re +import sys +import zlib -pos=$(grep "^\.$segment " $B_DIR/pd.map | awk '{print $6}') -len=$(grep "^\.$segment " $B_DIR/pd.map | awk '{print $3}') +def bdir(): + return 'build/%s' % os.environ['ROMID'] -mkdir -p $B_DIR/segments -dd if=$B_DIR/stage2.bin of=$B_DIR/segments/$segment.bin skip=$(($pos)) iflag=skip_bytes bs=$(($len)) count=1 status=none +def find_segment(segname): + fd = open(bdir() + '/pd.map', 'r') + ldmap = fd.read() + fd.close() + start = re.findall(r'0x([0-9a-f]+)\s+_' + segname + 'SegmentRomStart = ', ldmap)[0] + end = re.findall(r'0x([0-9a-f]+)\s+_' + segname + 'SegmentRomEnd = ', ldmap)[0] + + start = int(start, 16) + end = int(end, 16) + + return (start, end) + +def inflate(buffer): + header = int.from_bytes(buffer[0:2], 'big') + assert(header == 0x1173) + return zlib.decompress(buffer[5:], wbits=-15) + +def inflate_game(buffer): + binary = bytes() + i = 0 + + while True: + offset = int.from_bytes(buffer[i:i+4], 'big') + 2 + peek = int.from_bytes(buffer[offset:offset+2], 'big') + if peek == 0: + break + part = inflate(buffer[offset:offset+0x1000]) + binary += part + if len(part) != 0x1000: + break + i += 4 + + return binary + +def inflate_lib(buffer): + return buffer[0:0x2000] + inflate(buffer[0x2000:]) + +def inflate_data(buffer): + return inflate(buffer) + +def main(): + segname = sys.argv[1] + loadname = sys.argv[1] + + if segname in ['lib', 'game', 'data']: + loadname += 'zip' + + (start, end) = find_segment(loadname) + + fd = open(bdir() + '/pd.z64', 'rb') + fd.seek(start) + buffer = fd.read(end - start) + fd.close() + + if segname == 'lib': + buffer = inflate_lib(buffer) + elif segname == 'game': + buffer = inflate_game(buffer) + elif segname == 'data': + buffer = inflate_data(buffer) + + os.makedirs(bdir() + '/segments', exist_ok=True) + + fd = open(bdir() + '/segments/' + segname + '.bin', 'wb') + fd.write(buffer) + fd.close() + +main() diff --git a/tools/mkgamezips b/tools/mkgamezips deleted file mode 100755 index 9d16a50f0..000000000 --- a/tools/mkgamezips +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 - -import os -import zlib - -""" -mkgamezips - Creates the segments/gamezips.bin from segments/game.bin - -game.bin is the compiled game code from ld. This game code is split into -4KB chunks. Each chunk is individually zipped. - -The format of the gamezips binary is: -* Array of offsets to each zip, where each offset is 4 bytes and relative to the - start of the gamezips segment. -* After the array of offsets comes the data it points to. Each data consists of: - * A 2 byte checksum of the uncompressed chunk. - * Zip data (starting with 0x1173001000). - * Optional single byte to align it to the next 2 byte boundary. The added - byte is probable garbage data. -""" - -def main(): - bins = get_bins() - sums = get_sums(bins) - zips = get_zips(bins) - - fd = open('build/%s/segments/gamezips.bin' % os.environ['ROMID'], 'wb+') - pos = len(zips) * 4 + 4 - - # Write pointer array - for zip in zips: - fd.write(pos.to_bytes(4, byteorder='big')) - pos += 2 + len(zip) - if pos % 2 == 1: - pos += 1 - - # Last pointer points to end - fd.write(pos.to_bytes(4, byteorder='big')) - - # Write checksums and compressed data - padding_index = 0 - for index, zip in enumerate(zips): - if pos % 2 == 1: - try: - pad_value = padding[os.environ['ROMID']][padding_index] - except IndexError: - pad_value = 0 - fd.write(pad_value.to_bytes(1, 'big')) - - padding_index += 1 - pos += 1 - - fd.write(sums[index].to_bytes(2, 'big')) - fd.write(zip) - - pos += len(zip) - - # The previous two bytes are repeated at the end - fd.seek(-2, os.SEEK_CUR) - value = fd.read(2) - fd.write(value) - fd.close() - -def get_filecontents(filename): - fd = open(filename, 'rb') - binary = fd.read() - fd.close() - return binary - -def get_bins(): - binary = get_filecontents('build/%s/segments/game.bin' % os.environ['ROMID']) - return [binary[i:i+0x1000] for i in range(0, len(binary), 0x1000)] - -def get_sums(bins): - return [sum(part) for part in bins] - -def get_zips(bins): - return [zip(part, index) for part, index in enumerate(bins)] - -def sum(binary): - sum = 0 - for i in range(0, len(binary), 4): - sum += int.from_bytes(binary[i:i+4], 'big') - return sum & 0xffff - -def zip(index, binary): - obj = zlib.compressobj(level=9, wbits=-15) - stream = obj.compress(binary) + obj.flush() - - for override in overrides[os.environ['ROMID']]: - if override[0] == index and zlib.crc32(stream) == override[1]: - stream = stream[0:override[2]] - stream += override[3] - break - - return b'\x11\x73' + len(binary).to_bytes(3, 'big') + stream - -""" -Rare's zip compression method is different to any known version of zlib. -In a few cases (49 out of 2651) their algorithm produces differing files, -where the final match directive chooses a seemingly random backtrack distance. -To get matching zips, decomp patches the zips with this overrides list. -Values are chunk index, crc32 checksum, write offset, write bytes -""" -overrides = { - 'ntsc-beta': [], - 'ntsc-1.0': [], - 'ntsc-final': [], - 'pal-final': [ - (0x002e, 0x938a5928, 0x091a, b'\xc2\x9f\x7f\x03'), - (0x00e1, 0xae07cb57, 0x0b1b, b'\xe4\xd5\xff\x00'), - (0x00ea, 0xf80d6c37, 0x09ca, b'\xa9\xcb\xff\x02'), - (0x00f6, 0xdbd3f03a, 0x0002, b'\x6d\x6c\x1c\xd5\x15\x3d\x6f\x67\x6d\x6f\xec\xb5\x77\xe3\x6c\x94\x35\x8e\xda\x1d\x3c\x63\x2f\x09\x10\xdb\x81\x2a\x48\x9b\xb2\x80\xb1\x2a\xe2\x80\x0d\xa6\xb8\x52\x2b\x39\x35\x50\x23\x42\x30\xf0\xa3\x8e\x1a\x29\x43\x30\xb1\xd5\xec\x66\x36\x25\xb4\x06\x21\xb4\x24\x76\x92\xca\x6b\x8f\x93\x26\xad\x2d\x59\xaa\x91\x9d\x60\x68\x48\xd3\x12\x2a\x47\xca\x0f\x23\x8c\x40\xc2\x69\xa8\x14\xb5\xa9\x84\xb2\x3d\x77\x76\xed\xb8\x81\xf6\xc7\xe8\xcd\xbc\x77\xdf\xbd\xe7\xdd\x8f\x73\xdf\x38\x69\xac\x1d\xf1\x00\x88\x47\x4c\xcf\x64\xd4\xf4\xb4\xd7\x72\xdc\x64\x3a\x27\x11\x74\x4e\x20\xe4\x1c\x47\x38\x30\xd5\x3a\xe4\xdc\x2c\x67\x51\x2e\x9d\x97\x3b\x4e\xb9\x13\xdf\x22\x97\x5e\x26\xf7\xbb\x6f\xea\x33\xc2\x50\x22\xeb\x8c\x61\xed\x92\xfc\xa2\xfd\xbc\xbc\x2b\xcb\x3d\x86\x3f\x2f\x9b\xc9\xcb\xde\x8c\xf5\x78\x5e\x96\x38\x8c\x8a\xbc\xec\xf8\x32\xd9\xe5\x78\x97\xeb\x0d\xe4\x65\x9d\x65\x18\x96\x63\x5e\xa6\xd7\x1e\x51\x0b\x22\x9b\xfc\x90\xcf\x00\x82\xb5\xfb\x14\x54\x7b\xc4\x0c\x59\xf8\x34\xa6\x59\xc5\x89\x8e\xef\x3e\x69\x28\x68\xc1\x6d\xf0\x70\x2c\x09\x6d\x83\x3f\x16\xb6\x4a\xec\xf0\x5f\xee\xda\xe8\x87\xa7\xc5\xc2\x0a\xfb\xb0\x9a\xb3\x2f\x7a\x53\x5c\xf7\xd4\x0c\xa8\x89\x55\x3a\xb4\x9a\x41\x35\x16\x98\xfa\xf8\xaf\x35\x47\xd4\x49\x59\xef\x75\xd4\x07\xf6\x11\x75\xc9\x1e\x56\x13\xf6\x90\x1a\xa3\xdd\x93\xea\xd3\xa0\x6e\xb4\x50\x6f\x57\xf6\xba\xf1\x31\x4a\x9c\x53\x08\x7a\x40\xac\x11\xe2\x3c\x84\x3e\x7b\x40\x7d\xe1\x0c\xd3\xbf\x43\xe2\xdf\xbe\x7f\x38\x23\x58\x6b\x67\xb8\x9f\x7a\x8c\x52\xa8\xba\x37\x90\x45\x45\xf7\xfd\x46\x09\xa2\xc6\x2c\x14\x7d\x1e\xa4\xec\x5a\x9e\x3b\xc4\x7d\x61\xf7\xec\x79\x7d\x81\xa9\x44\x3b\xe3\x13\xa1\xdd\xdc\xfe\x0a\x44\xeb\x66\xb9\x3f\xdc\x7e\xbf\xf8\x8b\x7b\x22\x8c\x43\x68\xc9\xbf\x8b\x38\x88\x89\xfb\xc2\xb2\xdf\x38\x83\x62\xfb\x04\xcf\x3a\xaa\x2e\x89\x4c\xf5\xa3\xf0\x54\x07\xe1\x71\x46\xd5\xd9\xc0\xe9\xe6\x3f\x3b\x27\xd4\x79\x9e\xef\xaa\x73\x88\xb8\xd3\x6a\x74\x65\x1a\x5a\xac\xcc\x2a\x22\x3e\xe5\x0c\xa9\x51\xf3\xdc\x9a\x63\xc9\x77\x01\xfa\x63\xa1\xe6\xa4\x6a\x1d\x79\x99\x3e\xbf\x08\xef\xc8\x1b\xf0\x72\xde\xc7\x6f\x1f\xbf\xfd\xb1\xb2\xdd\x95\xe6\xb9\xfa\x34\xe7\xfd\xd3\x9d\x88\x37\x86\xe2\x31\x66\x1f\x1e\xf4\xa0\x24\xb5\x1d\x9b\x53\xcf\x23\x16\x2b\xd9\x5d\x89\xc0\x33\x54\xdc\x59\x85\xd2\x4e\x4b\x3d\xd7\xa6\x27\x9f\xad\xcf\x04\x81\x95\xb4\x35\xb1\x28\x97\x78\x1a\x0d\xf0\xd3\x28\x65\x35\xff\x64\xd5\xa2\x3c\xc2\x93\x96\x56\x36\xa3\xab\x11\xee\x7b\x0a\xfe\xc0\xe4\x2d\x31\xdb\x80\x9f\xd8\x27\x04\x5f\xea\xa7\xb4\xe1\xdd\x5d\x19\x2b\xa0\x1d\xcf\x36\xa8\x3d\xed\x55\xf0\x07\x2d\xec\x89\xe8\x38\x14\xd5\x13\x83\x77\x85\x02\xa7\x1f\xfd\x28\xd1\x73\xf7\x2e\x37\xfe\x83\xf4\x69\x07\x34\x63\x10\x9a\x33\xa8\xce\xfa\x71\xe7\x13\xce\x61\xfa\xc3\x51\x13\xb1\x95\x96\x2f\xb6\xca\xf2\x55\xbf\x84\xed\xd5\x75\xe8\xda\xbf\x09\xd8\xbf\x13\xe8\x1d\x51\x1f\xd4\x5d\x46\x56\x62\x25\xb1\x66\xce\x9c\x67\xbe\x9c\xf5\xcc\xb9\xf1\x8f\xd0\xef\x06\x9f\xa8\x1b\xd7\x71\xe6\xec\xe9\x1f\x37\x4b\x0e\xb8\xfe\xa5\xaf\x5d\x1b\xf4\xfd\xfe\xf5\xd4\xb7\x9d\x7e\x1d\x53\x13\xbd\xe3\xcc\xb3\xff\xa7\x6b\x88\xba\x24\xce\x63\x8c\xa9\xe8\x63\xbc\x97\xe7\xb9\xe8\x66\xae\x17\xf1\xdd\xcb\xb1\x80\x63\x01\x47\x2d\x9f\xfb\x95\x12\x07\xfb\x30\xfa\xc4\xae\xd1\x91\xfd\xd7\xe2\x39\x63\x01\xab\x28\x79\x65\x4d\x74\x11\x0b\x02\x9d\x96\x27\xd9\xa5\x27\xdf\xbb\x81\xcb\xb5\x2f\x38\x88\x67\x39\xc6\x6f\xc3\x24\x38\x02\xa7\x5b\x4c\xb7\x06\xf8\x6e\x1f\x52\x0b\xf6\x29\x75\x89\x76\x8a\xa7\xe3\xf8\x41\xf5\x29\x94\x34\x86\x90\xcb\x0b\x0d\x95\xa9\x2d\x68\x4f\x5e\x71\xcf\xe0\x6d\xf9\x04\xdf\xb1\x33\xe8\x49\x6d\x45\x3b\xb0\xc9\xbc\xed\x1d\x68\xad\x11\xac\x68\x28\x04\xe4\x49\x34\x20\x94\x78\x50\x6c\x3d\xba\xd5\x28\x82\x62\xce\x14\x1a\x65\x50\x8b\x6b\x32\x2f\xeb\x32\x17\x53\x96\x2f\x73\x27\xbe\xa6\x8c\x4f\x30\x88\x1d\xf1\xd3\xe3\xf4\x8d\xd8\xa0\xec\xd1\xe9\x26\x34\xbb\xf8\xf8\x2d\x79\x69\x3e\x5b\x9f\x56\xaf\xfb\xd6\x35\x47\xa0\x4d\x3f\x84\xb8\x73\x4a\x5d\x90\xb1\x31\xd4\x7c\x23\x8f\xb7\x32\x3f\xdb\x98\x63\x01\xe6\x57\xf8\x7d\x68\x9b\x66\xaa\x50\x31\x63\xa9\x7f\x77\xeb\xc9\x2b\xf5\xa3\x6e\x1e\x33\x77\x16\xe5\x12\x8d\xcc\xe3\xd2\x17\x20\xb2\xea\xc3\xae\xaa\x45\x79\x94\x75\x59\xea\x4f\xed\xba\xb6\x9e\xfb\x3e\x43\x50\xf2\x38\xd9\x43\x7f\x4a\x1e\x13\x53\x62\x2b\xfc\xf4\xe1\x04\x58\xcf\x2d\x69\x94\x4c\x37\x23\x2e\x32\x46\x21\x6e\x17\x2e\x31\x54\x16\xb8\xaf\xcb\xa8\xeb\x47\x56\xcd\xce\x98\xce\x78\x6e\x9f\xc8\xc5\xc2\xbb\x2b\x6b\xc2\xf5\xe9\xc6\x90\x75\x03\x77\x13\xf1\x6c\x21\xee\x52\xe2\x2e\xd9\x09\xf5\x65\x77\x15\x02\xdd\x16\xeb\x4a\x4f\x0e\xd7\x9f\x74\x71\x0f\xab\xb1\x45\x39\x17\xb7\x9f\xb8\x45\xf6\x6c\x57\xd5\xa2\xbc\x8b\x7b\x96\xf5\x77\x8d\xfb\x06\xe9\xf3\x5c\xfd\x85\x88\x7b\x2c\xe7\x4b\x75\xd5\xfe\xad\x9a\x7b\xdc\xc2\x2f\x25\x8e\xa9\x07\xd0\x2e\xf5\x42\xdf\x2b\xe6\x61\x48\xbd\x1f\xd7\x63\xea\x5e\xab\xe1\xe5\x14\xec\x71\x75\x81\x9c\x3a\xc1\x5a\x1b\x33\x6b\xa1\xcc\x95\xd9\xac\x70\x97\x70\x98\xe4\x8f\x27\x5d\x2b\x35\x20\xdc\x16\xfc\xfc\x04\x6b\xc1\xe5\xd3\x57\x6f\x95\xfe\x40\xbc\x35\x92\x5f\xc2\xdd\xd4\xbb\x4a\xe4\x99\xaf\x17\x5c\x9e\xce\xa8\x31\x77\x9f\xd4\x1f\x6b\x54\xfa\xd4\x74\x03\x9a\xff\x97\x0d\xd1\xf9\xf9\x00\x22\xd4\x59\x2e\x3a\x63\xea\x8f\xc4\x57\x0b\x72\xee\x05\x7b\x3c\xc7\xfb\x37\xef\x5d\xc2\x36\x42\x6c\x47\x88\x6d\x3c\x8f\x6d\x28\x57\x03\x6e\x0c\x1f\x21\x4f\x5e\x43\x70\x63\xa9\x02\x63\xd8\x28\xfe\x60\x0d\xc4\x97\xf2\x5f\xe2\xb2\x85\xfe\x6e\x62\x5c\x2a\x18\xb3\x7b\xea\xd3\xe2\x73\xf1\xb7\xeb\xe7\x8b\x61\x9d\x98\x2a\x13\x2f\x02\x4b\x72\xe1\x5c\xfc\x44\x4e\x62\x2c\x71\x59\x94\x77\xe3\xc9\xb8\x48\x7c\x6c\xd3\xed\x8b\x90\xf8\xb8\xb1\x8c\xa7\x4c\xc1\x14\xf3\x33\xcf\x13\xf5\xe9\xd4\x23\x88\x09\x96\xff\xe2\xe2\x4c\x58\x17\x6c\xc2\x9f\xc4\xe6\x4d\x3d\x8c\xcd\xb4\x5f\x92\xe8\x84\x4f\xde\x13\x4f\xc1\xe7\xea\x2a\x7b\x17\x5a\xd9\x64\x95\x70\x31\xfa\x23\xba\x6b\xa3\xa7\xfe\x18\xe2\x71\x93\xfe\xba\xe4\xfa\x95\x3e\xaa\xd9\x40\x5e\x76\xd8\x1b\xe9\x2b\xb7\x27\xb9\xbd\x3e\xf9\x82\x73\x8c\x31\xc9\xfb\x48\x7c\xe2\xf6\xd9\x8c\xfa\x60\x35\xfb\x8e\xf1\x22\x3c\x41\x78\x87\xb8\xf6\x85\x7a\xa5\x4d\x27\x4f\x1d\xa0\x4f\xfb\xd8\x8f\x7c\xc9\x73\x88\xe0\x61\xdc\x02\xf4\x97\xaf\x9e\x84\xdc\x6c\x80\x22\x94\xb2\x16\xb2\xab\xdf\x85\x97\x35\x8f\xd0\x7d\xf9\xf9\x42\x94\x72\xef\xeb\xf2\x2a\x3a\x10\x4f\x9b\xf0\xf4\x00\xc1\x69\x8d\xdf\x2f\x69\xb0\x4c\x72\x92\x2f\x79\x85\x3a\x9b\x67\x74\x6d\x83\xe8\xed\x2b\x5f\x3d\x77\x93\xde\x4f\x72\x7a\xd7\xe8\xdf\xd4\x9b\xd3\xb7\x5d\xf4\xfd\x5c\xa5\x2d\x93\x3c\xd2\x17\xf3\x59\xc5\xa6\xcf\x9c\x8d\x15\x5b\xc5\xc3\xe5\x00\xf7\x16\x9b\xfb\xb2\xd7\xcd\x87\xcc\x4b\x99\xf5\xe6\x25\xc9\xa9\x64\x03\xe7\xc9\xdd\xb1\x15\x5c\xbb\xcd\x9c\x33\x3a\xcc\x79\x73\x36\x9b\xc5\x6b\xbe\x75\xc3\x77\x53\x25\xf9\x27\xd3\xcd\x11\xde\x67\xc4\x0f\xf4\x3b\x70\xd2\xe5\x25\x6f\xb2\xd0\x9d\xef\x90\x79\x79\xf7\x54\xe7\xe7\x6b\xdc\xf9\x9f\xb8\xf3\x7c\x47\x73\x58\xc7\xcf\x72\x6b\x1e\xc3\xb7\x8e\x6b\x3f\x94\x35\x79\x97\x39\x3c\x45\x5b\x2d\xdc\x1f\x89\x98\xf2\x4e\x9b\x72\xce\xe2\x4c\x95\x39\x1f\x2b\x60\x5f\x19\x34\xe7\xf1\x51\x8e\x0b\x71\x90\xb2\xab\xa8\xf3\xa0\x2b\xe7\xfa\x61\xb8\x35\x77\xb6\x4c\xcd\x7c\x6e\x34\xe7\xbd\x81\xa9\xbf\x05\x25\xf6\xb1\x0a\xab\xc8\xbe\x67\x4d\x34\x56\xca\xf3\x0d\x7f\x96\xe6\x5d\x67\x92\xfc\x98\xe3\x1b\xf2\x24\xc7\xfc\x37\xb9\xe5\x72\x58\xb7\x8f\xb2\x56\xb6\x7b\xaf\xc9\xdd\xa4\x75\x01\x1b\x53\xd7\xc8\x3f\x5f\x42\xf8\xe1\x6a\x72\x07\x82\xbc\xef\x78\x1e\x03\xee\x90\xf9\x27\x22\x58\xef\xae\x3f\x5f\xfa\x55\xac\xe4\x0f\x10\xae\x47\xa0\xc3\x2a\xb0\x60\xf0\x7e\x57\x94\xd9\x02\x64\x1e\x86\xd7\x0e\x2f\xb3\x1f\x9e\x99\x5c\xe4\x5d\x8e\x3a\xc7\xfc\xf7\xfb\xa8\xd9\xe0\x8b\x1b\x3d\xf0\xa9\xcb\x51\x3d\x30\x75\x5e\x01\xb5\xa6\xd1\x04\x9f\x93\xe1\x39\x4a\x7e\xef\xea\x17\x7d\xa2\xd7\x3e\xef\xe9\xd9\xeb\x2d\xbc\x8b\xf1\xfe\xc5\xde\x82\xc2\xef\xc9\xf7\x19\x7f\xb8\xf5\x4c\x59\xb8\x6d\x6f\x61\x61\x2d\xe7\xdf\xdb\x5b\x54\x58\x2f\xe7\x11\x3f\xb8\x78\xa5\xff\x8c\xab\xb9\xe9\x2f\x58\x6b\x56\xbf\xd9\xe8\x4b\x7f\x3f\x57\xfb\x9e\x5f\x89\x5d\x59\xe3\x9d\xab\x96\xcf\xbc\x23\x3d\x73\x5c\x1d\x4c\x5d\x41\xc1\x8f\xe6\xd4\xe7\xee\x9d\x6c\x20\xdf\x43\x87\xd9\x43\x79\xa7\x49\x34\xe1\x49\xca\x46\x55\xa6\x4d\x37\x77\xb0\xbe\x46\xd4\xed\xd2\x67\x38\x37\x29\xbd\x6a\xb1\x27\xb9\x3d\x4b\xfa\xcc\xb8\xdc\x79\xea\xc7\x58\xcf\x05\xac\xf9\xcd\xbc\xe3\x08\xcf\x95\x92\x97\xbd\xf2\x2d\xb5\xee\xd6\xf4\x32\x1e\x40\x72\xa9\xa6\xc9\xa3\x2a\xe2\xd6\x29\xe5\xc9\x83\x11\xe2\x9d\x97\xde\xa3\xb6\x75\x19\x77\xcf\xe2\x5e\x67\x4c\x45\x56\x45\x98\x7a\x3b\x2d\x5d\xf8\x8e\x67\x3f\x28\xf7\xd4\xe4\x65\xb8\xb9\x60\x7c\x96\xcd\xd6\x65\xbc\x58\x6d\xf1\x6e\x72\x86\x3d\x6d\x48\x5d\x20\x2f\xe7\xee\x0c\x53\xc9\xfd\xee\x1d\x42\xb8\x60\x44\x7d\x85\x20\xe3\x53\xeb\xc6\x27\xae\x2e\x46\xf5\x9e\x81\xc2\x76\xf6\x86\x5a\xf2\xfa\x02\xf7\xcd\xab\xf9\x76\x9d\x71\xa9\xa5\x4f\x8c\xd7\x34\x14\xf0\xfc\xca\x1e\x47\xdf\xea\x29\xac\x80\xd5\x69\x52\x76\x4e\xdb\x10\xac\x32\xee\xcf\x5e\x57\x7b\xdc\x5c\xd5\xd4\xa1\x49\x5d\xf3\xb7\x55\xf1\x6e\x69\xa4\x46\x0a\x3b\x85\x9f\x2a\x26\x55\x6f\x9e\x9b\x7a\x9c\x41\x58\xf6\x18\x2c\xfa\x2f\x44\x1b\x93\xbf\xf1\x17\x3e\x2d\x58\xe8\xef\x05\xb5\x8f\x35\xa4\xb1\xb7\x5f\x0c\xea\xa9\x47\xbc\xd7\x93\x19\x89\x5c\xd4\x24\xa9\x5a\xc9\x7d\xf0\x69\x81\x49\xdd\xbd\x8f\x96\xd4\xea\xce\x11\x45\x4e\xc3\xae\xe4\x08\x7c\x40\xc4\x94\xff\x9e\xd6\x73\x8c\xef\x18\x76\x31\xa6\xa1\xa4\x03\x3f\xff\x9b\xb4\x96\x39\x68\xee\x5c\x86\xf6\x38\x0a\xc7\x90\xff\x6e\x7f\xc5\xe3\xcd\xd2\xfe\x57\xc4\x77\x30\x78\x1f\x6b\xce\x43\xfe\xdf\x11\xd5\x8d\xc1\x42\xf9\xf7\x30\x6c\x07\xbb\x8e\x8e\xab\xfe\xa3\x87\xd5\x9b\x47\x87\xd5\x5b\xaf\x2e\x78\xbf\xb6\x47\xd0\x03\x2d\xc4\x22\x9e\xd6\x78\x46\x5d\xed\xf0\xad\xd3\x10\x34\xa5\x46\x8f\x66\xd4\xdb\x86\x57\xfc\xa3\x2e\xf4\x0e\xab\xb7\xe9\x9b\xb0\x76\xeb\x8d\xfa\x95\x35\xf2\xed\xd2\x7b\xab\xa5\x36\x52\xdf\x2e\x62\x88\xd2\xdf\x21\xf6\xd0\x05\xde\x63\xd2\xcc\x97\x0c\x63\x3c\xb0\xb2\x1d\x65\xc4\x7c\x8c\x58\x23\xbd\xa3\xea\xd7\xec\xe9\xd5\x6e\x7c\x1b\xb3\x59\x9e\x3b\x28\x39\xe9\x1c\x66\x5c\x0f\x23\x2c\xdc\x2f\xb1\xf5\x94\x6d\xd2\xdd\xd8\x8e\xab\x30\x75\x5e\xb5\xef\x80\xb7\xce\x89\x83\x3e\x68\xe4\x5c\x68\x0d\xf0\x80\xf8\xf9\xf4\x28\xe2\xd4\xd9\x4f\x9d\x6f\xc9\x9d\xa5\xf7\xb8\x7a\x93\xf7\xce\xb7\xd5\xf9\x94\x2e\xff\x79\x0f\x02\x2b\xb4\xba\x7e\xfd\x80\x93\xeb\x4d\x08\xb0\xff\x04\x6e\xf4\x1f\xb9\xbb\x30\x57\x67\x0e\x9c\xc5\x66\xc3\x83\xd8\x81\x0f\x49\x49\x7b\x50\x11\xd3\x76\x57\x1a\x1d\xf5\xe9\xe4\x19\xe6\x37\x66\xca\xb1\x93\xff\x0c\x4f\x22\xaa\x0e\xa1\x02\xe8\x2e\xc7\xf3\x93\x6e\x9e\xdb\x05\x88\x0a\xf7\xb8\x77\xaa\x97\xa3\x46\xfe\x9e\xb5\x5b\xce\xc7\xbb\x6c\x0c\xf1\xa8\x49\xbe\xf6\xf1\x3f\x21\x92\xd8\x2e\xbc\xf7\x4f\xd4\x77\x6b\x59\x4f\x13\xf5\x84\xbb\x2c\xa3\x41\x62\x3e\x53\xbe\x71\xbd\x96\x45\xc9\xb3\x96\x7a\xa7\xdb\x54\x7f\x6f\x37\x33\x4d\xbc\x97\xb2\x55\xd7\x6f\xe0\x7c\xa0\xcb\xf2\x54\x88\xdd\xb6\xf2\xba\xe7\xf8\x5d\x3a\xc5\xff\x98\x76\x93\xfa\x1f\xa3\x9c\x57\xfe\x0d\xfe\x03'), - (0x00f8, 0xb681689e, 0x0994, b'\xf8\xf7\xbf'), - (0x0117, 0x230a2b89, 0x068f, b'\x77\xc6\xfe\x0f'), - (0x0144, 0x7965231d, 0x09b8, b'\xce\xeb\xff\x02'), - (0x017c, 0x328ab6aa, 0x0a99, b'\x6a\xd4\x7f\x01'), - (0x0199, 0x98c16cb9, 0x0a57, b'\xd1\xff\x00'), - ], -} - -padding = { - 'ntsc-beta': [], - 'ntsc-1.0': [ - 0x00, 0x00, 0x00, 0x73, 0x75, 0x47, 0xa4, 0x79, - 0x91, 0x1e, 0xd8, 0xd8, 0x7b, 0xe2, 0x13, 0x51, - 0xaa, 0x2a, 0xa1, 0x5a, 0xbd, 0x24, 0x87, 0x0d, - 0xdc, 0x50, 0xd3, 0xe2, 0x39, 0x9f, 0x1a, 0x86, - 0x73, 0xc5, 0x72, 0xdd, 0xc8, 0xa9, 0xd9, 0x21, - 0xf2, 0xd4, 0x66, 0xa5, 0xbb, 0xfa, 0xbe, 0x6d, - 0xc4, 0xce, 0x63, 0x6f, 0xba, 0xf7, 0xe3, 0xc9, - 0xd4, 0xd3, 0xfb, 0x9c, 0xff, 0x76, 0xcf, 0x20, - 0x6e, 0xbb, 0x2c, 0x90, 0xd3, 0xc1, 0xc4, 0x9f, - 0xb4, 0x70, 0x38, 0xe3, 0x22, 0x3e, 0xec, 0x5c, - 0xa4, 0xdc, 0x39, 0xe8, 0x8b, 0x5e, 0xcc, 0x3e, - 0xa2, 0x62, 0x87, 0x20, 0xd3, 0x05, 0x1c, 0xa5, - 0xd4, 0xca, 0x0e, 0xef, 0xed, 0x23, 0x67, 0xc6, - 0xff, 0x69, 0x66, 0x66, 0x9d, 0xaf, 0xb1, 0x2e, - 0x61, 0xc7, 0x9c, 0x56, 0x60, 0xe6, 0x95, 0xf8, - 0x21, 0x79, 0x93, 0x37, 0xe1, 0x8c, 0xb2, 0xcf, - 0x26, 0x01, 0x9d, 0xf8, 0x6a, 0x8b, 0x71, 0x79, - 0x57, 0xef, 0xaf, 0xb2, 0x27, 0xad, 0x4a, 0xf2, - 0xcc, 0x46, 0x3e, 0x34, 0x0f, 0x31, 0xec, 0xe3, - 0xc2, 0x11, 0xee, 0xb1, 0x4e, 0x3b, 0xcc, 0xd1, - 0xad, 0x4f, 0xbe, 0xef, 0x7b, 0x39, 0xd6, 0x97, - 0xf5, 0xf7, 0x02, 0xa8, 0x36, 0xa1, 0xd7, 0x50, - 0xd6, 0xaf, 0xdf, 0xb9, 0x22, 0x73, 0x4a, 0x37, - 0xf9, 0x46, 0x9d, 0x30, 0x1d, 0x1a, 0x13, 0x1c, - 0x9a, 0xa1, 0x22, 0xa7, 0xb4, 0x9b, 0x88, 0x19, - 0xd4, 0x00, 0xb9, 0xcf, 0xe6, 0xce, 0xf2, 0xd8, - 0xfd, 0x9c, 0x26, 0xe9, 0x4c, 0x59, 0x3d, 0xa6, - 0xa6, 0xa6, 0x07, 0x29, 0x24, 0x7a, 0xba, - ], - 'ntsc-final': [ - 0x00, 0xc2, 0xf9, 0x3a, 0x4d, 0x18, 0x8a, 0x07, - 0x4c, 0x68, 0x38, 0x17, 0x3c, 0x94, 0x98, 0x25, - 0x82, 0x6f, 0xf7, 0x2e, 0x41, 0xed, 0xe4, 0x88, - 0xf5, 0x48, 0x59, 0x5a, 0x7a, 0xb4, 0x3b, 0xf9, - 0xc7, 0xc7, 0xab, 0xf6, 0xef, 0x23, 0x8b, 0x6a, - 0x58, 0xb0, 0x27, 0x84, 0x77, 0x88, 0xe1, 0x34, - 0xf5, 0xf4, 0xa9, 0xb9, 0x30, 0x8a, 0x64, 0x23, - 0xb5, 0x6c, 0x87, 0xff, 0xd4, 0x84, 0xe4, 0x7c, - 0x93, 0xa0, 0x5b, 0x41, 0x28, 0xfa, 0x65, 0x3e, - 0xad, 0x51, 0x35, 0xf9, 0xec, 0x6a, 0xe9, 0xaf, - 0xe0, 0x7f, 0xe5, 0x8e, 0x0e, 0x6b, 0x42, 0x97, - 0xee, 0xad, 0x5d, 0xba, 0x91, 0x7c, 0xd6, 0x91, - 0xb3, 0xbd, 0x5f, 0x3d, 0x48, 0xd1, 0x37, 0xba, - 0xfc, 0x83, 0x51, 0x2b, 0xcb, 0x2f, 0x6b, 0xbf, - 0xb0, 0xe5, 0x9c, 0xac, 0x1d, 0x63, 0xcb, 0xa5, - 0x5e, 0x66, 0x24, 0x2d, 0xe3, 0x86, 0x3e, 0x0c, - 0xcf, 0x1a, 0x57, 0xc9, 0x4b, 0x29, 0x70, 0x31, - 0xbb, 0x55, 0xc4, 0x42, 0x62, 0x5e, 0x9a, 0xa0, - 0xff, 0x41, 0xf5, 0x62, 0x9f, 0xc9, 0x61, 0xee, - 0xbe, 0x7b, 0x0e, 0xf2, 0xd7, 0xf1, 0x90, 0x69, - 0xfa, 0xff, 0xf7, 0xb1, 0x3a, 0x27, 0xac, 0xc2, - 0x57, 0x7e, 0xcc, 0x92, 0xdd, 0x2d, 0x63, 0xa0, - 0x53, 0x74, 0x35, 0xbb, 0x24, 0xde, 0x6d, 0xbb, - 0x2c, 0xe5, 0xff, 0xeb, 0x37, 0xde, 0xd0, 0x6e, - 0x96, 0xfa, 0xbe, 0x79, 0xe3, 0x1e, 0x7f, 0xff, - 0x67, 0x86, 0x86, 0x86, 0x15, 0x6e, - ], - 'pal-final': [ - 0x00, 0x00, 0x00, 0x73, 0xe6, 0xc7, 0xd1, 0xe6, - 0x7c, 0x76, 0xec, 0xdc, 0x9a, 0xf1, 0x6e, 0x54, - 0x05, 0xd9, 0x72, 0x53, 0x7b, 0xb4, 0xbe, 0x5d, - 0xaf, 0xf8, 0x3e, 0xcc, 0x53, 0x8a, 0x75, 0x17, - 0x26, 0x40, 0x29, 0x29, 0x5f, 0xf3, 0x44, 0xc6, - 0xf9, 0xfe, 0x57, 0x1b, 0xf6, 0x7e, 0x45, 0xfa, - 0x9e, 0x75, 0x67, 0xb7, 0x71, 0xb2, 0x9f, 0x19, - 0x02, 0xb2, 0xc5, 0x7c, 0xe2, 0x5b, 0xbb, 0xef, - 0xb2, 0x0f, 0xbe, 0x9a, 0x9c, 0x6b, 0x0d, 0x9a, - 0xfd, 0xed, 0x6d, 0xee, 0xfb, 0x7c, 0x1b, 0xf7, - 0xcf, 0x53, 0xac, 0xf3, 0xeb, 0xcd, 0x8a, 0x4f, - 0x18, 0x4c, 0x5d, 0xeb, 0x52, 0x28, 0x9b, 0xf4, - 0x7d, 0xbf, 0x3f, 0xe9, 0x8f, 0x12, 0x6c, 0x67, - 0xd7, 0x82, 0x6a, 0xdd, 0x4a, 0x9e, 0x5c, 0x23, - 0x7c, 0x18, 0x8a, 0x7d, 0x87, 0x6e, 0x7b, 0x5f, - 0xd5, 0x9d, 0xd4, 0x3f, 0x9f, 0xf9, 0xb6, 0xf6, - 0xd4, 0x9f, 0x71, 0x21, 0x9d, 0xd1, 0xc0, 0x86, - 0x63, 0x6b, 0xd8, 0x87, 0xac, 0x8c, 0x53, 0x6d, - 0xac, 0xc9, 0xbc, 0x8e, 0xdf, 0x13, 0x73, 0x45, - 0x5a, 0xf7, 0x3b, 0xbb, 0x72, 0x66, 0xa5, 0xc0, - 0xb6, 0x9b, 0xe5, 0x98, 0x0b, 0x27, 0xed, 0xc2, - 0x7f, 0x78, 0xb6, 0x39, 0x26, 0xb6, 0x19, 0x7a, - 0x77, 0xea, 0x70, 0x79, 0xff, 0xd7, 0x0f, 0x5f, - 0x53, 0xea, 0x78, 0x30, 0x9b, 0xd2, 0x51, 0x62, - 0xd7, 0xb9, 0x7a, 0xfd, 0x9d, 0x10, 0x64, 0x72, - 0x35, 0x3b, 0xa9, 0xeb, 0x65, 0x8e, 0x25, 0xe8, - 0xe7, 0x51, 0xfa, 0x1f, 0xd1, 0xf7, 0x35, 0x82, - 0x4d, 0xf5, 0xdd, 0x55, 0x23, 0x9b, 0x15, 0xc8, - 0x5a, 0xa2, 0x63, 0x63, 0x63, 0x6f, - ], -} - -main() diff --git a/tools/mkrom/Makefile b/tools/mkrom/Makefile new file mode 100644 index 000000000..a7b513304 --- /dev/null +++ b/tools/mkrom/Makefile @@ -0,0 +1,7 @@ +C_FILES = $(wildcard *.c) + +%.o: %.c + gcc -O3 %< -o $@ + +mkrom: $(C_FILES) + gcc -O3 $(C_FILES) -o mkrom diff --git a/tools/mkrom/game.c b/tools/mkrom/game.c new file mode 100644 index 000000000..a9d9d134c --- /dev/null +++ b/tools/mkrom/game.c @@ -0,0 +1,152 @@ +#include +#include +#include +#include +#include +#include "mkrom.h" + +extern struct state state; + +/** + * This file handles creation of the gamezips segment. + * + * Before mkrom is called, the game segment (uncompressed) is placed past the + * end of the ROM by ld. This segment must be split into 4KB chunks. Each chunk + * is zipped and then placed in its final location within the ROM. + * + * The format of the gamezips segment is: + * - Array of offsets to each chunk, where each offset is 4 bytes and relative + * to the start of the gamezips segment. + * - A final offset that points to the end of the final chunk. + * - Chunk data, where each chunk consists of: + * - A 2 byte checksum of the uncompressed chunk. + * - Zip data (starting with 0x1173001000). + * - Optional single byte to align it to the next 2 byte boundary. + * The added byte is data from the previous chunk. + */ + +/** + * Calculate the checksum of this chunk's raw data. + * + * The game never verifies this, but it exists in the ROM so it has to be + * calculated by mkrom. + * + * It's a simple sum of each word, but then it gets stored as a short so the + * upper half is lost. + */ +static uint32_t crc(uint8_t *buffer, size_t len) +{ + uint32_t sum = 0; + uint32_t offset; + + for (offset = 0; offset < len; offset += 4) { + sum += ntohl(*(uint32_t *) &buffer[offset]); + } + + return sum; +} + +/** + * Create a game chunk. We just calculate and prepend the checksum, then call + * rarezip() which does the zipping and adding of the 0x1173 header. + */ +static void create_chunk(uint8_t *outbuf, size_t *outlen, uint8_t *inbuf, size_t inlen) +{ + uint32_t sum = crc(inbuf, inlen); + + outbuf[0] = (sum >> 8) & 0xff; + outbuf[1] = sum & 0xff; + + rarezip(&outbuf[2], outlen, inbuf, inlen, state.zipmagic); + + *outlen += 2; +} + +/** + * Generate the gamezips segment. + * + * This segment starts with an offset table followed by the chunk data. + * However, we need to keep the offset table zeroed here and build the table + * in a different allocation because the ROM is packed with a duplicate of + * this segment which has an empty offset table. + * + * So this function creates: + * - state.gamezips, which is big enough to hold the offset table plus zips, + * but has the offset table zeroed + * - state.gametable, which is just the table and has the entries populated. + * + * Each chunk is aligned to an even byte. If alignment needs to occur, the extra + * byte is taken from the same offset in the previous chunk. In other words, the + * same output buffer is used for every chunk and it's not cleared between uses. + */ +void game_zip(void) +{ + uint32_t end; + size_t len; + size_t num_chunks; + uint32_t tableoffset; + uint32_t dataoffset; + size_t len_remaining; + uint8_t outscratch[0x1100]; + uint32_t offset; + uint32_t tablelen; + + memset(outscratch, 0, 0x1000); + + // Find the game's position in the ROM and calculate the number of chunks + map_get_segment_rompos("game", &offset, &end); + + len = end - offset; + num_chunks = len / 0x1000; + + if (len % 0x1000) { + num_chunks++; + } + + // Allocate buffers + tablelen = num_chunks * 4 + 4; + + state.gamezips = malloc(tablelen + len); + state.gametable = malloc(tablelen); + state.gametablelen = tablelen; + + memset(state.gamezips, 0, tablelen); + + tableoffset = 0; + dataoffset = tablelen; + + len_remaining = len; + + // Generate the chunks + while (offset < end) { + size_t chunkoriglen = len_remaining >= 0x1000 ? 0x1000 : len_remaining; + size_t chunklen; + + // Write the table entry + state.gametable[tableoffset + 0] = (dataoffset >> 24) & 0xff; + state.gametable[tableoffset + 1] = (dataoffset >> 16) & 0xff; + state.gametable[tableoffset + 2] = (dataoffset >> 8) & 0xff; + state.gametable[tableoffset + 3] = dataoffset & 0xff; + tableoffset += 4; + + // Write the data + create_chunk(outscratch, &chunklen, &state.rom[offset], chunkoriglen); + + chunklen += chunklen % 2; + + memcpy(&state.gamezips[dataoffset], outscratch, chunklen); + + dataoffset += chunklen; + + len_remaining -= 0x1000; + offset += 0x1000; + } + + // The table contains an additional pointer to the end of the final chunk + state.gametable[tableoffset + 0] = (dataoffset >> 24) & 0xff; + state.gametable[tableoffset + 1] = (dataoffset >> 16) & 0xff; + state.gametable[tableoffset + 2] = (dataoffset >> 8) & 0xff; + state.gametable[tableoffset + 3] = dataoffset & 0xff; + + state.gamezipslen = dataoffset; +} diff --git a/tools/mkrom/gzip.h b/tools/mkrom/gzip.h new file mode 100644 index 000000000..614390844 --- /dev/null +++ b/tools/mkrom/gzip.h @@ -0,0 +1,178 @@ +/* gzip.h -- common declarations for all gzip modules + * Copyright (C) 1992-1993 Jean-loup Gailly. + * This is free software; you can redistribute it and/or modify it under the + * terms of the GNU General Public License, see the file COPYING. + */ + +#if defined(__STDC__) +# define OF(args) args +#else +# define OF(args) () +#endif + +#ifdef __STDC__ + typedef void *voidp; +#else + typedef char *voidp; +#endif + +/* I don't like nested includes, but the string and io functions are used + * too often + */ +#include +#if !defined(NO_STRING_H) || defined(STDC_HEADERS) +# include +# if !defined(STDC_HEADERS) && !defined(NO_MEMORY_H) && !defined(__GNUC__) +# include +# endif +# define memzero(s, n) memset ((voidp)(s), 0, (n)) +#else +# include +# define strchr index +# define strrchr rindex +# define memcpy(d, s, n) bcopy((s), (d), (n)) +# define memcmp(s1, s2, n) bcmp((s1), (s2), (n)) +# define memzero(s, n) bzero((s), (n)) +#endif + +#ifndef RETSIGTYPE +# define RETSIGTYPE void +#endif + +#define local static + +typedef unsigned char uch; +typedef unsigned short ush; +typedef unsigned long ulg; + +/* Compression methods (see algorithm.doc) */ +#define STORED 0 +#define COMPRESSED 1 +#define PACKED 2 +#define LZHED 3 +/* methods 4 to 7 reserved */ +#define DEFLATED 8 +#define MAX_METHODS 9 + +#ifndef INBUFSIZ +# ifdef SMALL_MEM +# define INBUFSIZ 0x2000 /* input buffer size */ +# else +# define INBUFSIZ 0x8000 /* input buffer size */ +# endif +#endif +#define INBUF_EXTRA 64 /* required by unlzw() */ + +#ifndef OUTBUFSIZ +# ifdef SMALL_MEM +# define OUTBUFSIZ 8192 /* output buffer size */ +# else +# define OUTBUFSIZ 16384 /* output buffer size */ +# endif +#endif +#define OUTBUF_EXTRA 2048 /* required by unlzw() */ + +#ifndef DIST_BUFSIZE +# ifdef SMALL_MEM +# define DIST_BUFSIZE 0x2000 /* buffer for distances, see trees.c */ +# else +# define DIST_BUFSIZE 0x8000 /* buffer for distances, see trees.c */ +# endif +#endif + +#define EXTERN(type, array) extern type array[] +#define DECLARE(type, array, size) type array[size] +#define ALLOC(type, array, size) +#define FREE(array) + +EXTERN(uch, inbuf); /* input buffer */ +EXTERN(uch, outbuf); /* output buffer */ +EXTERN(ush, d_buf); /* buffer for distances, see trees.c */ +EXTERN(uch, window); /* Sliding window and suffix table (unlzw) */ +#define tab_suffix window +#ifndef MAXSEG_64K +# define tab_prefix prev /* hash link (see deflate.c) */ +# define head (prev+WSIZE) /* hash head (see deflate.c) */ + EXTERN(ush, tab_prefix); /* prefix code (see unlzw.c) */ +#else +# define tab_prefix0 prev +# define head tab_prefix1 + EXTERN(ush, tab_prefix0); /* prefix for even codes */ + EXTERN(ush, tab_prefix1); /* prefix for odd codes */ +#endif + +extern unsigned outcnt; /* bytes in output buffer */ + +/* for compatibility with old zip sources (to be cleaned) */ + +typedef int file_t; /* Do not use stdio */ +#define NO_FILE (-1) /* in memory compression */ + +/* internal file attribute */ +#define UNKNOWN 0xffff +#define BINARY 0 +#define ASCII 1 + +#ifndef WSIZE +# define WSIZE 0x8000 /* window size--must be a power of two, and */ +#endif /* at least 32K for zip's deflate method */ + +#define MIN_MATCH 3 +#define MAX_MATCH 258 +/* The minimum and maximum match lengths */ + +#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) +/* Minimum amount of lookahead, except at the end of the input file. + * See deflate.c for comments about the MIN_MATCH+1. + */ + +#define MAX_DIST (WSIZE-MIN_LOOKAHEAD) +/* In order to simplify the code, particularly on 16 bit machines, match + * distances are limited to MAX_DIST instead of WSIZE. + */ + +/* put_byte is used for the compressed output, put_ubyte for the + * uncompressed output. However unlzw() uses window for its + * suffix table instead of its output buffer, so it does not use put_ubyte + * (to be cleaned up). + */ +#define put_byte(c) {outbuf[outcnt++]=(uch)(c); if (outcnt==OUTBUFSIZ)\ + flush_outbuf();} + +/* Output a 16 bit value, lsb first */ +#define put_short(w) \ + { if (outcnt < OUTBUFSIZ-2) { \ + outbuf[outcnt++] = (uch) ((w) & 0xff); \ + outbuf[outcnt++] = (uch) ((ush)(w) >> 8); \ + } else { \ + put_byte((uch)((w) & 0xff)); \ + put_byte((uch)((ush)(w) >> 8)); \ + } \ + } + +#define seekable() 0 /* force sequential output */ + +/* in gzip.c */ +RETSIGTYPE abort_gzip OF((void)); + +/* in deflate.c */ +void lm_init OF((void)); +ulg deflate OF((void)); + +/* in trees.c */ +void ct_init OF((void)); +int ct_tally OF((int dist, int lc)); +ulg flush_block OF((char *buf, ulg stored_len, int eof)); + +/* in bits.c */ +void bi_init OF((void)); +void send_bits OF((int value, int length)); +unsigned bi_reverse OF((unsigned value, int length)); +void bi_windup OF((void)); +void copy_block OF((char *buf, unsigned len, int header)); +extern int read_buf OF((char *buf, unsigned size)); + +/* in util.c: */ +extern void flush_outbuf OF((void)); +extern void write_buf OF((voidp buf, unsigned cnt)); +extern void warn OF((char *a, char *b)); diff --git a/tools/mkrom/gzip_bits.c b/tools/mkrom/gzip_bits.c new file mode 100644 index 000000000..07ead249b --- /dev/null +++ b/tools/mkrom/gzip_bits.c @@ -0,0 +1,118 @@ +/* bits.c -- output variable-length bit strings + * Copyright (C) 1992-1993 Jean-loup Gailly + * This is free software; you can redistribute it and/or modify it under the + * terms of the GNU General Public License, see the file COPYING. + */ + +#include "gzip.h" +#include "crypt.h" + +/* =========================================================================== + * Local data used by the "bit string" routines. + */ + +local unsigned short bi_buf; +/* Output buffer. bits are inserted starting at the bottom (least significant + * bits). + */ + +#define Buf_size (8 * 2*sizeof(char)) +/* Number of bits used within bi_buf. (bi_buf might be implemented on + * more than 16 bits on some systems.) + */ + +local int bi_valid; +/* Number of valid bits in bi_buf. All bits above the last valid bit + * are always zero. + */ + +/* =========================================================================== + * Initialize the bit string routines. + */ +void bi_init(void) +{ + bi_buf = 0; + bi_valid = 0; +} + +/* =========================================================================== + * Send a value on a given number of bits. + * IN assertion: length <= 16 and value fits in length bits. + */ +void send_bits(value, length) + int value; /* value to send */ + int length; /* number of bits */ +{ + /* If not enough room in bi_buf, use (valid) bits from bi_buf and + * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) + * unused bits in value. + */ + if (bi_valid > (int)Buf_size - length) { + bi_buf |= (value << bi_valid); + put_short(bi_buf); + bi_buf = (ush)value >> (Buf_size - bi_valid); + bi_valid += length - Buf_size; + } else { + bi_buf |= value << bi_valid; + bi_valid += length; + } +} + +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +unsigned bi_reverse(code, len) + unsigned code; /* the value to invert */ + int len; /* its bit length */ +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} + +/* =========================================================================== + * Write out any remaining bits in an incomplete byte. + */ +void bi_windup() +{ + if (bi_valid > 8) { + put_short(bi_buf); + } else if (bi_valid > 0) { + put_byte(bi_buf); + } + bi_buf = 0; + bi_valid = 0; +} + +/* =========================================================================== + * Copy a stored block to the zip file, storing first the length and its + * one's complement if requested. + */ +void copy_block(buf, len, header) + char *buf; /* the input data */ + unsigned len; /* its length */ + int header; /* true if block header must be written */ +{ + bi_windup(); /* align on byte boundary */ + + if (header) { + put_short((ush)len); + put_short((ush)~len); + } + while (len--) { + put_byte(*buf++); + } +} + +void flush_outbuf() +{ + if (outcnt == 0) return; + + write_buf((char *)outbuf, outcnt); + outcnt = 0; +} diff --git a/tools/mkrom/gzip_deflate.c b/tools/mkrom/gzip_deflate.c new file mode 100644 index 000000000..df5ce47e6 --- /dev/null +++ b/tools/mkrom/gzip_deflate.c @@ -0,0 +1,481 @@ +#include +#include "gzip.h" + +/** + * This file is based off gzip 1.2.4, but with unused functionality removed. + * No functionality has been added or changed. + */ + +/* Compile with MEDIUM_MEM to reduce the memory requirements or + * with SMALL_MEM to use as little memory as possible. Use BIG_MEM if the + * entire input file can be held in memory (not possible on 16 bit systems). + * Warning: defining these symbols affects HASH_BITS (see below) and thus + * affects the compression ratio. The compressed output + * is still correct, and might even be smaller in some cases. + */ + +#ifndef HASH_BITS +# define HASH_BITS 15 + /* For portability to 16 bit machines, do not use values above 15. */ +#endif + +/* To save space (see unlzw.c), we overlay prev+head with tab_prefix and + * window with tab_suffix. Check that we can do this: + */ +#define HASH_SIZE (unsigned)(1<= HASH_BITS + */ + +unsigned int prev_length; +/* Length of the best match at previous step. Matches not greater than this + * are discarded. This is used in the lazy match evaluation. + */ + +unsigned strstart; /* start of string to insert */ +unsigned match_start; /* start of matching string */ +local int eofile; /* flag set at end of input file */ +unsigned lookahead; /* number of valid bytes ahead in window */ + +unsigned max_chain_length; +/* To speed up deflation, hash chains are never searched beyond this length. + * A higher limit improves compression ratio but degrades the speed. + */ + +local unsigned int max_lazy_match; +/* Attempt to find a better match only when the current match is strictly + * smaller than this value. This mechanism is used only for compression + * levels >= 4. + */ +#define max_insert_length max_lazy_match +/* Insert new strings in the hash table only if the match length + * is not greater than this length. This saves time but degrades compression. + * max_insert_length is used only for compression levels <= 3. + */ + +unsigned good_match; +/* Use a faster search when the previous match is longer than this */ + + +/* Values for max_lazy_match, good_match and max_chain_length, depending on + * the desired pack level (0..9). The values given below have been tuned to + * exclude worst case performance for pathological files. Better values may be + * found for specific files. + */ + +typedef struct config { + ush good_length; /* reduce lazy search above this match length */ + ush max_lazy; /* do not perform lazy search above this match length */ + ush nice_length; /* quit search above this match length */ + ush max_chain; +} config; + +#ifdef FULL_SEARCH +# define nice_match MAX_MATCH +#else + int nice_match; /* Stop searching when current match exceeds this */ +#endif + +local config configuration_table[10] = { +/* good lazy nice chain */ +/* 0 */ {0, 0, 0, 0}, /* store only */ +/* 1 */ {4, 4, 8, 4}, /* maximum speed, no lazy matches */ +/* 2 */ {4, 5, 16, 8}, +/* 3 */ {4, 6, 32, 32}, + +/* 4 */ {4, 4, 16, 16}, /* lazy matches */ +/* 5 */ {8, 16, 32, 32}, +/* 6 */ {8, 16, 128, 128}, +/* 7 */ {8, 32, 128, 256}, +/* 8 */ {32, 128, 258, 1024}, +/* 9 */ {32, 258, 258, 4096}}; /* maximum compression */ + +/* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4 + * For deflate_fast() (levels <= 3) good is ignored and lazy has a different + * meaning. + */ + +#define EQUAL 0 +/* result of memcmp for equal strings */ + +/* =========================================================================== + * Prototypes for local functions. + */ +local void fill_window OF((void)); + +int longest_match OF((IPos cur_match)); + +/* =========================================================================== + * Update a hash value with the given input byte + * IN assertion: all calls to to UPDATE_HASH are made with consecutive + * input characters, so that a running hash key can be computed from the + * previous key instead of complete recalculation each time. + */ +#define UPDATE_HASH(h,c) (h = (((h)<= 1 + */ + +/* For MSDOS, OS/2 and 386 Unix, an optimized version is in match.asm or + * match.s. The code is functionally equivalent, so you can use the C version + * if desired. + */ +int longest_match(cur_match) + IPos cur_match; /* current match */ +{ + unsigned chain_length = max_chain_length; /* max hash chain length */ + register uch *scan = window + strstart; /* current string */ + register uch *match; /* matched string */ + register int len; /* length of current match */ + int best_len = prev_length; /* best match length so far */ + IPos limit = strstart > (IPos)MAX_DIST ? strstart - (IPos)MAX_DIST : NIL; + + /* Stop when cur_match becomes <= limit. To simplify the code, + * we prevent matches with the string of window index 0. + */ + + /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. + * It is easy to get rid of this optimization if necessary. + */ + register uch *strend = window + strstart + MAX_MATCH; + register uch scan_end1 = scan[best_len-1]; + register uch scan_end = scan[best_len]; + + /* Do not waste too much time if we already have a good match: */ + if (prev_length >= good_match) { + chain_length >>= 2; + } + + // Iterate backwards through the buffer + do { + match = window + cur_match; + + /* Skip to next match if the match length cannot increase + * or if the match length is less than 2: + */ + if (match[best_len] != scan_end || + match[best_len-1] != scan_end1 || + *match != *scan || + *++match != scan[1]) { + continue; + } + + /* The check at best_len-1 can be removed because it will be made + * again later. (This heuristic is not always a win.) + * It is not necessary to compare scan[2] and match[2] since they + * are always equal when the other bytes match, given that + * the hash keys are equal and that HASH_BITS >= 8. + */ + scan += 2, match++; + + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + do { + } while (*++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + scan < strend); + + len = MAX_MATCH - (int)(strend - scan); + + scan = strend - MAX_MATCH; + + if (len > best_len) { + match_start = cur_match; + best_len = len; + if (len >= nice_match) break; + scan_end1 = scan[best_len-1]; + scan_end = scan[best_len]; + } + } while ((cur_match = prev[cur_match & WMASK]) > limit + && --chain_length != 0); + + return best_len; +} + +/* =========================================================================== + * Fill the window when the lookahead becomes insufficient. + * Updates strstart and lookahead, and sets eofile if end of input file. + * IN assertion: lookahead < MIN_LOOKAHEAD && strstart + lookahead > 0 + * OUT assertions: at least one byte has been read, or eofile is set; + * file reads are performed for at least two bytes (required for the + * translate_eol option). + */ +local void fill_window() +{ + register unsigned n, m; + unsigned more = (unsigned)(window_size - (ulg)lookahead - (ulg)strstart); + /* Amount of free space at the end of the window. */ + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (more == (unsigned)EOF) { + /* Very unlikely, but possible on 16 bit machine if strstart == 0 + * and lookahead == 1 (input done one byte at time) + */ + more--; + } else if (strstart >= WSIZE+MAX_DIST) { + /* By the IN assertion, the window is not empty so we can't confuse + * more == 0 with more == 64K on a 16 bit machine. + */ + memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE); + match_start -= WSIZE; + strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */ + + block_start -= (long) WSIZE; + + for (n = 0; n < HASH_SIZE; n++) { + m = head[n]; + head[n] = (Pos)(m >= WSIZE ? m-WSIZE : NIL); + } + for (n = 0; n < WSIZE; n++) { + m = prev[n]; + prev[n] = (Pos)(m >= WSIZE ? m-WSIZE : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } + more += WSIZE; + } + /* At this point, more >= 2 */ + if (!eofile) { + n = read_buf((char*)window+strstart+lookahead, more); + if (n == 0 || n == (unsigned)EOF) { + eofile = 1; + } else { + lookahead += n; + } + } +} + +/* =========================================================================== + * Flush the current block, with given end-of-file flag. + * IN assertion: strstart is set to the end of the current match. + */ +#define FLUSH_BLOCK(eof) \ + flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \ + (char*)NULL, (long)strstart - block_start, (eof)) + +/* =========================================================================== + * Processes a new input file and return its compressed length. We use a lazy + * evaluation for matches: a match is finally adopted only if there is + * no better match at the next window position. + */ +ulg deflate() +{ + IPos hash_head; /* head of hash chain */ + IPos prev_match; /* previous match */ + int flush; /* set if current block must be flushed */ + int match_available = 0; /* set if previous match exists */ + register unsigned match_length = MIN_MATCH-1; /* length of best match */ + + /* Process the input block. */ + while (lookahead != 0) { + /* Insert the string window[strstart .. strstart+2] in the + * dictionary, and set hash_head to the head of the hash chain: + */ + INSERT_STRING(strstart, hash_head); + + /* Find the longest match, discarding those <= prev_length. */ + prev_length = match_length, prev_match = match_start; + match_length = MIN_MATCH-1; + + if (hash_head != NIL && prev_length < max_lazy_match && + strstart - hash_head <= MAX_DIST) { + /* To simplify the code, we prevent matches with the string + * of window index 0 (in particular we have to avoid a match + * of the string with itself at the start of the input file). + */ + match_length = longest_match (hash_head); + /* longest_match() sets match_start */ + if (match_length > lookahead) match_length = lookahead; + + /* Ignore a length 3 match if it is too distant: */ + if (match_length == MIN_MATCH && strstart-match_start > TOO_FAR){ + /* If prev_match is also MIN_MATCH, match_start is garbage + * but we will ignore the current match anyway. + */ + match_length--; + } + } + /* If there was a match at the previous step and the current + * match is not better, output the previous match: + */ + if (prev_length >= MIN_MATCH && match_length <= prev_length) { + + flush = ct_tally(strstart-1-prev_match, prev_length - MIN_MATCH); + + /* Insert in hash table all strings up to the end of the match. + * strstart-1 and strstart are already inserted. + */ + lookahead -= prev_length-1; + prev_length -= 2; + do { + strstart++; + INSERT_STRING(strstart, hash_head); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. If lookahead < MIN_MATCH + * these bytes are garbage, but it does not matter since the + * next lookahead bytes will always be emitted as literals. + */ + } while (--prev_length != 0); + match_available = 0; + match_length = MIN_MATCH-1; + strstart++; + if (flush) FLUSH_BLOCK(0), block_start = strstart; + + } else if (match_available) { + /* If there was no match at the previous position, output a + * single literal. If there was a match but the current match + * is longer, truncate the previous match to a single literal. + */ + if (ct_tally (0, window[strstart-1])) { + FLUSH_BLOCK(0), block_start = strstart; + } + strstart++; + lookahead--; + } else { + /* There is no previous match to compare with, wait for + * the next step to decide. + */ + match_available = 1; + strstart++; + lookahead--; + } + + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the next match, plus MIN_MATCH bytes to insert the + * string following the next match. + */ + while (lookahead < MIN_LOOKAHEAD && !eofile) fill_window(); + } + if (match_available) ct_tally (0, window[strstart-1]); + + return FLUSH_BLOCK(1); /* eof */ +} diff --git a/tools/mkrom/gzip_trees.c b/tools/mkrom/gzip_trees.c new file mode 100644 index 000000000..bb2a0c4de --- /dev/null +++ b/tools/mkrom/gzip_trees.c @@ -0,0 +1,911 @@ +/* trees.c -- output deflated data using Huffman coding + * Copyright (C) 1992-1993 Jean-loup Gailly + * This is free software; you can redistribute it and/or modify it under the + * terms of the GNU General Public License, see the file COPYING. + */ + +#include +#include "gzip.h" + +/* =========================================================================== + * Constants + */ + +#define MAX_BITS 15 +/* All codes must not exceed MAX_BITS bits */ + +#define MAX_BL_BITS 7 +/* Bit length codes must not exceed MAX_BL_BITS bits */ + +#define LENGTH_CODES 29 +/* number of length codes, not counting the special END_BLOCK code */ + +#define LITERALS 256 +/* number of literal bytes 0..255 */ + +#define END_BLOCK 256 +/* end of block literal code */ + +#define L_CODES (LITERALS+1+LENGTH_CODES) +/* number of Literal or Length codes, including the END_BLOCK code */ + +#define D_CODES 30 +/* number of distance codes */ + +#define BL_CODES 19 +/* number of codes used to transfer the bit lengths */ + + +local int extra_lbits[LENGTH_CODES] /* extra bits for each length code */ + = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0}; + +local int extra_dbits[D_CODES] /* extra bits for each distance code */ + = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +local int extra_blbits[BL_CODES]/* extra bits for each bit length code */ + = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7}; + +#define STORED_BLOCK 0 +#define STATIC_TREES 1 +#define DYN_TREES 2 +/* The three kinds of block type */ + +#ifndef LIT_BUFSIZE +# ifdef SMALL_MEM +# define LIT_BUFSIZE 0x2000 +# else +# ifdef MEDIUM_MEM +# define LIT_BUFSIZE 0x4000 +# else +# define LIT_BUFSIZE 0x8000 +# endif +# endif +#endif +#ifndef DIST_BUFSIZE +# define DIST_BUFSIZE LIT_BUFSIZE +#endif +/* Sizes of match buffers for literals/lengths and distances. There are + * 4 reasons for limiting LIT_BUFSIZE to 64K: + * - frequencies can be kept in 16 bit counters + * - if compression is not successful for the first block, all input data is + * still in the window so we can still emit a stored block even when input + * comes from standard input. (This can also be done for all blocks if + * LIT_BUFSIZE is not greater than 32K.) + * - if compression is not successful for a file smaller than 64K, we can + * even emit a stored file instead of a stored block (saving 5 bytes). + * - creating new Huffman trees less frequently may not provide fast + * adaptation to changes in the input data statistics. (Take for + * example a binary file with poorly compressible code followed by + * a highly compressible string table.) Smaller buffer sizes give + * fast adaptation but have of course the overhead of transmitting trees + * more frequently. + * - I can't count above 4 + * The current code is general and allows DIST_BUFSIZE < LIT_BUFSIZE (to save + * memory at the expense of compression). Some optimizations would be possible + * if we rely on DIST_BUFSIZE == LIT_BUFSIZE. + */ +#if LIT_BUFSIZE > INBUFSIZ + error cannot overlay l_buf and inbuf +#endif + +#define REP_3_6 16 +/* repeat previous bit length 3-6 times (2 bits of repeat count) */ + +#define REPZ_3_10 17 +/* repeat a zero length 3-10 times (3 bits of repeat count) */ + +#define REPZ_11_138 18 +/* repeat a zero length 11-138 times (7 bits of repeat count) */ + +/* =========================================================================== + * Local data + */ + +/* Data structure describing a single value and its code string. */ +typedef struct ct_data { + union { + ush freq; /* frequency count */ + ush code; /* bit string */ + } fc; + union { + ush dad; /* father node in Huffman tree */ + ush len; /* length of bit string */ + } dl; +} ct_data; + +#define Freq fc.freq +#define Code fc.code +#define Dad dl.dad +#define Len dl.len + +#define HEAP_SIZE (2*L_CODES+1) +/* maximum heap size */ + +local ct_data dyn_ltree[HEAP_SIZE]; /* literal and length tree */ +local ct_data dyn_dtree[2*D_CODES+1]; /* distance tree */ + +local ct_data static_ltree[L_CODES+2]; +/* The static literal tree. Since the bit lengths are imposed, there is no + * need for the L_CODES extra codes used during heap construction. However + * The codes 286 and 287 are needed to build a canonical tree (see ct_init + * below). + */ + +local ct_data static_dtree[D_CODES]; +/* The static distance tree. (Actually a trivial tree since all codes use + * 5 bits.) + */ + +local ct_data bl_tree[2*BL_CODES+1]; +/* Huffman tree for the bit lengths */ + +typedef struct tree_desc { + ct_data *dyn_tree; /* the dynamic tree */ + ct_data *static_tree; /* corresponding static tree or NULL */ + int *extra_bits; /* extra bits for each code or NULL */ + int extra_base; /* base index for extra_bits */ + int elems; /* max number of elements in the tree */ + int max_length; /* max bit length for the codes */ + int max_code; /* largest code with non zero frequency */ +} tree_desc; + +local tree_desc l_desc = +{dyn_ltree, static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS, 0}; + +local tree_desc d_desc = +{dyn_dtree, static_dtree, extra_dbits, 0, D_CODES, MAX_BITS, 0}; + +local tree_desc bl_desc = +{bl_tree, (ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS, 0}; + + +local ush bl_count[MAX_BITS+1]; +/* number of codes at each bit length for an optimal tree */ + +local uch bl_order[BL_CODES] + = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15}; +/* The lengths of the bit length codes are sent in order of decreasing + * probability, to avoid transmitting the lengths for unused bit length codes. + */ + +local int heap[2*L_CODES+1]; /* heap used to build the Huffman trees */ +local int heap_len; /* number of elements in the heap */ +local int heap_max; /* element of largest frequency */ +/* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used. + * The same heap array is used to build all trees. + */ + +local uch depth[2*L_CODES+1]; +/* Depth of each subtree used as tie breaker for trees of equal frequency */ + +local uch length_code[MAX_MATCH-MIN_MATCH+1]; +/* length code for each normalized match length (0 == MIN_MATCH) */ + +local uch dist_code[512]; +/* distance codes. The first 256 values correspond to the distances + * 3 .. 258, the last 256 values correspond to the top 8 bits of + * the 15 bit distances. + */ + +local int base_length[LENGTH_CODES]; +/* First normalized length for each code (0 = MIN_MATCH) */ + +local int base_dist[D_CODES]; +/* First normalized distance for each code (0 = distance of 1) */ + +#define l_buf inbuf +/* DECLARE(uch, l_buf, LIT_BUFSIZE); buffer for literals or lengths */ + +/* DECLARE(ush, d_buf, DIST_BUFSIZE); buffer for distances */ + +local uch flag_buf[(LIT_BUFSIZE/8)]; +/* flag_buf is a bit array distinguishing literals from lengths in + * l_buf, thus indicating the presence or absence of a distance. + */ + +local unsigned last_lit; /* running index in l_buf */ +local unsigned last_dist; /* running index in d_buf */ +local unsigned last_flags; /* running index in flag_buf */ +local uch flags; /* current flags not yet saved in flag_buf */ +local uch flag_bit; /* current bit used in flags */ +/* bits are filled in flags starting at bit 0 (least significant). + * Note: these flags are overkill in the current code since we don't + * take advantage of DIST_BUFSIZE == LIT_BUFSIZE. + */ + +local ulg opt_len; /* bit length of current block with optimal trees */ +local ulg static_len; /* bit length of current block with static trees */ + +local ulg compressed_len; /* total bit length of compressed file */ + +local ulg input_len; /* total byte length of input file */ +/* input_len is for debugging only since we can get it by other means. */ + +extern long block_start; /* window offset of current block */ +extern unsigned strstart; /* window offset of current string */ + +/* =========================================================================== + * Local (static) routines in this file. + */ + +local void init_block OF((void)); +local void pqdownheap OF((ct_data *tree, int k)); +local void gen_bitlen OF((tree_desc *desc)); +local void gen_codes OF((ct_data *tree, int max_code)); +local void build_tree OF((tree_desc *desc)); +local void scan_tree OF((ct_data *tree, int max_code)); +local void send_tree OF((ct_data *tree, int max_code)); +local int build_bl_tree OF((void)); +local void send_all_trees OF((int lcodes, int dcodes, int blcodes)); +local void compress_block OF((ct_data *ltree, ct_data *dtree)); + + +#define send_code(c, tree) send_bits(tree[c].Code, tree[c].Len) +/* Send a code of the given tree. c and tree must not have side effects */ + +#define d_code(dist) \ + ((dist) < 256 ? dist_code[dist] : dist_code[256+((dist)>>7)]) +/* Mapping from a distance to a distance code. dist is the distance - 1 and + * must not have side effects. dist_code[256] and dist_code[257] are never + * used. + */ + +#define MAX(a,b) (a >= b ? a : b) +/* the arguments must not have side effects */ + +/* =========================================================================== + * Allocate the match buffer, initialize the various tables and save the + * location of the internal file attribute (ascii/binary) and method + * (DEFLATE/STORE). + */ +void ct_init(void) +{ + int n; /* iterates over tree elements */ + int bits; /* bit counter */ + int length; /* length value */ + int code; /* code value */ + int dist; /* distance index */ + + compressed_len = input_len = 0L; + + if (static_dtree[0].Len != 0) return; /* ct_init already called */ + + /* Initialize the mapping length (0..255) -> length code (0..28) */ + length = 0; + for (code = 0; code < LENGTH_CODES-1; code++) { + base_length[code] = length; + for (n = 0; n < (1< dist code (0..29) */ + dist = 0; + for (code = 0 ; code < 16; code++) { + base_dist[code] = dist; + for (n = 0; n < (1<>= 7; /* from now on, all distances are divided by 128 */ + for ( ; code < D_CODES; code++) { + base_dist[code] = dist << 7; + for (n = 0; n < (1<<(extra_dbits[code]-7)); n++) { + dist_code[256 + dist++] = (uch)code; + } + } + + /* Construct the codes of the static literal tree */ + for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; + n = 0; + while (n <= 143) static_ltree[n++].Len = 8, bl_count[8]++; + while (n <= 255) static_ltree[n++].Len = 9, bl_count[9]++; + while (n <= 279) static_ltree[n++].Len = 7, bl_count[7]++; + while (n <= 287) static_ltree[n++].Len = 8, bl_count[8]++; + /* Codes 286 and 287 do not exist, but we must include them in the + * tree construction to get a canonical Huffman tree (longest code + * all ones) + */ + gen_codes((ct_data *)static_ltree, L_CODES+1); + + /* The static distance tree is trivial: */ + for (n = 0; n < D_CODES; n++) { + static_dtree[n].Len = 5; + static_dtree[n].Code = bi_reverse(n, 5); + } + + /* Initialize the first block of the first file: */ + init_block(); +} + +/* =========================================================================== + * Initialize a new block. + */ +local void init_block() +{ + int n; /* iterates over tree elements */ + + /* Initialize the trees. */ + for (n = 0; n < L_CODES; n++) dyn_ltree[n].Freq = 0; + for (n = 0; n < D_CODES; n++) dyn_dtree[n].Freq = 0; + for (n = 0; n < BL_CODES; n++) bl_tree[n].Freq = 0; + + dyn_ltree[END_BLOCK].Freq = 1; + opt_len = static_len = 0L; + last_lit = last_dist = last_flags = 0; + flags = 0; flag_bit = 1; +} + +#define SMALLEST 1 +/* Index within the heap array of least frequent node in the Huffman tree */ + + +/* =========================================================================== + * Remove the smallest element from the heap and recreate the heap with + * one less element. Updates heap and heap_len. + */ +#define pqremove(tree, top) \ +{\ + top = heap[SMALLEST]; \ + heap[SMALLEST] = heap[heap_len--]; \ + pqdownheap(tree, SMALLEST); \ +} + +/* =========================================================================== + * Compares to subtrees, using the tree depth as tie breaker when + * the subtrees have equal frequency. This minimizes the worst case length. + */ +#define smaller(tree, n, m) \ + (tree[n].Freq < tree[m].Freq || \ + (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m])) + +/* =========================================================================== + * Restore the heap property by moving down the tree starting at node k, + * exchanging a node with the smallest of its two sons if necessary, stopping + * when the heap property is re-established (each father smaller than its + * two sons). + */ +local void pqdownheap(tree, k) + ct_data *tree; /* the tree to restore */ + int k; /* node to move down */ +{ + int v = heap[k]; + int j = k << 1; /* left son of k */ + while (j <= heap_len) { + /* Set j to the smallest of the two sons: */ + if (j < heap_len && smaller(tree, heap[j+1], heap[j])) j++; + + /* Exit if v is smaller than both sons */ + if (smaller(tree, v, heap[j])) break; + + /* Exchange v with the smallest son */ + heap[k] = heap[j]; k = j; + + /* And continue down the tree, setting j to the left son of k */ + j <<= 1; + } + heap[k] = v; +} + +/* =========================================================================== + * Compute the optimal bit lengths for a tree and update the total bit length + * for the current block. + * IN assertion: the fields freq and dad are set, heap[heap_max] and + * above are the tree nodes sorted by increasing frequency. + * OUT assertions: the field len is set to the optimal bit length, the + * array bl_count contains the frequencies for each bit length. + * The length opt_len is updated; static_len is also updated if stree is + * not null. + */ +local void gen_bitlen(desc) + tree_desc *desc; /* the tree descriptor */ +{ + ct_data *tree = desc->dyn_tree; + int *extra = desc->extra_bits; + int base = desc->extra_base; + int max_code = desc->max_code; + int max_length = desc->max_length; + ct_data *stree = desc->static_tree; + int h; /* heap index */ + int n, m; /* iterate over the tree elements */ + int bits; /* bit length */ + int xbits; /* extra bits */ + ush f; /* frequency */ + int overflow = 0; /* number of elements with bit length too large */ + + for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; + + /* In a first pass, compute the optimal bit lengths (which may + * overflow in the case of the bit length tree). + */ + tree[heap[heap_max]].Len = 0; /* root of the heap */ + + for (h = heap_max+1; h < HEAP_SIZE; h++) { + n = heap[h]; + bits = tree[tree[n].Dad].Len + 1; + if (bits > max_length) bits = max_length, overflow++; + tree[n].Len = (ush)bits; + /* We overwrite tree[n].Dad which is no longer needed */ + + if (n > max_code) continue; /* not a leaf node */ + + bl_count[bits]++; + xbits = 0; + if (n >= base) xbits = extra[n-base]; + f = tree[n].Freq; + opt_len += (ulg)f * (bits + xbits); + if (stree) static_len += (ulg)f * (stree[n].Len + xbits); + } + if (overflow == 0) return; + + /* Find the first bit length which could increase: */ + do { + bits = max_length-1; + while (bl_count[bits] == 0) bits--; + bl_count[bits]--; /* move one leaf down the tree */ + bl_count[bits+1] += 2; /* move one overflow item as its brother */ + bl_count[max_length]--; + /* The brother of the overflow item also moves one step up, + * but this does not affect bl_count[max_length] + */ + overflow -= 2; + } while (overflow > 0); + + /* Now recompute all bit lengths, scanning in increasing frequency. + * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all + * lengths instead of fixing only the wrong ones. This idea is taken + * from 'ar' written by Haruhiko Okumura.) + */ + for (bits = max_length; bits != 0; bits--) { + n = bl_count[bits]; + while (n != 0) { + m = heap[--h]; + if (m > max_code) continue; + if (tree[m].Len != (unsigned) bits) { + opt_len += ((long)bits-(long)tree[m].Len)*(long)tree[m].Freq; + tree[m].Len = (ush)bits; + } + n--; + } + } +} + +/* =========================================================================== + * Generate the codes for a given tree and bit counts (which need not be + * optimal). + * IN assertion: the array bl_count contains the bit length statistics for + * the given tree and the field len is set for all tree elements. + * OUT assertion: the field code is set for all tree elements of non + * zero code length. + */ +local void gen_codes (tree, max_code) + ct_data *tree; /* the tree to decorate */ + int max_code; /* largest code with non zero frequency */ +{ + ush next_code[MAX_BITS+1]; /* next code value for each bit length */ + ush code = 0; /* running code value */ + int bits; /* bit index */ + int n; /* code index */ + + /* The distribution counts are first used to generate the code values + * without bit reversal. + */ + for (bits = 1; bits <= MAX_BITS; bits++) { + next_code[bits] = code = (code + bl_count[bits-1]) << 1; + } + /* Check that the bit counts in bl_count are consistent. The last code + * must be all ones. + */ + + for (n = 0; n <= max_code; n++) { + int len = tree[n].Len; + if (len == 0) continue; + /* Now reverse the bits */ + tree[n].Code = bi_reverse(next_code[len]++, len); + } +} + +/* =========================================================================== + * Construct one Huffman tree and assigns the code bit strings and lengths. + * Update the total bit length for the current block. + * IN assertion: the field freq is set for all tree elements. + * OUT assertions: the fields len and code are set to the optimal bit length + * and corresponding code. The length opt_len is updated; static_len is + * also updated if stree is not null. The field max_code is set. + */ +local void build_tree(desc) + tree_desc *desc; /* the tree descriptor */ +{ + ct_data *tree = desc->dyn_tree; + ct_data *stree = desc->static_tree; + int elems = desc->elems; + int n, m; /* iterate over heap elements */ + int max_code = -1; /* largest code with non zero frequency */ + int node = elems; /* next internal node of the tree */ + + /* Construct the initial heap, with least frequent element in + * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1]. + * heap[0] is not used. + */ + heap_len = 0, heap_max = HEAP_SIZE; + + for (n = 0; n < elems; n++) { + if (tree[n].Freq != 0) { + heap[++heap_len] = max_code = n; + depth[n] = 0; + } else { + tree[n].Len = 0; + } + } + + /* The pkzip format requires that at least one distance code exists, + * and that at least one bit should be sent even if there is only one + * possible code. So to avoid special checks later on we force at least + * two codes of non zero frequency. + */ + while (heap_len < 2) { + int new = heap[++heap_len] = (max_code < 2 ? ++max_code : 0); + tree[new].Freq = 1; + depth[new] = 0; + opt_len--; if (stree) static_len -= stree[new].Len; + /* new is 0 or 1 so it does not have extra bits */ + } + desc->max_code = max_code; + + /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree, + * establish sub-heaps of increasing lengths: + */ + for (n = heap_len/2; n >= 1; n--) pqdownheap(tree, n); + + /* Construct the Huffman tree by repeatedly combining the least two + * frequent nodes. + */ + do { + pqremove(tree, n); /* n = node of least frequency */ + m = heap[SMALLEST]; /* m = node of next least frequency */ + + heap[--heap_max] = n; /* keep the nodes sorted by frequency */ + heap[--heap_max] = m; + + /* Create a new node father of n and m */ + tree[node].Freq = tree[n].Freq + tree[m].Freq; + depth[node] = (uch) (MAX(depth[n], depth[m]) + 1); + tree[n].Dad = tree[m].Dad = (ush)node; + /* and insert the new node in the heap */ + heap[SMALLEST] = node++; + pqdownheap(tree, SMALLEST); + + } while (heap_len >= 2); + + heap[--heap_max] = heap[SMALLEST]; + + /* At this point, the fields freq and dad are set. We can now + * generate the bit lengths. + */ + gen_bitlen((tree_desc *)desc); + + /* The field len is now set, we can generate the bit codes */ + gen_codes ((ct_data *)tree, max_code); +} + +/* =========================================================================== + * Scan a literal or distance tree to determine the frequencies of the codes + * in the bit length tree. Updates opt_len to take into account the repeat + * counts. (The contribution of the bit length codes will be added later + * during the construction of bl_tree.) + */ +local void scan_tree (tree, max_code) + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + if (nextlen == 0) max_count = 138, min_count = 3; + tree[max_code+1].Len = (ush)0xffff; /* guard */ + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + bl_tree[curlen].Freq += count; + } else if (curlen != 0) { + if (curlen != prevlen) bl_tree[curlen].Freq++; + bl_tree[REP_3_6].Freq++; + } else if (count <= 10) { + bl_tree[REPZ_3_10].Freq++; + } else { + bl_tree[REPZ_11_138].Freq++; + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Send a literal or distance tree in compressed form, using the codes in + * bl_tree. + */ +local void send_tree (tree, max_code) + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + /* tree[max_code+1].Len = -1; */ /* guard already set */ + if (nextlen == 0) max_count = 138, min_count = 3; + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + do { send_code(curlen, bl_tree); } while (--count != 0); + + } else if (curlen != 0) { + if (curlen != prevlen) { + send_code(curlen, bl_tree); count--; + } + send_code(REP_3_6, bl_tree); send_bits(count-3, 2); + + } else if (count <= 10) { + send_code(REPZ_3_10, bl_tree); send_bits(count-3, 3); + + } else { + send_code(REPZ_11_138, bl_tree); send_bits(count-11, 7); + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Construct the Huffman tree for the bit lengths and return the index in + * bl_order of the last bit length code to send. + */ +local int build_bl_tree() +{ + int max_blindex; /* index of last bit length code of non zero freq */ + + /* Determine the bit length frequencies for literal and distance trees */ + scan_tree((ct_data *)dyn_ltree, l_desc.max_code); + scan_tree((ct_data *)dyn_dtree, d_desc.max_code); + + /* Build the bit length tree: */ + build_tree((tree_desc *)(&bl_desc)); + /* opt_len now includes the length of the tree representations, except + * the lengths of the bit lengths codes and the 5+5+4 bits for the counts. + */ + + /* Determine the number of bit length codes to send. The pkzip format + * requires that at least 4 bit length codes be sent. (appnote.txt says + * 3 but the actual value used is 4.) + */ + for (max_blindex = BL_CODES-1; max_blindex >= 3; max_blindex--) { + if (bl_tree[bl_order[max_blindex]].Len != 0) break; + } + /* Update opt_len to include the bit length tree and counts */ + opt_len += 3*(max_blindex+1) + 5+5+4; + + return max_blindex; +} + +/* =========================================================================== + * Send the header for a block using dynamic Huffman trees: the counts, the + * lengths of the bit length codes, the literal tree and the distance tree. + * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. + */ +local void send_all_trees(lcodes, dcodes, blcodes) + int lcodes, dcodes, blcodes; /* number of codes for each tree */ +{ + int rank; /* index in bl_order */ + + send_bits(lcodes-257, 5); /* not +255 as stated in appnote.txt */ + send_bits(dcodes-1, 5); + send_bits(blcodes-4, 4); /* not -3 as stated in appnote.txt */ + for (rank = 0; rank < blcodes; rank++) { + send_bits(bl_tree[bl_order[rank]].Len, 3); + } + + send_tree((ct_data *)dyn_ltree, lcodes-1); /* send the literal tree */ + + send_tree((ct_data *)dyn_dtree, dcodes-1); /* send the distance tree */ +} + +/* =========================================================================== + * Determine the best encoding for the current block: dynamic trees, static + * trees or store, and output the encoded block to the zip file. This function + * returns the total compressed length for the file so far. + */ +ulg flush_block(buf, stored_len, eof) + char *buf; /* input block, or NULL if too old */ + ulg stored_len; /* length of input block */ + int eof; /* true if this is the last block for a file */ +{ + ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ + int max_blindex; /* index of last bit length code of non zero freq */ + + flag_buf[last_flags] = flags; /* Save the flags for the last 8 items */ + + /* Construct the literal and distance trees */ + build_tree((tree_desc *)(&l_desc)); + + build_tree((tree_desc *)(&d_desc)); + /* At this point, opt_len and static_len are the total bit lengths of + * the compressed block data, excluding the tree representations. + */ + + /* Build the bit length tree for the above two trees, and get the index + * in bl_order of the last bit length code to send. + */ + max_blindex = build_bl_tree(); + + /* Determine the best encoding. Compute first the block length in bytes */ + opt_lenb = (opt_len+3+7)>>3; + static_lenb = (static_len+3+7)>>3; + input_len += stored_len; /* for debugging only */ + + if (static_lenb <= opt_lenb) opt_lenb = static_lenb; + + /* If compression failed and this is the first and last block, + * and if the zip file can be seeked (to rewrite the local header), + * the whole file is transformed into a stored file: + */ + if (stored_len <= opt_lenb && eof && compressed_len == 0L && seekable()) { + copy_block(buf, (unsigned)stored_len, 0); /* without header */ + compressed_len = stored_len << 3; + } else if (stored_len+4 <= opt_lenb && buf != (char*)0) { + /* 4: two words for the lengths */ + /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE. + * Otherwise we can't have processed more than WSIZE input bytes since + * the last block flush, because compression would have been + * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to + * transform a block into a stored block. + */ + send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */ + compressed_len = (compressed_len + 3 + 7) & ~7L; + compressed_len += (stored_len + 4) << 3; + + copy_block(buf, (unsigned)stored_len, 1); /* with header */ + } else if (static_lenb == opt_lenb) { + send_bits((STATIC_TREES<<1)+eof, 3); + compress_block((ct_data *)static_ltree, (ct_data *)static_dtree); + compressed_len += 3 + static_len; + } else { + send_bits((DYN_TREES<<1)+eof, 3); + send_all_trees(l_desc.max_code+1, d_desc.max_code+1, max_blindex+1); + compress_block((ct_data *)dyn_ltree, (ct_data *)dyn_dtree); + compressed_len += 3 + opt_len; + } + init_block(); + + if (eof) { + bi_windup(); + compressed_len += 7; /* align on byte boundary */ + } + + return compressed_len >> 3; +} + +/* =========================================================================== + * Save the match info and tally the frequency counts. Return true if + * the current block must be flushed. + */ +int ct_tally (dist, lc) + int dist; /* distance of matched string */ + int lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ +{ + l_buf[last_lit++] = (uch)lc; + if (dist == 0) { + /* lc is the unmatched char */ + dyn_ltree[lc].Freq++; + } else { + /* Here, lc is the match length - MIN_MATCH */ + dist--; /* dist = match distance - 1 */ + dyn_ltree[length_code[lc]+LITERALS+1].Freq++; + dyn_dtree[d_code(dist)].Freq++; + + d_buf[last_dist++] = (ush)dist; + flags |= flag_bit; + } + flag_bit <<= 1; + + /* Output the flags if they fill a byte: */ + if ((last_lit & 7) == 0) { + flag_buf[last_flags++] = flags; + flags = 0, flag_bit = 1; + } + /* Try to guess if it is profitable to stop the current block here */ + if ((last_lit & 0xfff) == 0) { + /* Compute an upper bound for the compressed length */ + ulg out_length = (ulg)last_lit*8L; + ulg in_length = (ulg)strstart-block_start; + int dcode; + for (dcode = 0; dcode < D_CODES; dcode++) { + out_length += (ulg)dyn_dtree[dcode].Freq*(5L+extra_dbits[dcode]); + } + out_length >>= 3; + if (last_dist < last_lit/2 && out_length < in_length/2) return 1; + } + return (last_lit == LIT_BUFSIZE-1 || last_dist == DIST_BUFSIZE); + /* We avoid equality with LIT_BUFSIZE because of wraparound at 64K + * on 16 bit machines and because stored blocks are restricted to + * 64K-1 bytes. + */ +} + +/* =========================================================================== + * Send the block data compressed using the given Huffman trees + */ +local void compress_block(ltree, dtree) + ct_data *ltree; /* literal tree */ + ct_data *dtree; /* distance tree */ +{ + unsigned dist; /* distance of matched string */ + int lc; /* match length or unmatched char (if dist == 0) */ + unsigned lx = 0; /* running index in l_buf */ + unsigned dx = 0; /* running index in d_buf */ + unsigned fx = 0; /* running index in flag_buf */ + uch flag = 0; /* current flags */ + unsigned code; /* the code to send */ + int extra; /* number of extra bits to send */ + + if (last_lit != 0) do { + if ((lx & 7) == 0) flag = flag_buf[fx++]; + lc = l_buf[lx++]; + if ((flag & 1) == 0) { + send_code(lc, ltree); /* send a literal byte */ + } else { + /* Here, lc is the match length - MIN_MATCH */ + code = length_code[lc]; + send_code(code+LITERALS+1, ltree); /* send the length code */ + extra = extra_lbits[code]; + if (extra != 0) { + lc -= base_length[code]; + send_bits(lc, extra); /* send the extra length bits */ + } + dist = d_buf[dx++]; + /* Here, dist is the match distance - 1 */ + code = d_code(dist); + + send_code(code, dtree); /* send the distance code */ + extra = extra_dbits[code]; + if (extra != 0) { + dist -= base_dist[code]; + send_bits(dist, extra); /* send the extra distance bits */ + } + } /* literal or match pair ? */ + flag >>= 1; + } while (lx < last_lit); + + send_code(END_BLOCK, ltree); +} diff --git a/tools/mkrom/main.c b/tools/mkrom/main.c new file mode 100644 index 000000000..a4928478b --- /dev/null +++ b/tools/mkrom/main.c @@ -0,0 +1,73 @@ +#include +#include +#include "mkrom.h" + +struct state state; + +/** + * mkrom - do ROM finalisation steps + * + * mkrom + * + * + * This is the path to the stage1 binary. This file is similar to the final ROM, + * but the allocations for compressed segments are vacant, and the compressed + * segments themselves are found uncompressed past the 32MB mark of the ROM. + * This program will compress those segments, put them in their allocated spaces + * and truncate the ROM to 32MB. + * + * + * This is the path to the linker map, which is used to determine where the + * uncompressed segments are and where they should be placed. + * + * + * This should be 0 or 1 to indicate whether this version of the ROM contains + * piracy checks or not. Some piracy checks work by checksumming functions in + * memory at runtime and comparing it with a known value. If set to 1, mkrom + * will calculate the checksums for these functions and patch them into the + * piracy checks. + * + * + * This is a two byte value which is used when zipping the game segments. + * The original code was influenced by uninitialised data. These two bytes are + * just setting that uninitialised data. + * + * + * The file to write the final ROM file to. + * + * eg. mkrom stage1.bin pd.map 1 0x1234 pd.z64 + */ +int main(int argc, char **argv) +{ + if (argc < 6) { + fprintf(stderr, "Usage: mkrom \n"); + exit(1); + } + + rom_load(argv[1]); + map_open(argv[2]); + + state.piracychecks = atoi(argv[3]); + state.zipmagic = strtol(argv[4], NULL, 16); + + // Compute piracy checksums if requested + if (state.piracychecks) { + piracy_patch(); + } + + // Slice the game segment into chunks and zip each of them to create the + // gamezips segment + game_zip(); + + // Pack each segment into their final locations + pack_lib(); + pack_data(); + pack_game(); + pack_fill(); + + rom_update_crc(); + + rom_write(argv[5]); + + return 0; +} diff --git a/tools/mkrom/map.c b/tools/mkrom/map.c new file mode 100644 index 000000000..7ad9685c2 --- /dev/null +++ b/tools/mkrom/map.c @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include +#include "mkrom.h" + +/** + * This file handles reading and parsing the linker map. + */ + +extern struct state state; + +void map_open(char *filename) +{ + state.mapfd = fopen(filename, "r"); + + if (!state.mapfd) { + fprintf(stderr, "Unable to open map file \"%s\" for reading\n", filename); + exit(1); + } +} + +/** + * Find the start and end offsets of the given function in the ROM and write + * their offsets to the start and end pointers. + * + * Return true if the function was found, false if not. + */ +bool map_get_function_rompos(char *funcname, uint32_t *start, uint32_t *end) +{ + char line[1024]; + char *ptr; + uint32_t segramaddr = 0; + uint32_t segromoffset = 0; + uint32_t ramaddr; + char find[1024]; + bool lookingforend = false; + + snprintf(find, sizeof(find), " %s\n", funcname); + + fseek(state.mapfd, 0, SEEK_SET); + + while (!feof(state.mapfd)) { + fgets(line, 1024, state.mapfd); + + if (lookingforend) { + ptr = line; + + while (isspace(*ptr)) { + ptr++; + } + + if (ptr[0] == '0' && ptr[1] == 'x') { + ramaddr = strtoul(ptr, NULL, 16); + + *end = ramaddr - segramaddr + segromoffset; + return true; + } + } else if (line[0] == '.') { + // Start of a segment + // ".game 0x000000007f000000 0x1b99e0 load address 0x00000000020ac170" + + // Jump to RAM address + ptr = strstr(line, "0x"); + segramaddr = strtoul(ptr, NULL, 16); + + // Jump to length + ptr++; + ptr = strstr(ptr, "0x"); + + // Jump to ROM offset + ptr++; + ptr = strstr(ptr, "0x"); + segromoffset = strtoul(ptr, NULL, 16); + } else if (strstr(line, find)) { + // Found the function + // "0x000000007f15d9a8 bgInflate" + ptr = strstr(line, "0x"); + ramaddr = strtoul(ptr, NULL, 16); + + *start = ramaddr - segramaddr + segromoffset; + lookingforend = true; + } + } + + fprintf(stderr, "Unable to find function \"%s\" in linker map\n", funcname); + + return false; +} + +/** + * Find the start and end offsets of the given segment in the ROM and write + * their offsets to the start and end pointers. + * + * Either point may be NULL. + * + * Return true if the segment was found, false if not. + */ +bool map_get_segment_rompos(char *segname, uint32_t *start, uint32_t *end) +{ + char startstring[64]; + char endstring[64]; + char line[1024]; + bool found_start = false; + bool found_end = false; + + snprintf(startstring, sizeof(startstring), "_%sSegmentRomStart = ", segname); + snprintf(endstring, sizeof(endstring), "_%sSegmentRomEnd = ", segname); + + fseek(state.mapfd, 0, SEEK_SET); + + // Find lines like this: + // " 0x0000000000001050 _libzipSegmentRomStart = __rompos" + + while (!feof(state.mapfd)) { + fgets(line, 1024, state.mapfd); + + if (!found_start && strstr(line, startstring)) { + char *ptr = strstr(line, "0x"); + + if (start != NULL) { + *start = strtoul(ptr, NULL, 16); + } + + if (found_end) { + return true; + } + + found_start = true; + } + + if (!found_end && strstr(line, endstring)) { + char *ptr = strstr(line, "0x"); + + if (end != NULL) { + *end = strtoul(ptr, NULL, 16); + } + + if (found_start) { + return true; + } + + found_end = true; + } + } + + fprintf(stderr, "Unable to find segment \"%s\" in linker map\n", segname); + + return false; +} diff --git a/tools/mkrom/mkrom.h b/tools/mkrom/mkrom.h new file mode 100644 index 000000000..99c7913dc --- /dev/null +++ b/tools/mkrom/mkrom.h @@ -0,0 +1,76 @@ +#include +#include +#include +#include + +struct state { + /** + * A pointer to the full working ROM area in memory. + */ + unsigned char *rom; + + /** + * The size of the above rom allocation in bytes. + * The value is the same filesize as the stage1 binary, + * which is 32MB plus some uncompressed segments on the end. + */ + size_t romlen; + + /** + * Whether piracy checks are enabled for this build or not. + * If enabled, mkrom will recalculate piracy-related checksums. + */ + bool piracychecks; + + /** + * Two bytes that are used to seed some uninitialised data in the input + * buffer when zipping game chunks. + */ + unsigned short zipmagic; + + /** + * File descriptor for the linker map. + */ + FILE *mapfd; + + /** + * A pointer to the gamezips segment, once created. + * The gamezips segment is the full segment but with the offest table + * zeroed. + */ + unsigned char *gamezips; + + /** + * Size of the above gamezips allocation in bytes. + */ + size_t gamezipslen; + + /** + * Pointer to a separate allocation for the gamezips offset table. + */ + unsigned char *gametable; + + /** + * Size of the above gametable allocation in bytes. + */ + size_t gametablelen; +}; + +void game_zip(void); + +void map_open(char *filename); +bool map_get_function_rompos(char *funcname, uint32_t *start, uint32_t *end); +bool map_get_segment_rompos(char *funcname, uint32_t *start, uint32_t *end); + +void pack_lib(void); +void pack_data(void); +void pack_game(void); +void pack_fill(void); + +void piracy_patch(void); + +void rarezip(uint8_t *outbuffer, size_t *outlen, uint8_t *inbuffer, size_t inlen, uint32_t magic); + +void rom_load(char *filename); +void rom_update_crc(void); +void rom_write(char *filename); diff --git a/tools/mkrom/pack.c b/tools/mkrom/pack.c new file mode 100644 index 000000000..4d04891a1 --- /dev/null +++ b/tools/mkrom/pack.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include "mkrom.h" + +extern struct state state; + +static void copy(char *segname, uint8_t *payload, size_t len, char *constname) +{ + uint32_t start; + uint32_t end; + char zipsegname[32]; + uint32_t allocation; + + snprintf(zipsegname, sizeof(zipsegname), "%szip", segname); + + map_get_segment_rompos(zipsegname, &start, &end); + + allocation = end - start; + + if (len > allocation) { + fprintf(stderr, "The %s segment is too big after compression to fit the allocation of 0x%x.\n", segname, allocation); + fprintf(stderr, "In ld/pd.ld, increase the value of %s to 0x%x or higher.\n", constname, len); + exit(1); + } + + memcpy(&state.rom[start], payload, len); +} + +/** + * To pack the data segment, zip it in full and copy it to the datazip segment. + */ +void pack_data(void) +{ + uint32_t start; + uint32_t end; + size_t ziplen; + + map_get_segment_rompos("data", &start, &end); + + uint8_t *buffer = malloc(end - start); + + rarezip(buffer, &ziplen, &state.rom[start], end - start, 0); + + copy("data", buffer, ziplen, "ROMALLOCATION_DATA"); + + free(buffer); +} + +/** + * On the ROM, the gamezips segment exists with its offset table and zip data. + * Then after that comes a copy of the gamezips segment but with the offset + * table cleared. The second segment is garbage data. + * + * mkrom has already built the gamezips segment but with a zeroed table. + * The real table is pointed to by state.gametable. + * + * So we have to copy the gamezips segment twice, then paste the gametable + * segment over the start of the first one. + */ +void pack_game(void) +{ + uint32_t gamezipstart; + uint32_t gamezipend; + size_t truncatedlen; + + // Copy the gamezips segment + copy("game", state.gamezips, state.gamezipslen, "ROMALLOCATION_GAME"); + + // Paste over the offset table + map_get_segment_rompos("gamezip", &gamezipstart, &gamezipend); + + memcpy(&state.rom[gamezipstart], state.gametable, state.gametablelen); + + // Paste the second segment, truncating it to fit the allocation + truncatedlen = gamezipend - gamezipstart - state.gamezipslen; + + if (truncatedlen > state.gamezipslen) { + truncatedlen = state.gamezipslen; + } + + memcpy(&state.rom[gamezipstart + state.gamezipslen], state.gamezips, truncatedlen); + + // The final two bytes from the real segment are duplicated into + // the first two bytes of the second segment's offset table + state.rom[gamezipstart + state.gamezipslen + 0] = state.rom[gamezipstart + state.gamezipslen - 2]; + state.rom[gamezipstart + state.gamezipslen + 1] = state.rom[gamezipstart + state.gamezipslen - 1]; +} + +/** + * The lib segment is zipped from 0x2000 onwards. + * + * It's placed twice in a row in the ROM within its allocation, where the second + * one is truncated and unused. + */ +void pack_lib(void) +{ + uint32_t libzipstart; + uint32_t libzipend; + uint32_t libstart; + uint32_t libend; + size_t ziplen; + size_t seglen; + size_t truncatedlen; + + map_get_segment_rompos("lib", &libstart, &libend); + + uint8_t *buffer = malloc(libend - libstart); + + // Read the first 0x2000 into a buffer + memcpy(buffer, &state.rom[libstart], 0x2000); + + // Compress the remainder from ROM, appending to the buffer + rarezip(&buffer[0x2000], &ziplen, &state.rom[libstart + 0x2000], libend - libstart - 0x2000, 0); + seglen = ziplen + 0x2000; + + // Copy the buffer to its real spot in the ROM + copy("lib", buffer, seglen, "ROMALLOCATION_LIB"); + + // Copy it truncated to its fake spot + map_get_segment_rompos("libzip", &libzipstart, &libzipend); + + truncatedlen = libzipend - libzipstart - seglen; + + if (truncatedlen > seglen) { + truncatedlen = seglen; + } + + memcpy(state.rom + libzipstart + seglen, buffer, truncatedlen); + + free(buffer); +} + +/** + * Fill from the end of the last segment to the end of the ROM with 0xff. + */ +void pack_fill(void) +{ + uint32_t offset; + + map_get_segment_rompos("accessingpak", NULL, &offset); + + if (offset == 0) { + // We're probably building ntsc-beta, which doesn't have the + // accessingpak segment. + map_get_segment_rompos("copyright", NULL, &offset); + } + + while (offset < 1024 * 1024 * 32) { + state.rom[offset] = 0xff; + offset++; + } +} diff --git a/tools/mkrom/piracy.c b/tools/mkrom/piracy.c new file mode 100644 index 000000000..c815246ef --- /dev/null +++ b/tools/mkrom/piracy.c @@ -0,0 +1,170 @@ +#include +#include +#include +#include +#include "mkrom.h" + +#define CHECKSUM_PLACEHOLDER 0x99aabbcc + +extern struct state state; + +typedef uint32_t (*Algo)(uint32_t sum, uint32_t word); + +static uint32_t algo01(uint32_t sum, uint32_t word) { return sum ^ word; } +static uint32_t algo02(uint32_t sum, uint32_t word) { return sum ^ ~word; } +static uint32_t algo03(uint32_t sum, uint32_t word) { return (sum + word) * 2; } +static uint32_t algo04(uint32_t sum, uint32_t word) { return sum + ~word; } +static uint32_t algo05(uint32_t sum, uint32_t word) { return sum * 2 + word; } +static uint32_t algo06(uint32_t sum, uint32_t word) { return sum + word; } +static uint32_t algo07(uint32_t sum, uint32_t word) { return (sum << 1) ^ word; } +static uint32_t algo08(uint32_t sum, uint32_t word) { return (sum + word) + (word >> 1); } +static uint32_t algo09(uint32_t sum, uint32_t word) { return sum - ~word; } +static uint32_t algo10(uint32_t sum, uint32_t word) { return (sum ^ word) << 1; } +static uint32_t algo11(uint32_t sum, uint32_t word) { return (sum ^ ~word) << 1; } + +static uint32_t algo12(uint32_t sum, uint32_t word) { + sum ^= ~word; + sum ^= word << 5; + sum ^= word >> 15; + return sum; +} + +/** + * Calculate the checksum of sumfunc. + * + * We just iterate each word in the function and run the algo function on each. + */ +static uint32_t calc_sum(char *sumfunc, Algo algo) +{ + uint32_t start; + uint32_t end; + uint32_t sum = 0; + uint32_t offset; + + if (!map_get_function_rompos(sumfunc, &start, &end)) { + fprintf(stderr, "Unable to find function \"%s\" in map file\n", sumfunc); + exit(1); + } + + for (offset = start; offset < end; offset += 4) { + sum = algo(sum, ntohl(*(uint32_t *) &state.rom[offset])); + } + + return sum; +} + +static bool is_branch_likely(uint32_t word) +{ + uint32_t op = word & 0xfc000000; + + if (op == 0x50000000) { // beql + return true; + } + + if (op == 0x54000000) { // bnel + return true; + } + + if (op == 0x58000000) { // blezl + return true; + } + + if (op == 0x5c000000) { // bgtzl + return true; + } + + if (op == 0x01000000 && (word & 0x001f0000) == 0x00020000) { // bltzl + return true; + } + + if (op == 0x01000000 && (word & 0x001f0000) == 0x00030000) { // bgezl + return true; + } + + return false; +} + +/** + * Search the patchfunc for the placeholder checksum and replace it with the one + * we calculated. + * + * Checksums are always written into $at with lui and ori instructions. + * + * 3c0199aa lui $at,0x99aa + * 3421bbcc ori $at,$at,0xbbcc + */ +static void write_sum(char *patchfunc, uint32_t sum) +{ + uint32_t start; + uint32_t end; + + if (!map_get_function_rompos(patchfunc, &start, &end)) { + fprintf(stderr, "Unable to find function \"%s\" in map file\n", patchfunc); + exit(1); + } + + bool in_branchlikely = false; + + uint32_t upperpos = 0; + uint32_t lowerpos = 0; + uint32_t offset; + + for (offset = start; offset < end && (!upperpos || !lowerpos); offset += 4) { + uint32_t word = ntohl(*(uint32_t *) &state.rom[offset]); + + if (in_branchlikely) { + in_branchlikely = false; + } else { + if (is_branch_likely(word)) { + in_branchlikely = true; + } else if (word == (0x3c010000 | (CHECKSUM_PLACEHOLDER >> 16))) { + upperpos = offset; + } else if (upperpos && word == (0x34210000 | (CHECKSUM_PLACEHOLDER & 0xffff))) { + lowerpos = offset; + } + } + } + + if (!upperpos || !lowerpos) { + fprintf(stderr, "Unable to find placeholder checksum in %s.\n", patchfunc); + fprintf(stderr, "This can happen if you've turned PIRACYCHECKS off, built the files, then turned it on without rebuilding.\n"); + fprintf(stderr, "To fix, try running the following:\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " touch $(grep -lr PIRACYCHECKS src)\n"); + fprintf(stderr, " make\n"); + fprintf(stderr, "\n"); + exit(1); + } + + state.rom[upperpos + 2] = (sum >> 24) & 0xff; + state.rom[upperpos + 3] = (sum >> 16) & 0xff; + state.rom[lowerpos + 2] = (sum >> 8) & 0xff; + state.rom[lowerpos + 3] = sum & 0xff; +} + +static void patch(Algo algo, char *patchfunc, char *sumfunc) +{ + uint32_t sum = calc_sum(sumfunc, algo); + + write_sum(patchfunc, sum); +} + +/** + * Patch all the piracy functions in the game. + */ +void piracy_patch(void) +{ + // algorithm, patch function, sum function + patch(algo01, "__scHandleTasks", "bootPhase1"); + patch(algo02, "cheatMenuHandleDialog", "__scHandleTasks"); + patch(algo03, "propobjHandlePickupByAibot", "func0f08e2ac"); + patch(algo04, "chrUncloak", "propobjHandlePickupByAibot"); + patch(algo05, "chrsCheckForNoise", "__scHandleRetrace"); + patch(algo06, "lvInit", "lvGetSlowMotionType"); + patch(algo07, "propAllocateEyespy", "lvInit"); + patch(algo08, "chrConsiderGrenadeThrow", "bgInit"); + patch(algo09, "bgun0f09e144", "tagsAllocatePtrs"); + patch(algo10, "explosionAlertChrs", "glassDestroy"); + patch(algo11, "func0f0069dc", "mtxGetObfuscatedRomBase"); + patch(algo12, "func0f15c920", "func0f0069dc"); +} diff --git a/tools/mkrom/rarezip.c b/tools/mkrom/rarezip.c new file mode 100644 index 000000000..d7e7de26d --- /dev/null +++ b/tools/mkrom/rarezip.c @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include "gzip.h" + +/** + * This file is mkrom's interface to gzip. + * + * The key thing here is that we need to be able to set the uninitialised data + * in the window buffer that gzip uses. This is required for a matching build + * and is the reason why mkrom must be written in C. + * + * Much of the gzip code has been ripped out, including file functionality and + * many, many global variables that it uses. + */ + +#define BITS 16 + +// This global variable is required by gzip +unsigned outcnt; /* bytes in gzip's output buffer (not ours) */ + +DECLARE(uch, inbuf, INBUFSIZ +INBUF_EXTRA); +DECLARE(uch, outbuf, OUTBUFSIZ+OUTBUF_EXTRA); +DECLARE(ush, d_buf, DIST_BUFSIZE); +DECLARE(uch, window, 2L*WSIZE); +DECLARE(ush, tab_prefix, 1L<> 8) & 0xff; + window[0x1001] = magic & 0xff; + + outcnt = 0; + + // Begin calling gzip routines + bi_init(); + ct_init(); + lm_init(); + + deflate(); + flush_outbuf(); + + *outlen = outptr - outbuffer; +} + +/** + * Write the RareZip header (0x1173 followed by original file size) + * to the buffer followed by the compressed data. + * + * It's up to the caller to allocate an output buffer big enough. + */ +void rarezip(uint8_t *outbuffer, size_t *outlen, uint8_t *inbuffer, size_t inlen, uint32_t magic) +{ + outbuffer[0] = 0x11; + outbuffer[1] = 0x73; + outbuffer[2] = (inlen >> 16) & 0xff; + outbuffer[3] = (inlen >> 8) & 0xff; + outbuffer[4] = inlen & 0xff; + + zip(&outbuffer[5], outlen, inbuffer, inlen, magic); + + *outlen += 5; +} + +/** + * This function is called by gzip when it wants more data. + */ +int read_buf(char *buf, unsigned size) +{ + if (size > len_remaining) { + size = len_remaining; + } + + memcpy(buf, inptr, size); + + len_remaining -= size; + inptr += size; + + return size; +} + +/** + * This function is called by gzip when it wants to output data. + */ +void write_buf(voidp buf, unsigned size) +{ + memcpy(outptr, buf, size); + outptr += size; +} diff --git a/tools/mkrom/rom.c b/tools/mkrom/rom.c new file mode 100644 index 000000000..d548f23ec --- /dev/null +++ b/tools/mkrom/rom.c @@ -0,0 +1,123 @@ +#include +#include +#include "mkrom.h" + +extern struct state state; + +/** + * Load the stage1 ROM into memory. + */ +void rom_load(char *filename) +{ + FILE *fp = fopen(filename, "rb"); + + if (!fp) { + fprintf(stderr, "Unable to open \"%s\" for reading\n", filename); + exit(1); + } + + fseek(fp, 0, SEEK_END); + state.romlen = ftell(fp); + + state.rom = malloc(state.romlen); + + if (!state.rom) { + fprintf(stderr, "Unable to allocate memory for ROM\n"); + exit(1); + } + + fseek(fp, 0, SEEK_SET); + fread(state.rom, state.romlen, 1, fp); + fclose(fp); +} + +/** + * Write the ROM to the given filename and truncate it to 32MB. + */ +void rom_write(char *filename) +{ + FILE *fp = fopen(filename, "wb"); + + if (!fp) { + fprintf(stderr, "Unable to open \"%s\" for writing\n", filename); + exit(1); + } + + fwrite(state.rom, 1024 * 1024 * 32, 1, fp); + fclose(fp); +} + +static uint32_t rol(uint32_t i, uint32_t b) +{ + return (i << b) | (i >> (32 - b)); +} + +static uint32_t r4(unsigned char *b) +{ + return b[0] * 0x1000000 + b[1] * 0x10000 + b[2] * 0x100 + b[3]; +} + +static void crc(unsigned char *rom, uint32_t *crc1, uint32_t *crc2) +{ + uint32_t seed = 0xdf26f436; + uint32_t t1 = seed; + uint32_t t2 = seed; + uint32_t t3 = seed; + uint32_t t4 = seed; + uint32_t t5 = seed; + uint32_t t6 = seed; + uint32_t offset; + uint32_t d; + uint32_t r; + uint32_t temp; + + unsigned char *lookup = &rom[0x40 + 0x0710]; + + for (offset = 0x1000; offset < 0x101000; offset += 4) { + d = r4(&rom[offset]); + + if ((t6 + d) < t6) { + t4++; + } + + t6 += d; + t3 ^= d; + + r = rol(d, d & 0x1f); + + t5 += r; + + if (t2 > d) { + t2 ^= r; + } else { + t2 ^= t6 ^ d; + } + + temp = r4(&lookup[offset & 0xff]); + t1 += temp ^ d; + } + + *crc1 = t6 ^ t4 ^ t3; + *crc2 = t5 ^ t2 ^ t1; +} + +/** + * Calculate the checksum of the ROM and write it to the ROM header. + */ +void rom_update_crc(void) +{ + uint32_t crc1; + uint32_t crc2; + + crc(state.rom, &crc1, &crc2); + + state.rom[0x10] = (crc1 >> 24) & 0xff; + state.rom[0x11] = (crc1 >> 16) & 0xff; + state.rom[0x12] = (crc1 >> 8) & 0xff; + state.rom[0x13] = crc1 & 0xff; + + state.rom[0x14] = (crc2 >> 24) & 0xff; + state.rom[0x15] = (crc2 >> 16) & 0xff; + state.rom[0x16] = (crc2 >> 8) & 0xff; + state.rom[0x17] = crc2 & 0xff; +} diff --git a/tools/packrom b/tools/packrom deleted file mode 100755 index 8c2f02095..000000000 --- a/tools/packrom +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import subprocess -import sys - -""" -packrom - performs code compression, writing of garbage data (required for a -matching ROM), ROM truncation to 32MB, and filling the tail end of the ROM with -0xff bytes. - -Usage: -packrom -""" - -def zip(binary): - filename = bdir() + '/tmp.bin'; - - fd = open(filename, 'wb') - fd.write(binary) - fd.close() - - zipped = subprocess.check_output(['tools/rarezip', filename]) - os.remove(filename) - return zipped - -def bdir(): - return 'build/%s' % os.environ['ROMID'] - -def edir(): - return 'extracted/%s' % os.environ['ROMID'] - -def get_start(locations, segname): - return next(filter(lambda l: l['name'] == segname, locations))['addr'] - -def get_end(locations, start): - best = 0xffffffff - - for location in locations: - if location['addr'] > start and location['addr'] < best: - best = location['addr'] - - return best; - -def attempt(fd, locations, segname, payload, constname): - # Get location to write to - start = get_start(locations, segname + 'zip') - end = get_end(locations, start) - - # Check it'll fit - allocation = end - start - - if len(payload) > allocation: - print('The %s segment is too big after compression to fit the allocation of 0x%x. In ld/pd.ld, increase the value of %s to 0x%x or higher.' % ( - segname, allocation, constname, len(payload) - )) - exit(1) - - # Write it - fd.seek(start) - fd.write(payload) - -def get_segment(fd, locations, segname): - start = get_start(locations, segname) - end = get_end(locations, start) - - fd.seek(start) - return fd.read(end - start) - -# lib is compressed from offset 0x2000 onwards -def pack_lib(fd, locations): - lib = get_segment(fd, locations, 'lib') - zipped = lib[0:0x2000] + zip(lib[0x2000:]) - attempt(fd, locations, 'lib', zipped, 'ROMALLOCATION_LIB') - -def pack_data(fd, locations): - data = get_segment(fd, locations, 'data') - zipped = zip(data) - attempt(fd, locations, 'data', zipped, 'ROMALLOCATION_DATA') - -def pack_game(fd, locations): - fd2 = open(bdir() + '/segments/gamezips.bin', 'rb') - zips = fd2.read() - fd2.close() - - attempt(fd, locations, 'game', zips, 'ROMALLOCATION_GAME') - -def get_locations(): - fd = open(bdir() + '/pd.map', 'r') - ldmap = fd.read() - fd.close() - - matches = re.findall(r'^\.(\S+)\s+0x[0-9a-f]+\s+0x[0-9a-f]+\s+load address\s+0x([0-9a-f]+)', ldmap, re.MULTILINE) - - def make_numeric(match): - return {'addr': int(match[1], 16), 'name': match[0]} - - return list(map(make_numeric, matches)) - -def write_garbage_part(fd, addr, filename): - fd2 = open(edir() + '/' + filename, 'rb') - binary = fd2.read() - fd2.close() - - fd.seek(addr) - fd.write(binary) - -def write_garbage(fd): - if os.environ['ROMID'] == 'pal-final': - write_garbage_part(fd, 0x2eb21, 'garbage1.bin') - write_garbage_part(fd, 0x158038, 'garbage2.bin') - elif os.environ['ROMID'] == 'ntsc-final': - write_garbage_part(fd, 0x2ea6c, 'garbage1.bin') - write_garbage_part(fd, 0x157800, 'garbage2.bin') - else: - write_garbage_part(fd, 0x2ea22, 'garbage1.bin') - write_garbage_part(fd, 0x1574a0, 'garbage2.bin') - -def fill_tail(fd): - fd2 = open(bdir() + '/pd.map', 'r') - ldmap = fd2.read() - fd2.close() - - match = re.findall(r'^\s*0x([0-9a-f]+)\s+_accessingpakSegmentRomEnd', ldmap, re.MULTILINE) - - pos = int(match[0], 16) - fd.seek(pos) - - while pos < 1024 * 1024 * 32: - fd.write(b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff') - pos += 0x10 - -def main(): - locations = get_locations() - - fd = open(sys.argv[1], 'rb+') - - write_garbage(fd) - - pack_lib(fd, locations) - pack_data(fd, locations) - pack_game(fd, locations) - - fill_tail(fd) - - # Truncate to 32MB - fd.seek(0) - fd.truncate(1024 * 1024 * 32) - fd.close() - -main() diff --git a/tools/patchpiracysums b/tools/patchpiracysums deleted file mode 100755 index ef8a5e227..000000000 --- a/tools/patchpiracysums +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import sys - -""" -patchpiracysums - calculates the expected checksums that are used in piracy -checks and replaces the expected values in the ROM. - -Usage: -patchpiracysums - -To avoid piracy, the game calculates checksums of functions in memory and -compares them with expected values. This script is armed with a list of -locations where these piracy checks happen, as well as the algorithms used in -each, and calculates the expected checksums. - -Locations are referenced by function name and resolved to addresses using a -linker map so it works with shifted ROMs. To find the location of the checksum -within a function, it expects the function to use CHECKSUM_PLACEHOLDER. It -searches for lui and ori instructions that load the placeholder value and -replaces the value with the calculated one. -""" - -CHECKSUM_PLACEHOLDER = 0x99aabbcc - -def algo01(checksum, word): - return checksum ^ word - -def algo02(checksum, word): - return checksum ^ ~word - -def algo03(checksum, word): - return ((checksum + word) & 0xffffffff) * 2 - -def algo04(checksum, word): - return checksum + ~word - -def algo05(checksum, word): - return checksum * 2 + word - -def algo06(checksum, word): - return checksum + word - -def algo07(checksum, word): - checksum = (checksum << 1) & 0xffffffff - return checksum ^ word - -def algo08(checksum, word): - checksum = (checksum + word) & 0xffffffff - return checksum + (word >> 1) - -def algo09(checksum, word): - return checksum - ~word - -def algo10(checksum, word): - return (checksum ^ word) << 1 - -def algo11(checksum, word): - return (checksum ^ ~word) << 1 - -def algo12(checksum, word): - checksum ^= ~word - checksum ^= (word << 5) & 0xffffffff - checksum ^= word >> 15 - return checksum - -class Tool: - def load_map(self): - fd = open(sys.argv[2], 'r') - ldmap = fd.read() - fd.close() - - self.symbols = re.findall(r'^\s*0x([0-9a-f]+)\s+(\S+)$', ldmap, re.MULTILINE) - - # Matching the following line: - # .boot 0x0000000070001000 0x2050 load address 0x0000000000001000 - self.segpositions = re.findall(r'^\.(\S+)\s+0x([0-9a-f]+)\s+0x([0-9a-f]+)\s+load address\s+0x([0-9a-f]+)', ldmap, re.MULTILINE) - - def ramtorom(self, ramaddr): - for pos in self.segpositions: - segname = pos[0] - rampos = int(pos[1], 16) - length = int(pos[2], 16) - rompos = int(pos[3], 16) - - if ramaddr >= rampos and ramaddr < rampos + length: - return rompos + (ramaddr - rampos) - - print('Couldn\'t translate RAM address 0x%08x to ROM' & romaddr) - exit(1) - - def get_function_address(self, funcname): - startram = None - endram = None - - for (index, symbol) in enumerate(list(self.symbols)): - if symbol[1] == funcname: - startram = int(symbol[0], 16) - endram = int(self.symbols[index + 1][0], 16) - break - - if startram is None: - raise ValueError('Unable to find %s in map' % funcname) - - startrom = self.ramtorom(startram) - endrom = self.ramtorom(endram) - return (startrom, endrom) - - def is_branch_likely(self, word): - if word & 0xfc000000 == 0x50000000: # beql - return True - if word & 0xfc000000 == 0x54000000: # bnel - return True - if word & 0xfc000000 == 0x58000000: # blezl - return True - if word & 0xfc000000 == 0x5c000000: # bgtzl - return True - if word & 0xfc000000 == 0x01000000 and word & 0x001f0000 == 0x00020000: # bltzl - return True - if word & 0xfc000000 == 0x01000000 and word & 0x001f0000 == 0x00030000: # bgezl - return True - return False - - def calc_checksum(self, sumfunc, algo): - (pos, end) = self.get_function_address(sumfunc) - self.fd.seek(pos) - checksum = 0 - - while pos < end: - word = int.from_bytes(self.fd.read(4), 'big') - checksum = algo(checksum, word) & 0xffffffff - pos += 4 - - return checksum - - # Checksums are always written into $at with lui and ori - # 3c0199aa lui $at,0x99aa - # 3421bbcc ori $at,$at,0xbbcc - def write_checksum(self, patchfunc, checksum): - (pos, end) = self.get_function_address(patchfunc) - self.fd.seek(pos) - in_branchlikely = False - upperpos = None - lowerpos = None - - while pos < end: - word = int.from_bytes(self.fd.read(4), 'big') - - if in_branchlikely: - in_branchlikely = False - else: - if self.is_branch_likely(word): - in_branchlikely = True - elif word == 0x3c010000 | (CHECKSUM_PLACEHOLDER >> 16): - upperpos = pos - elif upperpos and word == 0x34210000 | (CHECKSUM_PLACEHOLDER & 0xffff): - lowerpos = pos - - pos += 4 - - if upperpos is None or lowerpos is None: - print('Unable to find placeholder checksum in %s.' % patchfunc) - print('This can happen if you\'ve turned PIRACYCHECKS off, built the files, then turned it on without rebuilding.') - print('To fix, try running the following:') - print('') - print(' touch $(grep -lr PIRACYCHECKS src)') - print(' make') - print('') - exit(1) - - self.fd.seek(upperpos) - self.fd.write((0x3c010000 | (checksum >> 16)).to_bytes(4, 'big')) - - self.fd.seek(lowerpos) - self.fd.write((0x34210000 | (checksum & 0xffff)).to_bytes(4, 'big')) - - def patch(self, algo, patchfunc, sumfunc): - checksum = self.calc_checksum(sumfunc, algo) - self.write_checksum(patchfunc, checksum) - - def run(self): - self.load_map() - - self.fd = open(sys.argv[1], 'rb+') - - self.patch(algo01, '__scHandleTasks', 'bootPhase1') - self.patch(algo02, 'cheatMenuHandleDialog', '__scHandleTasks') - self.patch(algo03, 'propobjHandlePickupByAibot', 'func0f08e2ac') - self.patch(algo04, 'chrUncloak', 'propobjHandlePickupByAibot') - self.patch(algo05, 'chrsCheckForNoise', '__scHandleRetrace') - self.patch(algo06, 'lvInit', 'lvGetSlowMotionType') - self.patch(algo07, 'propAllocateEyespy', 'lvInit') - self.patch(algo08, 'chrConsiderGrenadeThrow', 'bgInit') - self.patch(algo09, 'bgun0f09e144', 'tagsAllocatePtrs') - self.patch(algo10, 'explosionAlertChrs', 'glassDestroy') - self.patch(algo11, 'func0f0069dc', 'mtxGetObfuscatedRomBase') - self.patch(algo12, 'func0f15c920', 'func0f0069dc') - - self.fd.close() - -# Piracy checks disabled for ntsc-beta for now... -# it's possible they don't exist in that version. -if os.environ['PIRACYCHECKS'] == '1' and os.environ['ROMID'] != 'ntsc-beta': - tool = Tool() - tool.run() - diff --git a/tools/patchromcrc b/tools/patchromcrc deleted file mode 100755 index 690e5328c..000000000 --- a/tools/patchromcrc +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 - -import sys; - -class Tool: - - def ROL(self, i, b): - return ((i << b) | (i >> (32 - b))) & 0xffffffff - - def R4(self, b): - return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3] - - def crc(self, f): - seed = 0xdf26f436 - t1 = t2 = t3 = t4 = t5 = t6 = seed - - f.seek(0x0710 + 0x40) - lookup = f.read(0x100) - - f.seek(0x1000) - for i in range(0x1000, 0x101000, 4): - d = self.R4(f.read(4)) - - if ((t6 + d) & 0xffffffff) < t6: - t4 += 1 - t4 &= 0xffffffff - - t6 += d - t6 &= 0xffffffff - - t3 ^= d - - r = self.ROL(d, d & 0x1F) - - t5 += r - t5 &= 0xffffffff - - if t2 > d: - t2 ^= r - else: - t2 ^= t6 ^ d - - o = i & 0xFF - temp = self.R4(lookup[o:o + 4]) - t1 += temp ^ d - t1 &= 0xffffffff - - crc1 = t6 ^ t4 ^ t3 - crc2 = t5 ^ t2 ^ t1 - - return crc1 & 0xffffffff, crc2 & 0xffffffff - -fd = open(sys.argv[1], 'rb') - -# Read existing CRC -fd.seek(0x10) -old = [ - int.from_bytes(fd.read(4), 'big'), - int.from_bytes(fd.read(4), 'big'), -] - -# Calculate new CRC -tool = Tool() -new = tool.crc(fd) -fd.close() - -if '--verbose' in sys.argv: - print('Old CRCs: %08x %08x' % (old[0], old[1])) - print('New CRCs: %08x %08x' % (new[0], new[1])) - -if new != old and '--write' in sys.argv: - fd = open(sys.argv[1], 'r+b') - fd.seek(0x10) - fd.write(new[0].to_bytes(4, 'big')) - fd.write(new[1].to_bytes(4, 'big')) - fd.close()