Game-Decompilation/papermario
1
0
Fork 0
mirror of https://github.com/pmret/papermario.git synced 2024-11-27 15:40:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Snug bugs unrelated to and never been inside of a rug (#1082)

Browse Source 
* Fix enum name, offset

* Fix bugs

Some assets were slipping by the asset stack
Tex archve building wasn't respecting the asset stack (Fixes #1074)

* Fixes #1081

* fix paths kinda

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "818924683b"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "818924683b"
git-subrepo:
  version:  "0.4.5"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "aa416e4"

* Fix stuff after splupdate
This commit is contained in:
Ethan Roseman 2023-07-13 17:56:16 +09:00 committed by GitHub
parent 4f77ffbc3e
commit f91fe539a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1096 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/enums.h
View File

@ -5941,7 +5941,7 @@ enum FileMenuMessages {
/* 33 */ FILE_MESSAGE_QUESTION, // ?[End]
/* 34 */ FILE_MESSAGE_PERIOD_34, // .[End]
#if VERSION_PAL
UNK3,
FILE_MESSAGE_BASE_UNK,
#endif
};

6
src/filemenu/filemenu_main.c
View File

@ -567,7 +567,7 @@ void filemenu_draw_contents_file_info(s32 fileIdx,
#if VERSION_PAL
xOffset = D_filemenu_802508D8[gCurrentLanguage];
#else
xOffset = 0x22;
xOffset = 34;
#endif
filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_LEVEL), baseX + xOffset, baseY + 10, 255, 0xA, 1);
temp_s3_2 = gSaveSlotMetadata[fileIdx].level;
@ -625,12 +625,12 @@ void filemenu_draw_contents_file_title(
filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_OK), baseX + FILE_X, baseY + 1, 255, 0, 1);
if (!gSaveSlotHasData[fileIdx]) {
filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3),
filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK),
baseX + D_filemenu_802508D0[gCurrentLanguage], baseY + 1, 255, 0, 1);
} else {
s32 tmp = D_filemenu_802508D0[gCurrentLanguage];
filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3),
filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK),
baseX + tmp, baseY + 1, 255, 0, 1);
tmp += D_filemenu_802508D4[gCurrentLanguage];

22
tools/build/configure.py
View File

@ -16,6 +16,9 @@ DO_SHA1_CHECK = True
# Paths:
ROOT = Path(__file__).parent.parent.parent
if ROOT.is_absolute():
ROOT = ROOT.relative_to(Path.cwd())
BUILD_TOOLS = Path("tools/build")
YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress"
CRC_TOOL = f"{BUILD_TOOLS}/rom/n64crc"
@ -241,7 +244,7 @@ def write_ninja_rules(
ninja.rule(
"tex",
description="tex $out",
command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_dir",
command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_name $asset_stack",
)
ninja.rule(
@ -408,6 +411,9 @@ class Configure:
@lru_cache(maxsize=None)
def resolve_asset_path(self, path: Path) -> Path:
# Remove nonsense
path = Path(os.path.normpath(path))
parts = list(path.parts)
if parts[0] != "assets":
@ -493,7 +499,7 @@ class Configure:
ninja.build(
outputs=object_strs, # $out
rule=task,
inputs=self.resolve_src_paths(src_paths), # $in
inputs=inputs, # $in
implicit=implicit,
order_only=order_only,
variables={"version": self.version, **variables},
@ -743,6 +749,7 @@ class Configure:
"sprite_name": sprite_name,
"asset_stack": ",".join(self.asset_stack),
},
asset_deps=[str(sprite_dir)],
)
build(yay0_path, [bin_path], "yay0")
@ -801,9 +808,8 @@ class Configure:
)
build(entry.object_path, [entry.object_path.with_suffix(".bin")], "bin")
elif seg.type == "pm_map_data":
bin_yay0s: List[
Path
] = [] # flat list of (uncompressed path, compressed? path) pairs
# flat list of (uncompressed path, compressed? path) pairs
bin_yay0s: List[Path] = []
src_dir = Path("assets/x") / seg.name
for path in entry.src_paths:
@ -906,7 +912,11 @@ class Configure:
bin_path,
[tex_dir, path.parent / (name + ".json")],
"tex",
variables={"tex_dir": str(tex_dir)},
variables={
"tex_name": name,
"asset_stack": ",".join(self.asset_stack),
},
asset_deps=[f"mapfs/tex/{name}"],
)
elif name.endswith("_shape"):
map_name = "_".join(name.split("_")[:-1])

190
tools/build/mapfs/tex.py
View File

@ -1,19 +1,199 @@
#!/usr/bin/env python3
import argparse
import json
import os
from pathlib import Path
from sys import argv, path
from sys import path
from typing import Tuple
path.append(str(Path(__file__).parent.parent.parent / "splat"))
path.append(str(Path(__file__).parent.parent.parent / "splat_ext"))
from tex_archives import TexArchive
path.append(str(Path(__file__).parent.parent.parent / "build"))
from common import get_asset_path
path.append(str(Path(__file__).parent.parent.parent))
from splat_ext.tex_archives import (
AUX_COMBINE_MODES_INV,
TILES_BASIC,
TILES_INDEPENDENT_AUX,
TILES_MIPMAPS,
TILES_SHARED_AUX,
TexImage,
get_format_code,
)
# read texture properties from dictionary and load images
def img_from_json(json_data, tex_name: str, asset_stack: Tuple[Path, ...]) -> TexImage:
ret = TexImage()
ret.img_name = json_data["name"]
if "ext" in json_data:
ret.raw_ext = json_data["ext"]
else:
ret.raw_ext = "tif"
# read data for main tile
main_data = json_data.get("main")
if main_data == None:
raise Exception(f"Texture {ret.img_name} has no definition for 'main'")
(main_fmt_name, ret.main_hwrap, ret.main_vwrap) = ret.read_json_img(
main_data, "main", ret.img_name
)
(ret.main_fmt, ret.main_depth) = get_format_code(main_fmt_name)
# read main image
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}.png"), asset_stack
)
if not os.path.isfile(img_path):
raise Exception(f"Could not find main image for texture: {ret.img_name}")
(
ret.main_img,
ret.main_pal,
ret.main_width,
ret.main_height,
) = ret.get_img_file(main_fmt_name, str(img_path))
# read data for aux tile
ret.has_aux = "aux" in json_data
if ret.has_aux:
aux_data = json_data.get("aux")
(aux_fmt_name, ret.aux_hwrap, ret.aux_vwrap) = ret.read_json_img(
aux_data, "aux", ret.img_name
)
if aux_fmt_name == "Shared":
# aux tiles have blank attributes in SHARED mode
aux_fmt_name = main_fmt_name
ret.aux_fmt = 0
ret.aux_depth = 0
ret.aux_hwrap = 0
ret.aux_vwrap = 0
ret.extra_tiles = TILES_SHARED_AUX
else:
(ret.aux_fmt, ret.aux_depth) = get_format_code(aux_fmt_name)
ret.extra_tiles = TILES_INDEPENDENT_AUX
# read aux image
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}_AUX.png"), asset_stack
)
if not os.path.isfile(img_path):
raise Exception(f"Could not find AUX image for texture: {ret.img_name}")
(
ret.aux_img,
ret.aux_pal,
ret.aux_width,
ret.aux_height,
) = ret.get_img_file(aux_fmt_name, str(img_path))
if ret.extra_tiles == TILES_SHARED_AUX:
# aux tiles have blank sizes in SHARED mode
ret.main_height *= 2
ret.aux_width = 0
ret.aux_height = 0
else:
ret.aux_fmt = 0
ret.aux_depth = 0
ret.aux_hwrap = 0
ret.aux_vwrap = 0
ret.aux_width = 0
ret.aux_height = 0
ret.extra_tiles = TILES_BASIC
# read mipmaps
ret.has_mipmaps = json_data.get("hasMipmaps", False)
if ret.has_mipmaps:
ret.mipmaps = []
mipmap_idx = 1
divisor = 2
if ret.main_width >= (32 >> ret.main_depth):
while True:
if (ret.main_width // divisor) <= 0:
break
mmw = ret.main_width // divisor
mmh = ret.main_height // divisor
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}_MM{mipmap_idx}.png"),
asset_stack,
)
if not os.path.isfile(img_path):
raise Exception(
f"Texture {ret.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})"
)
(raster, pal, width, height) = ret.get_img_file(
main_fmt_name, str(img_path)
)
ret.mipmaps.append(raster)
if width != mmw or height != mmh:
raise Exception(
f"Texture {ret.img_name} has wrong size for mipmap level {mipmap_idx} \n"
+ f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}"
)
divisor = divisor * 2
mipmap_idx += 1
if (ret.main_width // divisor) < (16 >> ret.main_depth):
break
ret.extra_tiles = TILES_MIPMAPS
# read filter mode
if json_data.get("filter", False):
ret.filter_mode = 2
else:
ret.filter_mode = 0
# read tile combine mode
combine_str = json_data.get("combine", "Missing")
ret.combine = AUX_COMBINE_MODES_INV.get(combine_str)
if ret.combine == None:
raise Exception(f"Texture {ret.img_name} has invalid 'combine'")
ret.is_variant = json_data.get("variant", False)
return ret
def build(
out_path: Path, tex_name: str, asset_stack: Tuple[Path, ...], endian: str = "big"
):
out_bytes = bytearray()
json_path = get_asset_path(Path(f"mapfs/tex/{tex_name}.json"), asset_stack)
with open(json_path) as json_file:
json_str = json_file.read()
json_data = json.loads(json_str)
if len(json_data) > 128:
raise Exception(
f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`"
)
for img_data in json_data:
img = img_from_json(img_data, tex_name, asset_stack)
img.add_bytes(tex_name, out_bytes)
with open(out_path, "wb") as out_bin:
out_bin.write(out_bytes)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Texture archives")
parser.add_argument("bin_out", type=Path, help="Output binary file path")
parser.add_argument("tex_dir", type=Path, help="File path to input tex subdirectory")
parser.add_argument("name", help="Name of tex subdirectory")
parser.add_argument("asset_stack", help="comma-separated asset stack")
parser.add_argument(
"--endian", choices=["big", "little"], default="big", help="Output endianness"
)
args = parser.parse_args()
TexArchive.build(args.bin_out, args.tex_dir, args.endian)
asset_stack = tuple(Path(d) for d in args.asset_stack.split(","))
build(args.bin_out, args.name, asset_stack, args.endian)

4
tools/splat/.gitrepo
View File

@ -6,7 +6,7 @@
[subrepo]
remote = https://github.com/ethteck/splat.git
branch = master
commit = e72a868f9f7e9da25f13194b51b93e64c2dcc83a
parent = 4dfc35713736d65d8d607a0e0f4121bc6938d6e3
commit = 818924683bfb1145129b5e43ff02abe7b4be37a3
parent = c122fe97362f81b15d8ede79f24b28a62e859a68
method = merge
cmdver = 0.4.5

32
tools/splat/CHANGELOG.md
View File

@ -1,5 +1,29 @@
# splat Release Notes
### 0.15.1
* Made some modifications such that linker object paths should be simpler in some circumstances
### 0.15.0
* New options:
* `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII.
* `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1.
* level 0: Completely disable the guessing feature.
* level 1: The most conservative guessing level. Imposes the following restrictions:
* Do not try to guess if the user provided a type for the symbol.
* Do no try to guess if type information for the symbol can be inferred by other means.
* A string symbol must be referenced only once.
* Strings must not be empty.
* level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization.
* level 3: Empty strings are allowed.
* level 4: Symbols with autodetected type information but no user type information can still be guessed as strings.
* `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2.
* `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True.
### 0.14.1
* Fix bug, cod cleanup
### 0.14.0
* Add support for PSX's GTE instruction set
@ -40,7 +64,7 @@
### 0.13.3
* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieve this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays.
* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays.
### 0.13.2
@ -61,7 +85,7 @@
### 0.12.14
* New option: `pair_rodata_to_text`.
* If enabled, splat will try to find to which text segment an unpaired rodata segment belongs and it will hint it to the user.
* If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user.
### 0.12.13
@ -91,7 +115,7 @@
### 0.12.8
* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration.
* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component.
* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component.
### 0.12.7
@ -115,7 +139,7 @@
* Update minimal spimdisasm version to 1.7.1.
* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols.
* An check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user.
* A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user.
### 0.12.4

17
tools/splat/disassembler/spimdisasm_disassembler.py
View File

@ -8,7 +8,7 @@ from typing import Set
class SpimdisasmDisassembler(disassembler.Disassembler):
# This value should be kept in sync with the version listed on requirements.txt
SPIMDISASM_MIN = (1, 13, 0)
SPIMDISASM_MIN = (1, 15, 0)
def configure(self, opts: SplatOpts):
# Configure spimdisasm
@ -28,6 +28,16 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False
if opts.rodata_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = (
opts.rodata_string_guesser_level
)
if opts.data_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = (
opts.data_string_guesser_level
)
rabbitizer.config.regNames_userFpcCsr = False
rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False
@ -70,6 +80,11 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro
spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label
if opts.asm_emit_size_directive is not None:
spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = (
opts.asm_emit_size_directive
)
if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl":
spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent"
spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True

283
tools/splat/disassembler_section.py Normal file
View File

@ -0,0 +1,283 @@
import spimdisasm
from util import symbols
from typing import Optional, Set, Tuple
from segtypes.segment import Segment
from util import log, options, symbols
from abc import ABC, abstractmethod
from typing import Callable
class DisassemblerSection(ABC):
@abstractmethod
def disassemble(self):
raise NotImplementedError("disassemble")
@abstractmethod
def analyze(self):
raise NotImplementedError("analyze")
@abstractmethod
def set_comment_offset(self, rom_start: int):
raise NotImplementedError("set_comment_offset")
@abstractmethod
def make_bss_section(
self,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_bss_section")
@abstractmethod
def make_data_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_data_section")
@abstractmethod
def get_section(self):
raise NotImplementedError("get_section")
@abstractmethod
def make_rodata_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_rodata_section")
@abstractmethod
def make_text_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_text_section")
class SpimdisasmDisassemberSection(DisassemblerSection):
def __init__(self):
self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None
def disassemble(self) -> str:
assert self.spim_section is not None
return self.spim_section.disassemble()
def analyze(self):
assert self.spim_section is not None
self.spim_section.analyze()
def set_comment_offset(self, rom_start: int):
assert self.spim_section is not None
self.spim_section.setCommentOffset(rom_start)
def make_bss_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionBss(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
def make_data_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionData(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]:
return self.spim_section
def make_rodata_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionRodata(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_text_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionText(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]:
if options.opts.platform in ["n64", "psx", "ps2"]:
return SpimdisasmDisassemberSection()
raise NotImplementedError("No disassembler section for requested platform")
return None
def make_text_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_text_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_data_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_data_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_rodata_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_rodata_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_bss_section(
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_bss_section(
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
return section

2
tools/splat/requirements.txt
View File

@ -4,7 +4,7 @@ tqdm
intervaltree
colorama
# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
spimdisasm>=1.13.0
spimdisasm>=1.15.0
rabbitizer>=1.7.0
pygfxd
n64img>=0.1.4

14
tools/splat/segtypes/common/bss.py
View File

@ -1,8 +1,9 @@
import spimdisasm
from util import options, symbols, log
from segtypes.common.data import CommonSegData
from disassembler_section import make_bss_section
class CommonSegBss(CommonSegData):
def get_linker_section(self) -> str:
@ -37,8 +38,7 @@ class CommonSegBss(CommonSegData):
bss_end = next_subsegment.vram_start
assert isinstance(bss_end, int), f"{self.name} {bss_end}"
self.spim_section = spimdisasm.mips.sections.SectionBss(
symbols.spim_context,
self.spim_section = make_bss_section(
self.rom_start,
self.rom_end,
self.vram_start,
@ -48,10 +48,12 @@ class CommonSegBss(CommonSegData):
self.get_exclusive_ram_id(),
)
self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start)
assert self.spim_section is not None
for spim_sym in self.spim_section.symbolList:
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
for spim_sym in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), spim_sym.contextSym
)

12
tools/splat/segtypes/common/c.py
View File

@ -153,10 +153,10 @@ class CommonSegC(CommonSegCodeSubsegment):
self.print_file_boundaries()
assert self.spim_section is not None and isinstance(
self.spim_section, spimdisasm.mips.sections.SectionText
self.spim_section.get_section(), spimdisasm.mips.sections.SectionText
), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}"
rodata_spim_segment = None
rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None
if (
options.opts.migrate_rodata_to_functions
and self.rodata_sibling is not None
@ -166,15 +166,15 @@ class CommonSegC(CommonSegCodeSubsegment):
), self.rodata_sibling.type
if self.rodata_sibling.spim_section is not None:
assert isinstance(
self.rodata_sibling.spim_section,
self.rodata_sibling.spim_section.get_section(),
spimdisasm.mips.sections.SectionRodata,
)
rodata_spim_segment = self.rodata_sibling.spim_section
rodata_spim_segment = self.rodata_sibling.spim_section.get_section()
# Precompute function-rodata pairings
symbols_entries = (
spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections(
self.spim_section, rodata_spim_segment
self.spim_section.get_section(), rodata_spim_segment
)
)
@ -319,7 +319,7 @@ class CommonSegC(CommonSegCodeSubsegment):
macro_name: str,
) -> str:
if options.opts.compiler == IDO:
# IDO uses the asm processor to embeed assembly and it doesn't require a special directive to include symbols
# IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols
asm_outpath = Path(
os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s")
)

17
tools/splat/segtypes/common/code.py
View File

@ -278,8 +278,17 @@ class CommonSegCode(CommonSegGroup):
)
segment.sibling = base_segments.get(segment.name, None)
if segment.is_rodata() and segment.sibling is not None:
segment.sibling.rodata_sibling = segment
if segment.sibling is not None:
if self.section_order.index(".text") < self.section_order.index(
".rodata"
):
if segment.is_rodata():
segment.sibling.rodata_sibling = segment
else:
if segment.is_text() and segment.sibling.is_rodata():
segment.rodata_sibling = segment.sibling
segment.sibling.sibling = segment
segment.parent = self
if segment.special_vram_segment:
@ -300,6 +309,10 @@ class CommonSegCode(CommonSegGroup):
if segment.is_text():
base_segments[segment.name] = segment
if self.section_order.index(".rodata") < self.section_order.index(".text"):
if segment.is_rodata() and segment.sibling is None:
base_segments[segment.name] = segment
prev_start = start
if end is not None:
last_rom_end = end

31
tools/splat/segtypes/common/codesubsegment.py
View File

@ -10,6 +10,8 @@ from segtypes.common.code import CommonSegCode
from segtypes.segment import Segment
from disassembler_section import DisassemblerSection, make_text_section
# abstract class for c, asm, data, etc
class CommonSegCodeSubsegment(Segment):
@ -24,7 +26,7 @@ class CommonSegCodeSubsegment(Segment):
self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None
)
self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None
self.spim_section: Optional[DisassemblerSection] = None
self.instr_category = rabbitizer.InstrCategory.CPU
if options.opts.platform == "ps2":
self.instr_category = rabbitizer.InstrCategory.R5900
@ -56,8 +58,7 @@ class CommonSegCodeSubsegment(Segment):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = spimdisasm.mips.sections.SectionText(
symbols.spim_context,
self.spim_section = make_text_section(
self.rom_start,
self.rom_end,
self.vram_start,
@ -67,13 +68,15 @@ class CommonSegCodeSubsegment(Segment):
self.get_exclusive_ram_id(),
)
self.spim_section.isHandwritten = is_hasm
self.spim_section.instrCat = self.instr_category
assert self.spim_section is not None
self.spim_section.get_section().isHandwritten = is_hasm
self.spim_section.get_section().instrCat = self.instr_category
self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start)
self.spim_section.set_comment_offset(self.rom_start)
for func in self.spim_section.symbolList:
for func in self.spim_section.get_section().symbolList:
assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction)
self.process_insns(func)
@ -85,7 +88,7 @@ class CommonSegCodeSubsegment(Segment):
jtbl_label_vram, True, type="jtbl_label", define=True
)
sym.type = "jtbl_label"
symbols.add_symbol_to_spim_section(self.spim_section, sym)
symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym)
def process_insns(
self,
@ -103,7 +106,7 @@ class CommonSegCodeSubsegment(Segment):
# Gather symbols found by spimdisasm and create those symbols in splat's side
for referenced_vram in func_spim.instrAnalyzer.referencedVrams:
context_sym = self.spim_section.getSymbol(
context_sym = self.spim_section.get_section().getSymbol(
referenced_vram, tryPlusOffset=False
)
if context_sym is not None:
@ -124,7 +127,7 @@ class CommonSegCodeSubsegment(Segment):
if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset:
sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset]
context_sym = self.spim_section.getSymbol(
context_sym = self.spim_section.get_section().getSymbol(
sym_address, tryPlusOffset=False
)
if context_sym is not None:
@ -141,7 +144,7 @@ class CommonSegCodeSubsegment(Segment):
assert isinstance(self.rom_start, int)
for in_file_offset in self.spim_section.fileBoundaries:
for in_file_offset in self.spim_section.get_section().fileBoundaries:
if (in_file_offset % 16) != 0:
continue
@ -150,8 +153,10 @@ class CommonSegCodeSubsegment(Segment):
# Look up for the last symbol in this boundary
sym_addr = 0
for sym in self.spim_section.symbolList:
symOffset = sym.inFileOffset - self.spim_section.inFileOffset
for sym in self.spim_section.get_section().symbolList:
symOffset = (
sym.inFileOffset - self.spim_section.get_section().inFileOffset
)
if in_file_offset == symOffset:
break
sym_addr = sym.vram

22
tools/splat/segtypes/common/data.py
View File

@ -7,6 +7,8 @@ from util import options, symbols, log
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
from segtypes.common.group import CommonSegGroup
from disassembler_section import DisassemblerSection, make_data_section
class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
def out_path(self) -> Optional[Path]:
@ -89,8 +91,7 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = spimdisasm.mips.sections.SectionData(
symbols.spim_context,
self.spim_section = make_data_section(
self.rom_start,
self.rom_end,
self.vram_start,
@ -100,12 +101,25 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
# Set rodata string encoding
# First check the global configuration
if options.opts.data_string_encoding is not None:
self.spim_section.get_section().stringEncoding = (
options.opts.data_string_encoding
)
# Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None:
self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start)
self.spim_section.set_comment_offset(self.rom_start)
rodata_encountered = False
for symbol in self.spim_section.symbolList:
for symbol in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym
)

17
tools/splat/segtypes/common/rodata.py
View File

@ -5,6 +5,8 @@ from util import log, options, symbols
from segtypes.common.data import CommonSegData
from disassembler_section import make_rodata_section
class CommonSegRodata(CommonSegData):
def get_linker_section(self) -> str:
@ -53,8 +55,7 @@ class CommonSegRodata(CommonSegData):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = spimdisasm.mips.sections.SectionRodata(
symbols.spim_context,
self.spim_section = make_rodata_section(
self.rom_start,
self.rom_end,
self.vram_start,
@ -64,21 +65,25 @@ class CommonSegRodata(CommonSegData):
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
# Set rodata string encoding
# First check the global configuration
if options.opts.string_encoding is not None:
self.spim_section.stringEncoding = options.opts.string_encoding
self.spim_section.get_section().stringEncoding = (
options.opts.string_encoding
)
# Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None:
self.spim_section.stringEncoding = self.str_encoding
self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start)
self.spim_section.set_comment_offset(self.rom_start)
possible_text_segments: Set[Segment] = set()
for symbol in self.spim_section.symbolList:
for symbol in self.spim_section.get_section().symbolList:
generated_symbol = symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym
)

5
tools/splat/segtypes/linker_entry.py
View File

@ -39,7 +39,10 @@ def path_to_object_path(path: Path) -> Path:
full_suffix = ".o"
else:
full_suffix = path.suffix + ".o"
return clean_up_path(options.opts.build_path / path.with_suffix(full_suffix))
if not str(path).startswith(str(options.opts.build_path)):
path = options.opts.build_path / path
return clean_up_path(path.with_suffix(full_suffix))
def write_file_if_different(path: Path, new_content: str):

4
tools/splat/split.py
View File

@ -19,7 +19,7 @@ from segtypes.linker_entry import (
from segtypes.segment import Segment
from util import log, options, palettes, symbols, relocs
VERSION = "0.14.0"
VERSION = "0.15.1"
parser = argparse.ArgumentParser(
description="Split a rom given a rom, a config, and output directory"
@ -349,7 +349,7 @@ def main(config_path, modes, verbose, use_cache=True, skip_version_check=False):
if (
options.opts.is_mode_active("ld") and options.opts.platform != "gc"
): # TODO move this to platform initialization when it gets implemented
# Calculate list of segments for which we need to find the largest so we can safely place the symbol after it
# Calculate list of segments for which we need to find the largest, so we can safely place the symbol after it
max_vram_end_syms: Dict[str, List[Segment]] = {}
for sym in symbols.appears_after_overlays_syms:
max_vram_end_syms[sym.name] = [

272
tools/splat/test.py
View File

@ -1,8 +1,9 @@
from spimdisasm.common import FileSectionType
from split import *
import unittest
import io
import filecmp
import pprint
from util import symbols, options
import spimdisasm
from segtypes.common.rodata import CommonSegRodata
@ -10,7 +11,6 @@ from segtypes.common.code import CommonSegCode
from segtypes.common.c import CommonSegC
from segtypes.common.bss import CommonSegBss
import difflib
from segtypes.common.group import CommonSegGroup
class Testing(unittest.TestCase):
@ -73,8 +73,6 @@ class Testing(unittest.TestCase):
# can't diff binary
if file[0] == ".splache":
continue
file1_lines = []
file2_lines = []
with open(f"{file[1]}/{file[0]}") as file1:
file1_lines = file1.readlines()
with open(f"{file[2]}/{file[0]}") as file2:
@ -126,12 +124,12 @@ class Symbols(unittest.TestCase):
disassembler_instance.create_disassembler_instance("n64")
# first char is uppercase
assert symbols.check_valid_type("Symbol") == True
assert symbols.check_valid_type("Symbol")
splat_sym_types = {"func", "jtbl", "jtbl_label", "label"}
for type in splat_sym_types:
assert symbols.check_valid_type(type) == True
assert symbols.check_valid_type(type)
spim_types = [
"char*",
@ -151,20 +149,15 @@ class Symbols(unittest.TestCase):
]
for type in spim_types:
assert symbols.check_valid_type(type) == True
assert symbols.check_valid_type(type)
def test_add_symbol_to_spim_segment(self):
context = None
vromStart = 0x0
vromEnd = 0x10
vramStart = 0x40000000 + 0x0
vramEnd = 0x40000000 + 0x10
segment = spimdisasm.common.SymbolsSegment(
context=context,
vromStart=vromStart,
vromEnd=vromEnd,
vramStart=vramStart,
vramEnd=vramEnd,
context=spimdisasm.common.Context(),
vromStart=0x0,
vromEnd=0x10,
vramStart=0x40000000 + 0x0,
vramEnd=0x40000000 + 0x10,
)
sym = symbols.Symbol(0x40000000)
sym.user_declared = False
@ -177,16 +170,15 @@ class Symbols(unittest.TestCase):
assert sym.defined == result.isDefined
def test_add_symbol_to_spim_section(self):
context = spimdisasm.common.Context()
section = spimdisasm.mips.sections.SectionBase(
context=context,
vromStart=0x100,
vromEnd=None,
vram=None,
filename=None,
words=None,
sectionType=None,
segmentVromStart=None,
context=spimdisasm.common.Context(),
vromStart=0x0,
vromEnd=0x10,
vram=0x40000000,
filename="test",
words=[],
sectionType=FileSectionType.Text,
segmentVromStart=0x0,
overlayCategory=None,
)
sym = symbols.Symbol(0x100)
@ -199,36 +191,28 @@ class Symbols(unittest.TestCase):
assert sym.defined == result.isDefined
def test_create_symbol_from_spim_symbol(self):
rom_start = 0x0
rom_end = 0x100
type = "func"
name = "MyFunc"
vram_start = 0x40000000
args = None
yaml = None
# need to init otherwise options.opts isn't defined.
# used in initializing a Segment
test_init()
segment = Segment(
rom_start=rom_start,
rom_end=rom_end,
type=type,
name=name,
vram_start=vram_start,
rom_start=0x0,
rom_end=0x100,
type="func",
name="MyFunc",
vram_start=0x40000000,
args=[],
yaml=yaml,
yaml=None,
)
context_sym = spimdisasm.common.ContextSymbol(address=0)
result = symbols.create_symbol_from_spim_symbol(segment, context_sym)
assert result.referenced == True
assert result.extract == True
assert result.referenced
assert result.extract
assert result.name == "D_0"
def get_yaml():
yaml = {
return {
"name": "basic_app",
"type": "code",
"start": 0,
@ -236,7 +220,6 @@ def get_yaml():
"subalign": 4,
"subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]],
}
return yaml
class Rodata(unittest.TestCase):
@ -252,11 +235,11 @@ class Rodata(unittest.TestCase):
yaml=None,
)
rom_data = []
for i in range(0, 0x100):
for i in range(0x100):
rom_data.append(i)
common_seg_rodata.disassemble_data(bytes(rom_data))
assert common_seg_rodata.spim_section is not None
assert common_seg_rodata.spim_section.words[0] == 0x0010203
assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203
assert symbols.get_all_symbols()[0].vram_start == 0x400
assert symbols.get_all_symbols()[0].segment == common_seg_rodata
assert symbols.get_all_symbols()[0].linker_section == ".rodata"
@ -270,11 +253,11 @@ class Rodata(unittest.TestCase):
rodata_sym = spimdisasm.mips.symbols.SymbolRodata(
context=context,
vromStart=0x100,
vromEnd=None,
inFileOffset=None,
vromEnd=0x200,
inFileOffset=0,
vram=0x100,
words=[0, 1, 2, 3, 4, 5, 6, 7],
segmentVromStart=None,
segmentVromStart=0,
overlayCategory=None,
)
rodata_sym.contextSym.forceMigration = True
@ -282,7 +265,7 @@ class Rodata(unittest.TestCase):
context_sym = spimdisasm.common.ContextSymbol(address=0)
context_sym.address = result_symbol_addr
rodata_sym.contextSym.referenceFunctions = [context_sym]
rodata_sym.contextSym.referenceFunctions = {context_sym}
# Segment __init__ requires opts to be initialized
test_init()
@ -340,9 +323,192 @@ class Bss(unittest.TestCase):
rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7])
bss.disassemble_data(rom_bytes)
assert isinstance(bss.spim_section, spimdisasm.mips.sections.SectionBss)
assert bss.spim_section.bssVramStart == 0x40000000
assert bss.spim_section.bssVramEnd == 0x300
assert bss.spim_section is not None
assert isinstance(
bss.spim_section.get_section(), spimdisasm.mips.sections.SectionBss
)
assert bss.spim_section.get_section().bssVramStart == 0x40000000
assert bss.spim_section.get_section().bssVramEnd == 0x300
class SymbolsInitialize(unittest.TestCase):
def test_attrs(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = [
"func_1 = 0x100 // type:func size:10 rom:100 segment:test_segment name_end:the_name_end "
"appears_after_overlays_addr:1234"
]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.all_symbols[0].given_name == "func_1"
assert symbols.all_symbols[0].type == "func"
assert symbols.all_symbols[0].given_size == 10
assert symbols.all_symbols[0].rom == 100
assert symbols.all_symbols[0].segment == all_segments[0]
assert symbols.all_symbols[0].given_name_end == "the_name_end"
assert symbols.appears_after_overlays_syms[0] == symbols.all_symbols[0]
def test_boolean_attrs(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = [
"func_1 = 0x100 // dead:True defined:True extract:True force_migration:True force_not_migration:True "
"allow_addend:True dont_allow_addend:True"
]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.all_symbols[0].dead == True
assert symbols.all_symbols[0].defined == True
assert symbols.all_symbols[0].force_migration == True
assert symbols.all_symbols[0].force_not_migration == True
assert symbols.all_symbols[0].allow_addend == True
assert symbols.all_symbols[0].dont_allow_addend == True
# test spim ban range
def test_ignore(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = ["func_1 = 0x100 // ignore:True size:4"]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.spim_context.bannedRangedSymbols[0].start == 16
assert symbols.spim_context.bannedRangedSymbols[0].end == 20
class InitializeSpimContext(unittest.TestCase):
def test_overlay(self):
symbols.reset_symbols()
test_init()
yaml = {
"name": "boot",
"type": "code",
"start": 4096,
"vram": 2147484672,
"bss_size": 128,
"exclusive_ram_id": "overlay",
"subsegments": [
[4096, "c", "main"],
[4336, "hasm", "handwritten"],
[4352, "data", "main"],
[4368, "rodata", "main"],
{"type": "bss", "vram": 2147484992, "name": "main"},
],
}
all_segments: List["Segment"] = [
CommonSegCode(
rom_start=0x0,
rom_end=0x200,
type="code",
name="main",
vram_start=0x100,
args=[],
yaml=yaml,
)
]
# force this since it's hard to set up
all_segments[0].exclusive_ram_id = "overlay"
symbols.initialize_spim_context(all_segments)
# spim should have added something to overlaySegments
assert (
type(symbols.spim_context.overlaySegments["overlay"][0])
== spimdisasm.common.SymbolsSegment
)
# test globalSegment settings
def test_global(self):
symbols.reset_symbols()
test_init()
yaml = {
"name": "boot",
"type": "code",
"start": 4096,
"vram": 2147484672,
"bss_size": 128,
"exclusive_ram_id": "overlay",
"subsegments": [
[4096, "c", "main"],
[4336, "hasm", "handwritten"],
[4352, "data", "main"],
[4368, "rodata", "main"],
{"type": "bss", "vram": 2147484992, "name": "main"},
],
}
all_segments: List["Segment"] = [
CommonSegCode(
rom_start=0x0,
rom_end=0x200,
type="code",
name="main",
vram_start=0x100,
args=[],
yaml=yaml,
)
]
assert symbols.spim_context.globalSegment.vramStart == 2147483648
assert symbols.spim_context.globalSegment.vramEnd == 2147487744
symbols.initialize_spim_context(all_segments)
assert symbols.spim_context.globalSegment.vramStart == 256
assert symbols.spim_context.globalSegment.vramEnd == 896
if __name__ == "__main__":

0
tools/splat/test/basic_app/build.sh Normal file → Executable file
View File

2
tools/splat/test/basic_app/expected/asm/data/main.data.s
View File

@ -4,8 +4,10 @@
glabel D_80000500
/* 1100 80000500 */ .word 0x00000001
.size D_80000500, . - D_80000500
glabel D_80000504
/* 1104 80000504 */ .word 0x00000000
/* 1108 80000508 */ .word 0x00000000
/* 110C 8000050C */ .word 0x00000000
.size D_80000504, . - D_80000504

1
tools/splat/test/basic_app/expected/asm/handwritten.s
View File

@ -13,3 +13,4 @@ glabel func_800004F0
/* 10F4 800004F4 03E00008 */ jr $ra
/* 10F8 800004F8 00000000 */ nop
/* 10FC 800004FC 00000000 */ nop
.size func_800004F0, . - func_800004F0

1
tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s
View File

@ -52,3 +52,4 @@ glabel .L80000480
/* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp)
/* 1098 80000498 03E00008 */ jr $ra
/* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8
.size func_80000400, . - func_80000400

1
tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s
View File

@ -23,3 +23,4 @@ glabel func_800004A0
/* 10E4 800004E4 03E00008 */ jr $ra
/* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18
/* 10EC 800004EC 00000000 */ nop
.size func_800004A0, . - func_800004A0

8
tools/splat/util/color.py
View File

@ -17,11 +17,3 @@ def unpack_color(data):
b = ceil(0xFF * (b / 31))
return r, g, b, a
def pack_color(r, g, b, a):
r = r >> 3
g = g >> 3
b = b >> 3
a = a >> 7
return (r << 11) | (g << 6) | (b << 1) | a

2
tools/splat/util/n64/rominfo.py
View File

@ -106,7 +106,7 @@ class N64EntrypointInfo:
register_values[insn.rt.value] = insn.getProcessedImmediate() << 16
elif insn.canBeLo():
if insn.isLikelyHandwritten():
# Try to skip this instructions:
# Try to skip these instructions:
# addi $t0, $t0, 0x8
# addi $t1, $t1, -0x8
pass

27
tools/splat/util/options.py
View File

@ -1,4 +1,5 @@
from dataclasses import dataclass
import os
from pathlib import Path
from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar
@ -98,7 +99,7 @@ class SplatOpts:
ld_section_labels: List[str]
# Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
ld_wildcard_sections: bool
# Determines whether to use use "follows" settings to determine locations of overlays in the linker script.
# Determines whether to use "follows" settings to determine locations of overlays in the linker script.
# If disabled, this effectively ignores "follows" directives in the yaml.
ld_use_follows: bool
# If enabled, the end symbol for each segment will be placed before the alignment directive for the segment
@ -141,6 +142,8 @@ class SplatOpts:
asm_data_macro: str
# Determines the macro used at the end of a function, such as endlabel or .end
asm_end_label: str
# Toggles the .size directive emitted by the disassembler
asm_emit_size_directive: Optional[bool]
# Determines including the macro.inc file on non-migrated rodata variables
include_macro_inc: bool
# Determines the number of characters to left align before the TODO finish documenting
@ -154,12 +157,18 @@ class SplatOpts:
# o32 is highly recommended, as it provides logically named registers for floating point instructions
# For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d
mips_abi_float_regs: str
# Determines whether to ad ".set gp=64 to asm/hasm files"
# Determines whether to add ".set gp=64" to asm/hasm files
add_set_gp_64: bool
# Generate .asmproc.d dependency files for each C file which still reference functions in assembly files
create_asm_dependencies: bool
# Global option for rodata string encoding. This can be overriden per segment
string_encoding: Optional[str]
# Global option for data string encoding. This can be overriden per segment
data_string_encoding: Optional[str]
# Global option for the rodata string guesser. 0 disables the guesser completely.
rodata_string_guesser_level: Optional[int]
# Global option for the data string guesser. 0 disables the guesser completely.
data_string_guesser_level: Optional[int]
# Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol
allow_data_addends: bool
# Determines whether to include the "Generated by spimdisasm" text in the asm
@ -240,7 +249,7 @@ class OptParser:
def parse_path(
self, base_path: Path, opt: str, default: Optional[str] = None
) -> Path:
return base_path / Path(self.parse_opt(opt, str, default))
return Path(os.path.normpath(base_path / self.parse_opt(opt, str, default)))
def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]:
if opt not in self._yaml:
@ -275,7 +284,9 @@ def _parse_yaml(
platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"])
comp = compiler.for_name(p.parse_opt("compiler", str, "IDO"))
base_path = Path(config_paths[0]).parent / p.parse_opt("base_path", str)
base_path = Path(
os.path.normpath(Path(config_paths[0]).parent / p.parse_opt("base_path", str))
)
asm_path: Path = p.parse_path(base_path, "asm_path", "asm")
def parse_endianness() -> Literal["big", "little"]:
@ -378,6 +389,7 @@ def _parse_yaml(
),
asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro),
asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label),
asm_emit_size_directive=p.parse_optional_opt("asm_emit_size_directive", bool),
include_macro_inc=p.parse_opt(
"include_macro_inc", bool, comp.include_macro_inc
),
@ -398,6 +410,13 @@ def _parse_yaml(
add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True),
create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False),
string_encoding=p.parse_optional_opt("string_encoding", str),
data_string_encoding=p.parse_optional_opt("data_string_encoding", str),
rodata_string_guesser_level=p.parse_optional_opt(
"rodata_string_guesser_level", int
),
data_string_guesser_level=p.parse_optional_opt(
"data_string_guesser_level", int
),
allow_data_addends=p.parse_opt("allow_data_addends", bool, True),
header_encoding=p.parse_opt("header_encoding", str, "ASCII"),
gfx_ucode=p.parse_opt_within(

363
tools/splat/util/symbols.py
View File

@ -6,6 +6,7 @@ import spimdisasm
import tqdm
from intervaltree import IntervalTree
from disassembler import disassembler_instance
from pathlib import Path
# circular import
if TYPE_CHECKING:
@ -71,6 +72,175 @@ def to_cname(symbol_name: str) -> str:
return symbol_name
def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
def get_seg_for_name(name: str) -> Optional["Segment"]:
for segment in all_segments:
if segment.name == name:
return segment
return None
for line_num, line in enumerate(
tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
):
line = line.strip()
if not line == "" and not line.startswith("//"):
comment_loc = line.find("//")
line_main = line
line_ext = ""
if comment_loc != -1:
line_ext = line[comment_loc + 2 :].strip()
line_main = line[:comment_loc].strip()
try:
line_split = line_main.split("=")
name = line_split[0].strip()
addr = int(line_split[1].strip()[:-1], 0)
except:
log.parsing_error_preamble(path, line_num, line)
log.write("Line should be of the form")
log.write("<function_name> = <address> // attr0:val0 attr1:val1 [...]")
log.write("with <address> in hex preceded by 0x, or dec")
log.write("")
raise
sym = Symbol(addr, given_name=name)
ignore_sym = False
if line_ext:
for info in line_ext.split(" "):
if ":" in info:
if info.count(":") > 1:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Too many ':'s in '{info}'")
log.error("")
attr_name, attr_val = info.split(":")
if attr_name == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute name in '{info}', is there extra whitespace?"
)
log.error("")
if attr_val == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute value in '{info}', is there extra whitespace?"
)
log.error("")
# Non-Boolean attributes
try:
if attr_name == "type":
if not check_valid_type(attr_val):
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Unrecognized symbol type in '{info}', it should be one of"
)
log.write(
[
*splat_sym_types,
*spimdisasm.common.gKnownTypes,
]
)
log.write(
"You may use a custom type that starts with a capital letter"
)
log.error("")
type = attr_val
sym.type = type
continue
if attr_name == "size":
size = int(attr_val, 0)
sym.given_size = size
continue
if attr_name == "rom":
rom_addr = int(attr_val, 0)
sym.rom = rom_addr
continue
if attr_name == "segment":
seg = get_seg_for_name(attr_val)
if seg is None:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Cannot find segment '{attr_val}'")
log.error("")
else:
# Add segment to symbol
sym.segment = seg
continue
if attr_name == "name_end":
sym.given_name_end = attr_val
continue
if attr_name == "appears_after_overlays_addr":
sym.appears_after_overlays_addr = int(attr_val, 0)
appears_after_overlays_syms.append(sym)
continue
except:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"value of attribute '{attr_name}' could not be read:"
)
log.write("")
raise
# Boolean attributes
tf_val = (
True
if is_truey(attr_val)
else False
if is_falsey(attr_val)
else None
)
if tf_val is None:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of"
)
log.write([*TRUEY_VALS, *FALSEY_VALS])
log.error("")
else:
if attr_name == "dead":
sym.dead = tf_val
continue
if attr_name == "defined":
sym.defined = tf_val
continue
if attr_name == "extract":
sym.extract = tf_val
continue
if attr_name == "ignore":
ignore_sym = tf_val
continue
if attr_name == "force_migration":
sym.force_migration = tf_val
continue
if attr_name == "force_not_migration":
sym.force_not_migration = tf_val
continue
if attr_name == "allow_addend":
sym.allow_addend = tf_val
continue
if attr_name == "dont_allow_addend":
sym.dont_allow_addend = tf_val
continue
if ignore_sym:
if sym.given_size is None or sym.given_size == 0:
ignored_addresses.add(sym.vram_start)
else:
spim_context.addBannedSymbolRangeBySize(
sym.vram_start, sym.given_size
)
continue
if sym.segment:
sym.segment.add_symbol(sym)
sym.user_declared = True
add_symbol(sym)
def initialize(all_segments: "List[Segment]"):
global all_symbols
global all_symbols_dict
@ -80,187 +250,12 @@ def initialize(all_segments: "List[Segment]"):
all_symbols_dict = {}
all_symbols_ranges = IntervalTree()
def get_seg_for_name(name: str) -> Optional["Segment"]:
for segment in all_segments:
if segment.name == name:
return segment
return None
# Manual list of func name / addrs
for path in options.opts.symbol_addrs_paths:
if path.exists():
with open(path) as f:
sym_addrs_lines = f.readlines()
for line_num, line in enumerate(
tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
):
line = line.strip()
if not line == "" and not line.startswith("//"):
comment_loc = line.find("//")
line_main = line
line_ext = ""
if comment_loc != -1:
line_ext = line[comment_loc + 2 :].strip()
line_main = line[:comment_loc].strip()
try:
line_split = line_main.split("=")
name = line_split[0].strip()
addr = int(line_split[1].strip()[:-1], 0)
except:
log.parsing_error_preamble(path, line_num, line)
log.write("Line should be of the form")
log.write(
"<function_name> = <address> // attr0:val0 attr1:val1 [...]"
)
log.write("with <address> in hex preceded by 0x, or dec")
log.write("")
raise
sym = Symbol(addr, given_name=name)
ignore_sym = False
if line_ext:
for info in line_ext.split(" "):
if ":" in info:
if info.count(":") > 1:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Too many ':'s in '{info}'")
log.error("")
attr_name, attr_val = info.split(":")
if attr_name == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute name in '{info}', is there extra whitespace?"
)
log.error("")
if attr_val == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute value in '{info}', is there extra whitespace?"
)
log.error("")
# Non-Boolean attributes
try:
if attr_name == "type":
if not check_valid_type(attr_val):
log.parsing_error_preamble(
path, line_num, line
)
log.write(
f"Unrecognized symbol type in '{info}', it should be one of"
)
log.write(
[
*splat_sym_types,
*spimdisasm.common.gKnownTypes,
]
)
log.write(
"You may use a custom type that starts with a capital letter"
)
log.error("")
type = attr_val
sym.type = type
continue
if attr_name == "size":
size = int(attr_val, 0)
sym.given_size = size
continue
if attr_name == "rom":
rom_addr = int(attr_val, 0)
sym.rom = rom_addr
continue
if attr_name == "segment":
seg = get_seg_for_name(attr_val)
if seg is None:
log.parsing_error_preamble(
path, line_num, line
)
log.write(
f"Cannot find segment '{attr_val}'"
)
log.error("")
else:
# Add segment to symbol
sym.segment = seg
continue
if attr_name == "name_end":
sym.given_name_end = attr_val
continue
if attr_name == "appears_after_overlays_addr":
sym.appears_after_overlays_addr = int(
attr_val, 0
)
appears_after_overlays_syms.append(sym)
continue
except:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"value of attribute '{attr_name}' could not be read:"
)
log.write("")
raise
# Boolean attributes
tf_val = (
True
if is_truey(attr_val)
else False
if is_falsey(attr_val)
else None
)
if tf_val is None:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of"
)
log.write([*TRUEY_VALS, *FALSEY_VALS])
log.error("")
else:
if attr_name == "dead":
sym.dead = tf_val
continue
if attr_name == "defined":
sym.defined = tf_val
continue
if attr_name == "extract":
sym.extract = tf_val
continue
if attr_name == "ignore":
ignore_sym = tf_val
continue
if attr_name == "force_migration":
sym.force_migration = tf_val
continue
if attr_name == "force_not_migration":
sym.force_not_migration = tf_val
continue
if attr_name == "allow_addend":
sym.allow_addend = tf_val
continue
if attr_name == "dont_allow_addend":
sym.dont_allow_addend = tf_val
continue
if ignore_sym:
if sym.given_size == None or sym.given_size == 0:
ignored_addresses.add(sym.vram_start)
else:
spim_context.addBannedSymbolRangeBySize(
sym.vram_start, sym.given_size
)
ignore_sym = False
continue
if sym.segment:
sym.segment.add_symbol(sym)
sym.user_declared = True
add_symbol(sym)
handle_sym_addrs(path, sym_addrs_lines, all_segments)
def initialize_spim_context(all_segments: "List[Segment]") -> None:
@ -292,6 +287,7 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None:
continue
ram_id = segment.get_exclusive_ram_id()
if ram_id is None:
if global_vram_start is None:
global_vram_start = segment.vram_start
@ -644,3 +640,18 @@ class Symbol:
def get_all_symbols():
global all_symbols
return all_symbols
def reset_symbols():
global all_symbols
global all_symbols_dict
global all_symbols_ranges
global ignored_addresses
global to_mark_as_defined
global appears_after_overlays_syms
all_symbols = []
all_symbols_dict = {}
all_symbols_ranges = IntervalTree()
ignored_addresses = set()
to_mark_as_defined = set()
appears_after_overlays_syms = []

18
tools/splat_ext/pm_map_data.py
View File

@ -1,9 +1,8 @@
from math import ceil
import os, sys
from pathlib import Path
from typing import List
from segtypes.n64.segment import N64Segment
from util.n64.Yay0decompress import Yay0Decompressor
from util.color import unpack_color
from segtypes.n64.palette import iter_in_groups
from util import options
import png # type: ignore
@ -30,6 +29,21 @@ def decode_null_terminated_ascii(data):
def parse_palette(data):
palette = []
# RRRRRGGG GGBBBBBA
def unpack_color(data):
s = int.from_bytes(data[0:2], byteorder="big")
r = (s >> 11) & 0x1F
g = (s >> 6) & 0x1F
b = (s >> 1) & 0x1F
a = (s & 1) * 0xFF
r = ceil(0xFF * (r / 31))
g = ceil(0xFF * (g / 31))
b = ceil(0xFF * (b / 31))
return r, g, b, a
for a, b in iter_in_groups(data, 2):
palette.append(unpack_color([a, b]))

183
tools/splat_ext/tex_archives.py
View File

@ -1,12 +1,11 @@
from dataclasses import dataclass
import os
from math import ceil
import struct
import json
from pathlib import Path
import png
import n64img.image
from util.color import unpack_color, pack_color
from segtypes.n64.palette import iter_in_groups
from sys import path
@ -28,6 +27,21 @@ def decode_null_terminated_ascii(data):
def parse_palette(data):
palette = []
# RRRRRGGG GGBBBBBA
def unpack_color(data):
s = int.from_bytes(data[0:2], byteorder="big")
r = (s >> 11) & 0x1F
g = (s >> 6) & 0x1F
b = (s >> 1) & 0x1F
a = (s & 1) * 0xFF
r = ceil(0xFF * (r / 31))
g = ceil(0xFF * (g / 31))
b = ceil(0xFF * (b / 31))
return r, g, b, a
for a, b in iter_in_groups(data, 2):
palette.append(unpack_color([a, b]))
@ -50,13 +64,13 @@ TILES_MIPMAPS = 1
TILES_SHARED_AUX = 2
TILES_INDEPENDENT_AUX = 3
aux_combine_modes = {
AUX_COMBINE_MODES = {
0x00: "None", # multiply main * prim, ignore aux
0x08: "Multiply", # multiply main * aux * prim
0x0D: "ModulateAlpha", # use prim color, but multiply alpha by the difference between main and aux red channels
0x10: "LerpMainAux", # use prim alpha to lerp between main and aux color, use main alpha
}
aux_combine_modes_inv = {v: k for k, v in aux_combine_modes.items()}
AUX_COMBINE_MODES_INV = {v: k for k, v in AUX_COMBINE_MODES.items()}
wrap_modes = {
0: "Repeat",
@ -363,7 +377,7 @@ class TexImage:
if self.filter_mode == 2:
out["filter"] = True
out["combine"] = aux_combine_modes.get(self.combine_mode)
out["combine"] = AUX_COMBINE_MODES.get(self.combine_mode)
if self.is_variant:
out["variant"] = True
@ -395,7 +409,15 @@ class TexImage:
return fmt_str, hwrap, vwrap
def get_img_file(self, fmt_str, img_file):
def get_img_file(self, fmt_str, img_file: str):
def pack_color(r, g, b, a):
r = r >> 3
g = g >> 3
b = b >> 3
a = a >> 7
return (r << 11) | (g << 6) | (b << 1) | a
(out_img, out_w, out_h) = Converter(
mode=fmt_str.lower(), infile=img_file, flip_y=True
).convert()
@ -414,132 +436,6 @@ class TexImage:
return (out_img, out_pal, out_w, out_h)
# read texture properties from dictionary and load images
def from_json(self, tex_path: Path, json_data):
self.img_name = json_data["name"]
if "ext" in json_data:
self.raw_ext = json_data["ext"]
else:
self.raw_ext = "tif"
# read data for main tile
main_data = json_data.get("main")
if main_data == None:
raise Exception(f"Texture {self.img_name} has no definition for 'main'")
(main_fmt_name, self.main_hwrap, self.main_vwrap) = self.read_json_img(
main_data, "main", self.img_name
)
(self.main_fmt, self.main_depth) = get_format_code(main_fmt_name)
# read main image
img_path = str(tex_path / f"{self.img_name}.png")
if not os.path.isfile(img_path):
raise Exception(f"Could not find main image for texture: {self.img_name}")
(
self.main_img,
self.main_pal,
self.main_width,
self.main_height,
) = self.get_img_file(main_fmt_name, img_path)
# read data for aux tile
self.has_aux = "aux" in json_data
if self.has_aux:
aux_data = json_data.get("aux")
(aux_fmt_name, self.aux_hwrap, self.aux_vwrap) = self.read_json_img(
aux_data, "aux", self.img_name
)
if aux_fmt_name == "Shared":
# aux tiles have blank attributes in SHARED mode
aux_fmt_name = main_fmt_name
self.aux_fmt = 0
self.aux_depth = 0
self.aux_hwrap = 0
self.aux_vwrap = 0
self.extra_tiles = TILES_SHARED_AUX
else:
(self.aux_fmt, self.aux_depth) = get_format_code(aux_fmt_name)
self.extra_tiles = TILES_INDEPENDENT_AUX
# read aux image
img_path = str(tex_path / f"{self.img_name}_AUX.png")
if not os.path.isfile(img_path):
raise Exception(
f"Could not find AUX image for texture: {self.img_name}"
)
(
self.aux_img,
self.aux_pal,
self.aux_width,
self.aux_height,
) = self.get_img_file(aux_fmt_name, img_path)
if self.extra_tiles == TILES_SHARED_AUX:
# aux tiles have blank sizes in SHARED mode
self.main_height *= 2
self.aux_width = 0
self.aux_height = 0
else:
self.aux_fmt = 0
self.aux_depth = 0
self.aux_hwrap = 0
self.aux_vwrap = 0
self.aux_width = 0
self.aux_height = 0
self.extra_tiles = TILES_BASIC
# read mipmaps
self.has_mipmaps = json_data.get("hasMipmaps", False)
if self.has_mipmaps:
self.mipmaps = []
mipmap_idx = 1
divisor = 2
if self.main_width >= (32 >> self.main_depth):
while True:
if (self.main_width // divisor) <= 0:
break
mmw = self.main_width // divisor
mmh = self.main_height // divisor
img_path = str(tex_path / f"{self.img_name}_MM{mipmap_idx}.png")
if not os.path.isfile(img_path):
raise Exception(
f"Texture {self.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})"
)
(raster, pal, width, height) = self.get_img_file(
main_fmt_name, img_path
)
self.mipmaps.append(raster)
if width != mmw or height != mmh:
raise Exception(
f"Texture {self.img_name} has wrong size for mipmap level {mipmap_idx} \n"
+ f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}"
)
divisor = divisor * 2
mipmap_idx += 1
if (self.main_width // divisor) < (16 >> self.main_depth):
break
self.extra_tiles = TILES_MIPMAPS
# read filter mode
if json_data.get("filter", False):
self.filter_mode = 2
else:
self.filter_mode = 0
# read tile combine mode
combine_str = json_data.get("combine", "Missing")
self.combine = aux_combine_modes_inv.get(combine_str)
if self.combine == None:
raise Exception(f"Texture {self.img_name} has invalid 'combine'")
self.is_variant = json_data.get("variant", False)
# write texture header and image raster/palettes to byte array
def add_bytes(self, tex_name: str, bytes: bytearray):
# form raw name and write to header
@ -617,26 +513,3 @@ class TexArchive:
json_fn = str(tex_path) + ".json"
with open(json_fn, "w") as f:
f.write(json_out)
@staticmethod
def build(out_path: Path, tex_path: Path, endian: str = "big"):
out_bytes = bytearray()
tex_name = os.path.basename(tex_path)
json_fn = str(tex_path) + ".json"
with open(json_fn, "r") as json_file:
json_str = json_file.read()
json_data = json.loads(json_str)
if len(json_data) > 128:
raise Exception(
f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`"
)
for img_data in json_data:
img = TexImage()
img.from_json(tex_path, img_data)
img.add_bytes(tex_name, out_bytes)
with open(out_path, "wb") as out_bin:
out_bin.write(out_bytes)