diff --git a/common/util/BitUtils.h b/common/util/BitUtils.h index f6fec2790..189917aff 100644 --- a/common/util/BitUtils.h +++ b/common/util/BitUtils.h @@ -66,6 +66,16 @@ std::optional get_power_of_two(T in) { bool integer_fits(s64 in, int size, bool is_signed); u32 float_as_u32(float x); +template +T align64(T in) { + return (in + 63) & (~T(63)); +} + +template +T align32(T in) { + return (in + 31) & (~T(31)); +} + template T align16(T in) { return (in + 15) & (~T(15)); @@ -82,8 +92,8 @@ T align4(T in) { } template -T align64(T in) { - return (in + 63) & (~T(63)); +T align2(T in) { + return (in + 1) & (~T(1)); } inline u32 count_leading_zeros_u32(u32 in) { diff --git a/common/util/FontUtils.cpp b/common/util/FontUtils.cpp index 8e53bbfdb..03d832cf8 100644 --- a/common/util/FontUtils.cpp +++ b/common/util/FontUtils.cpp @@ -1017,8 +1017,8 @@ static std::vector s_replace_info_jak2 = { ""}, {"~Y~1L~Z~39L~~-1H~Y~1L~Z~39L~Z~-11H~7L~Z~-11H~3L<" "SYM43>~Z~+26H", - ""}, - {"~Y~1L~~-1H~Y~1L~Z~-11H~3L~Z~+26H", ""}, + ""}, + {"~Y~1L~~-1H~Y~1L~Z~-11H~3L~Z~+26H", ""}, // weird stuff // - descenders @@ -1028,7 +1028,7 @@ static std::vector s_replace_info_jak2 = { {"~+7Vq~-7V", "q"}, {"~+1Vj~-1V", "j"}, - {"\\\\\\\\", "\\n"}, + {"\\\\\\\\", "\\n"}, // wtf happened here!? // - symbols and ligatures {"~-4H~-3V\\c19~+3V~-4H", @@ -1051,9 +1051,9 @@ static std::vector s_encode_info_jak2 = { {"º", {0x16}}, // numero/overring {"¡", {0x17}}, // inverted exclamation mark {"¿", {0x18}}, // inverted question mark - {"Ç", {0x1d}}, // c-cedilla - - {"ß", {0x1f}}, // eszett + {"ç", {0x1d}}, // c-cedilla + {"Ç", {0x1e}}, // c-cedilla + {"ß", {0x1f}}, // eszett {"œ", {0x5e}}, // ligature o+e // Re-purposed japanese/korean symbols that are used as part of drawing icons/flags/pad buttons @@ -1104,11 +1104,373 @@ static std::vector s_encode_info_jak2 = { {"", {0xb1}}, {"", {0xb3}}, {"", {0xb2}}, + // {"入", {1, 0x00}}, + // {"年", {1, 0x01}}, + // punctuation + {"・", {1, 0x10}}, + {"゛", {1, 0x11}}, + {"゜", {1, 0x12}}, + {"ー", {1, 0x13}}, + {"『", {1, 0x14}}, + {"』", {1, 0x15}}, + // hiragana + {"ぁ", {1, 0x16}}, // -a + {"あ", {1, 0x17}}, // a + {"ぃ", {1, 0x18}}, // -i + {"い", {1, 0x19}}, // i + {"ぅ", {1, 0x1a}}, // -u + {"う", {1, 0x1b}}, // u + {"ぇ", {1, 0x1c}}, // -e + {"え", {1, 0x1d}}, // e + {"ぉ", {1, 0x1e}}, // -o + {"お", {1, 0x1f}}, // o + {"か", {1, 0x20}}, // ka + {"き", {1, 0x21}}, // ki + {"く", {1, 0x22}}, // ku + {"け", {1, 0x23}}, // ke + {"こ", {1, 0x24}}, // ko + {"さ", {1, 0x25}}, // sa + {"し", {1, 0x26}}, // shi + {"す", {1, 0x27}}, // su + {"せ", {1, 0x28}}, // se + {"そ", {1, 0x29}}, // so + {"た", {1, 0x2a}}, // ta + {"ち", {1, 0x2b}}, // chi + {"っ", {1, 0x2c}}, // sokuon + {"つ", {1, 0x2d}}, // tsu + {"て", {1, 0x2e}}, // te + {"と", {1, 0x2f}}, // to + {"な", {1, 0x30}}, // na + {"に", {1, 0x31}}, // ni + {"ぬ", {1, 0x32}}, // nu + {"ね", {1, 0x33}}, // ne + {"の", {1, 0x34}}, // no + {"は", {1, 0x35}}, // ha + {"ひ", {1, 0x36}}, // hi + {"ふ", {1, 0x37}}, // fu + {"へ", {1, 0x38}}, // he + {"ほ", {1, 0x39}}, // ho + {"ま", {1, 0x3a}}, // ma + {"み", {1, 0x3b}}, // mi + {"む", {1, 0x3c}}, // mu + {"め", {1, 0x3d}}, // me + {"も", {1, 0x3e}}, // mo + {"ゃ", {1, 0x3f}}, // youon ya + {"や", {1, 0x40}}, // ya + {"ゅ", {1, 0x41}}, // youon yu + {"ゆ", {1, 0x42}}, // yu + {"ょ", {1, 0x43}}, // youon yo + {"よ", {1, 0x44}}, // yo + {"ら", {1, 0x45}}, // ra + {"り", {1, 0x46}}, // ri + {"る", {1, 0x47}}, // ru + {"れ", {1, 0x48}}, // re + {"ろ", {1, 0x49}}, // ro + {"ゎ", {1, 0x4a}}, // -wa + {"わ", {1, 0x4b}}, // wa + {"を", {1, 0x4c}}, // wo + {"ん", {1, 0x4d}}, // -n + // katakana + {"ァ", {1, 0x4e}}, // -a + {"ア", {1, 0x4f}}, // a + {"ィ", {1, 0x50}}, // -i + {"イ", {1, 0x51}}, // i + {"ゥ", {1, 0x52}}, // -u + {"ウ", {1, 0x53}}, // u + {"ェ", {1, 0x54}}, // -e + {"エ", {1, 0x55}}, // e + {"ォ", {1, 0x56}}, // -o + {"オ", {1, 0x57}}, // o + {"カ", {1, 0x58}}, // ka + {"キ", {1, 0x59}}, // ki + {"ク", {1, 0x5a}}, // ku + {"ケ", {1, 0x5b}}, // ke + {"コ", {1, 0x5c}}, // ko + {"サ", {1, 0x5d}}, // sa + {"シ", {1, 0x5e}}, // shi + {"ス", {1, 0x5f}}, // su + {"セ", {1, 0x60}}, // se + {"ソ", {1, 0x61}}, // so + {"タ", {1, 0x62}}, // ta + {"チ", {1, 0x63}}, // chi + {"ッ", {1, 0x64}}, // sokuon + {"ツ", {1, 0x65}}, // tsu + {"テ", {1, 0x66}}, // te + {"ト", {1, 0x67}}, // to + {"ナ", {1, 0x68}}, // na + {"ニ", {1, 0x69}}, // ni + {"ヌ", {1, 0x6a}}, // nu + {"ネ", {1, 0x6b}}, // ne + {"ノ", {1, 0x6c}}, // no + {"ハ", {1, 0x6d}}, // ha + {"ヒ", {1, 0x6e}}, // hi + {"フ", {1, 0x6f}}, // fu + {"ヘ", {1, 0x70}}, // he + {"ホ", {1, 0x71}}, // ho + {"マ", {1, 0x72}}, // ma + {"ミ", {1, 0x73}}, // mi + {"ム", {1, 0x74}}, // mu + {"メ", {1, 0x75}}, // me + {"モ", {1, 0x76}}, // mo + {"ャ", {1, 0x77}}, // youon ya + {"ヤ", {1, 0x78}}, // ya + {"ュ", {1, 0x79}}, // youon yu + {"ユ", {1, 0x7a}}, // yu + {"ョ", {1, 0x7b}}, // youon yo + {"ヨ", {1, 0x7c}}, // yo + {"ラ", {1, 0x7d}}, // ra + {"リ", {1, 0x7e}}, // ri + {"ル", {1, 0x7f}}, // ru + {"レ", {1, 0x80}}, // re + {"ロ", {1, 0x81}}, // ro + {"ヮ", {1, 0x82}}, // -wa + {"ワ", {1, 0x83}}, // wa + {"ヲ", {1, 0x84}}, // wo + {"ン", {1, 0x85}}, // -n + + {"位", {1, 0x8c}}, + {"遺", {1, 0x8d}}, + {"院", {1, 0x8e}}, + {"映", {1, 0x8f}}, + {"衛", {1, 0x90}}, + {"応", {1, 0x91}}, + {"下", {1, 0x92}}, + {"画", {1, 0x93}}, + {"解", {1, 0x94}}, + {"開", {1, 0x95}}, + {"外", {1, 0x96}}, + {"害", {1, 0x97}}, + {"蓋", {1, 0x98}}, + {"完", {1, 0x99}}, + {"換", {1, 0x9a}}, + {"監", {1, 0x9b}}, + {"間", {1, 0x9c}}, + {"器", {1, 0x9d}}, + {"記", {1, 0x9e}}, + {"逆", {1, 0x9f}}, + {"救", {1, 0xa0}}, + {"金", {1, 0xa1}}, + {"空", {1, 0xa2}}, + {"掘", {1, 0xa3}}, + {"警", {1, 0xa4}}, + {"迎", {1, 0xa5}}, + {"撃", {1, 0xa6}}, + {"建", {1, 0xa7}}, + {"源", {1, 0xa8}}, + {"現", {1, 0xa9}}, + {"言", {1, 0xaa}}, + {"限", {1, 0xab}}, + {"個", {1, 0xac}}, + {"庫", {1, 0xad}}, + {"後", {1, 0xae}}, + {"語", {1, 0xaf}}, + {"護", {1, 0xb0}}, + {"交", {1, 0xb1}}, + {"功", {1, 0xb2}}, + {"向", {1, 0xb3}}, + {"工", {1, 0xb4}}, + {"攻", {1, 0xb5}}, + {"溝", {1, 0xb6}}, + {"行", {1, 0xb7}}, + {"鉱", {1, 0xb8}}, + {"降", {1, 0xb9}}, + {"合", {1, 0xba}}, + {"告", {1, 0xbb}}, + {"獄", {1, 0xbc}}, + {"彩", {1, 0xbd}}, + {"作", {1, 0xbe}}, + {"山", {1, 0xbf}}, + {"使", {1, 0xc0}}, + {"始", {1, 0xc1}}, + {"試", {1, 0xc2}}, + {"字", {1, 0xc3}}, + {"寺", {1, 0xc4}}, + {"時", {1, 0xc5}}, + {"示", {1, 0xc6}}, + {"自", {1, 0xc7}}, + {"式", {1, 0xc8}}, + {"矢", {1, 0xc9}}, + {"射", {1, 0xca}}, + {"者", {1, 0xcb}}, + {"守", {1, 0xcc}}, + {"手", {1, 0xcd}}, + {"終", {1, 0xce}}, + {"週", {1, 0xcf}}, + {"出", {1, 0xd0}}, + {"所", {1, 0xd1}}, + {"書", {1, 0xd2}}, + {"勝", {1, 0xd3}}, + {"章", {1, 0xd4}}, + {"上", {1, 0xd5}}, + {"乗", {1, 0xd6}}, + {"場", {1, 0xd7}}, + {"森", {1, 0xd8}}, + {"進", {1, 0xd9}}, + {"人", {1, 0xda}}, + {"水", {1, 0xdb}}, + {"数", {1, 0xdc}}, + {"制", {1, 0xdd}}, + {"性", {1, 0xde}}, + {"成", {1, 0xdf}}, + {"聖", {1, 0xe0}}, + {"石", {1, 0xe1}}, + {"跡", {1, 0xe2}}, + {"先", {1, 0xe3}}, + {"戦", {1, 0xe4}}, + {"船", {1, 0xe5}}, + {"選", {1, 0xe6}}, + {"走", {1, 0xe7}}, + {"送", {1, 0xe8}}, + {"像", {1, 0xe9}}, + {"造", {1, 0xea}}, + {"続", {1, 0xeb}}, + {"対", {1, 0xec}}, + {"袋", {1, 0xed}}, + {"台", {1, 0xee}}, + {"弾", {1, 0xef}}, + {"地", {1, 0xf0}}, + {"中", {1, 0xf1}}, + {"敵", {1, 0xf2}}, + {"転", {1, 0xf3}}, + {"電", {1, 0xf4}}, + {"塔", {1, 0xf5}}, + {"頭", {1, 0xf6}}, + {"動", {1, 0xf7}}, + {"内", {1, 0xf8}}, + {"日", {1, 0xf9}}, + {"入", {1, 0xfa}}, + {"年", {1, 0xfb}}, + {"能", {1, 0xfc}}, + {"廃", {1, 0xfd}}, + {"排", {1, 0xfe}}, + {"敗", {1, 0xff}}, + + {"発", {2, 0x10}}, + {"反", {2, 0x11}}, + {"必", {2, 0x12}}, + {"表", {2, 0x13}}, + {"武", {2, 0x14}}, + {"壁", {2, 0x15}}, + {"墓", {2, 0x16}}, + {"放", {2, 0x17}}, + {"方", {2, 0x18}}, + {"砲", {2, 0x19}}, + {"妨", {2, 0x1a}}, + {"北", {2, 0x1b}}, + {"本", {2, 0x1c}}, + {"幕", {2, 0x1d}}, + {"無", {2, 0x1e}}, + {"迷", {2, 0x1f}}, + {"面", {2, 0x20}}, + {"戻", {2, 0x21}}, + {"紋", {2, 0x22}}, + {"薬", {2, 0x23}}, + {"輸", {2, 0x24}}, + {"勇", {2, 0x25}}, + {"友", {2, 0x26}}, + {"遊", {2, 0x27}}, + {"容", {2, 0x28}}, + {"要", {2, 0x29}}, + {"利", {2, 0x2a}}, + {"了", {2, 0x2b}}, + {"量", {2, 0x2c}}, + {"力", {2, 0x2d}}, + {"練", {2, 0x2e}}, + {"連", {2, 0x2f}}, + {"録", {2, 0x30}}, + {"話", {2, 0x31}}, + {"墟", {2, 0x32}}, + {"脱", {2, 0x33}}, + // {"成", {2, 0x34}}, + {"旗", {2, 0x35}}, + {"破", {2, 0x36}}, + {"壊", {2, 0x37}}, + {"全", {2, 0x38}}, + {"滅", {2, 0x39}}, + {"機", {2, 0x3a}}, + {"仲", {2, 0x3b}}, + {"渓", {2, 0x3c}}, + {"谷", {2, 0x3d}}, + {"優", {2, 0x3e}}, + {"探", {2, 0x3f}}, + {"部", {2, 0x40}}, + {"索", {2, 0x41}}, + // {"乗", {2, 0x42}}, + {"前", {2, 0x43}}, + {"右", {2, 0x44}}, + {"左", {2, 0x45}}, + {"会", {2, 0x46}}, + {"高", {2, 0x47}}, + {"低", {2, 0x48}}, + {"押", {2, 0x49}}, + {"切", {2, 0x4a}}, + {"替", {2, 0x4b}}, + // {"対", {2, 0x4c}}, + {"秒", {2, 0x4d}}, + {"箱", {2, 0x4e}}, + {"泳", {2, 0x4f}}, + {"~", {2, 0x50}}, + + {"闇", {2, 0x56}}, + {"以", {2, 0x57}}, + {"屋", {2, 0x58}}, + {"俺", {2, 0x59}}, + {"化", {2, 0x5a}}, + {"界", {2, 0x5b}}, + {"感", {2, 0x5c}}, + {"気", {2, 0x5d}}, + {"却", {2, 0x5e}}, + {"曲", {2, 0x5f}}, + {"継", {2, 0x60}}, + {"権", {2, 0x61}}, + {"見", {2, 0x62}}, + {"古", {2, 0x63}}, + {"好", {2, 0x64}}, + // {"高", {2, 0x65}}, + {"才", {2, 0x66}}, + {"士", {2, 0x67}}, + {"子", {2, 0x68}}, + {"次", {2, 0x69}}, + {"主", {2, 0x6a}}, + {"種", {2, 0x6b}}, + {"讐", {2, 0x6c}}, + {"女", {2, 0x6d}}, + {"小", {2, 0x6e}}, + {"焼", {2, 0x6f}}, + {"証", {2, 0x70}}, + {"神", {2, 0x71}}, + {"身", {2, 0x72}}, + {"寸", {2, 0x73}}, + {"世", {2, 0x74}}, + {"想", {2, 0x75}}, + {"退", {2, 0x76}}, + {"第", {2, 0x77}}, + {"着", {2, 0x78}}, + {"天", {2, 0x79}}, + {"倒", {2, 0x7a}}, + {"到", {2, 0x7b}}, + {"突", {2, 0x7c}}, + {"爆", {2, 0x7d}}, + {"番", {2, 0x7e}}, + {"負", {2, 0x7f}}, + {"復", {2, 0x80}}, + {"物", {2, 0x81}}, + {"眠", {2, 0x82}}, + {"予", {2, 0x83}}, + {"用", {2, 0x84}}, + {"落", {2, 0x85}}, + // {"録", {2, 0x86}}, + + {"封", {2, 0x88}}, + {"印", {2, 0x89}}, + {"扉", {2, 0x8a}}, + {"最", {2, 0x8b}}, + {"刻", {2, 0x8c}}, + {"足", {2, 0x8d}}, }; GameTextFontBank g_font_bank_jak2(GameTextVersion::JAK2, &s_encode_info_jak2, - //&s_replace_info_null, &s_replace_info_jak2, &s_passthrus_jak2); diff --git a/decompiler/ObjectFile/LinkedObjectFile.cpp b/decompiler/ObjectFile/LinkedObjectFile.cpp index 417b553f1..4fd6083f0 100644 --- a/decompiler/ObjectFile/LinkedObjectFile.cpp +++ b/decompiler/ObjectFile/LinkedObjectFile.cpp @@ -276,6 +276,10 @@ std::string LinkedObjectFile::print_words() { auto& word = words_by_seg[seg][i]; append_word_to_string(result, word); + + if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "string") { + result += "; " + get_goal_string(seg, i) + "\n"; + } } } diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index 2618e70f0..2332feade 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -29,6 +29,7 @@ #include "decompiler/data/StrFileReader.h" #include "decompiler/data/dir_tpages.h" #include "decompiler/data/game_count.h" +#include "decompiler/data/game_subs.h" #include "decompiler/data/game_text.h" #include "decompiler/data/tpage.h" @@ -204,18 +205,20 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO", config); } - lg::info("-Loading {} streaming object files...", str_files.size()); - for (auto& obj : str_files) { - StrFileReader reader(obj); - // name from the file name - std::string base_name = obj_filename_to_name(obj.string()); - // name from inside the file (this does a lot of sanity checking) - auto obj_name = reader.get_full_name(base_name + ".STR"); - for (int i = 0; i < reader.chunk_count(); i++) { - // append the chunk ID to the full name - std::string name = obj_name + fmt::format("+{}", i); - auto& data = reader.get_chunk(i); - add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO", config); + if (config.read_spools) { + lg::info("-Loading {} streaming object files...", str_files.size()); + for (auto& obj : str_files) { + StrFileReader reader(obj, version()); + // name from the file name + std::string base_name = obj_filename_to_name(obj.string()); + // name from inside the file (this does a lot of sanity checking) + auto obj_name = reader.get_full_name(base_name + ".STR"); + for (int i = 0; i < reader.chunk_count(); i++) { + // append the chunk ID to the full name + std::string name = obj_name + fmt::format("+{}", i); + auto& data = reader.get_chunk(i); + add_obj_from_dgo(name, name, data.data(), data.size(), "ALLSPOOL", config, obj_name); + } } } @@ -369,7 +372,8 @@ void ObjectFileDB::add_obj_from_dgo(const std::string& obj_name, const uint8_t* obj_data, uint32_t obj_size, const std::string& dgo_name, - const Config& config) { + const Config& config, + const std::string& cut_name) { if (config.banned_objects.find(obj_name) != config.banned_objects.end()) { return; } @@ -415,6 +419,7 @@ void ObjectFileDB::add_obj_from_dgo(const std::string& obj_name, // if this is the first time we've seen this object file name, add it in the order. obj_file_order.push_back(obj_name); } + data.base_name_from_chunk = cut_name; data.record.version = obj_files_by_name[obj_name].size(); data.name_in_dgo = name_in_dgo; data.obj_version = version; @@ -489,9 +494,10 @@ std::string ObjectFileDB::generate_obj_listing(const std::unordered_set= 5); - std::string new_str = y == "NO-XGO" ? y : y.substr(0, y.length() - 4); + for (auto& name : x.dgo_names) { + ASSERT(name.length() >= 5); + std::string new_str = + (name == "NO-XGO" || name == "ALLSPOOL") ? name : name.substr(0, name.length() - 4); dgos += "\"" + new_str + "\", "; } dgos.pop_back(); @@ -725,6 +731,58 @@ std::string ObjectFileDB::process_tpages(TextureDB& tex_db, const fs::path& outp return result; } +std::string ObjectFileDB::process_all_spool_subtitles(const Config& cfg, + const fs::path& image_out) { + try { + lg::info("- Finding spool subtitles..."); + Timer timer; + int obj_count = 0; + int string_count = 0; + int image_count = 0; + int subs_count = 0; + std::unordered_map> all_subs; + + for_each_obj_in_dgo("ALLSPOOL", [&](ObjectFileData& data) { + int this_string_count = 0; + int this_image_count = 0; + obj_count++; + auto this_spool_subs = process_spool_subtitles(data, cfg.text_version); + if (!this_spool_subs.empty()) { + for (auto& s : this_spool_subs) { + subs_count++; + for (int i = 0; i < 8; ++i) { + if (s.message[i].kind == SpoolSubtitleMessage::Kind::IMAGE) { + this_image_count++; + } else if (s.message[i].kind == SpoolSubtitleMessage::Kind::STRING) { + this_string_count++; + } + } + } + auto& spool_subs = all_subs[data.base_name_from_chunk]; + for (auto& x : this_spool_subs) { + bool skip = false; + for (auto& other : spool_subs) { + skip |= other == x; + } + if (!skip) { + all_subs[data.base_name_from_chunk].push_back(x); + image_count += this_image_count; + string_count += this_string_count; + } + } + } + }); + + lg::info("Processed {} subtitles in {} spool objects ({} strings, {} images) in {:.2f} ms", + subs_count, obj_count, string_count, image_count, timer.getMs()); + + return write_spool_subtitles(cfg.text_version, image_out, all_subs); + } catch (std::runtime_error& e) { + lg::warn("Error when extracting spool subtitles: {}", e.what()); + return {}; + } +} + std::string ObjectFileDB::process_game_text_files(const Config& cfg) { try { lg::info("- Finding game text..."); diff --git a/decompiler/ObjectFile/ObjectFileDB.h b/decompiler/ObjectFile/ObjectFileDB.h index 767c5361c..76769f8f5 100644 --- a/decompiler/ObjectFile/ObjectFileDB.h +++ b/decompiler/ObjectFile/ObjectFileDB.h @@ -47,6 +47,7 @@ struct ObjectFileData { std::string name_in_dgo; std::string name_from_map; std::string to_unique_name() const; + std::string base_name_from_chunk; uint32_t reference_count = 0; // number of times its used. std::string full_output; @@ -246,6 +247,7 @@ class ObjectFileDB { std::string process_tpages(TextureDB& tex_db, const fs::path& output_path); std::string process_game_count_file(); std::string process_game_text_files(const Config& cfg); + std::string process_all_spool_subtitles(const Config& cfg, const fs::path& image_out); const ObjectFileData& lookup_record(const ObjectFileRecord& rec) const; DecompilerTypeSystem dts; @@ -255,7 +257,6 @@ class ObjectFileDB { const Config& config, TypeSpec* result); - public: void load_map_file(const std::string& map_data); void get_objs_from_dgo(const fs::path& filename, const Config& config); void add_obj_from_dgo(const std::string& obj_name, @@ -263,7 +264,8 @@ class ObjectFileDB { const uint8_t* obj_data, uint32_t obj_size, const std::string& dgo_name, - const Config& config); + const Config& config, + const std::string& cut_name = ""); /*! * Apply f to all ObjectFileData's. Does it in the right order. @@ -273,7 +275,20 @@ class ObjectFileDB { ASSERT(obj_files_by_name.size() == obj_file_order.size()); for (const auto& name : obj_file_order) { for (auto& obj : obj_files_by_name.at(name)) { - // lg::info("{}...", name); + f(obj); + } + } + } + + /*! + * Apply f to all ObjectFileData's in a specific DGO. Does it in the right order. + */ + template + void for_each_obj_in_dgo(const std::string& dgo_name, Func f) { + ASSERT(obj_files_by_name.size() == obj_file_order.size()); + const auto& dgo_objs = obj_files_by_dgo.at(dgo_name); + for (const auto& rec : dgo_objs) { + for (auto& obj : obj_files_by_name.at(rec.name)) { f(obj); } } diff --git a/decompiler/config.cpp b/decompiler/config.cpp index 90f308f6b..8ed3eee26 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -62,6 +62,12 @@ Config make_config_via_json(nlohmann::json& json) { config.process_game_text = json.at("process_game_text").get(); config.process_game_count = json.at("process_game_count").get(); config.process_art_groups = json.at("process_art_groups").get(); + if (json.contains("process_subtitle_text")) { + config.process_subtitle_text = json.at("process_subtitle_text").get(); + } + if (json.contains("process_subtitle_images")) { + config.process_subtitle_images = json.at("process_subtitle_images").get(); + } config.dump_art_group_info = json.at("dump_art_group_info").get(); config.hexdump_code = json.at("hexdump_code").get(); config.hexdump_data = json.at("hexdump_data").get(); @@ -73,6 +79,9 @@ Config make_config_via_json(nlohmann::json& json) { config.rip_levels = json.at("rip_levels").get(); config.extract_collision = json.at("extract_collision").get(); config.generate_all_types = json.at("generate_all_types").get(); + if (json.contains("read_spools")) { + config.read_spools = json.at("read_spools").get(); + } if (json.contains("old_all_types_file")) { config.old_all_types_file = json.at("old_all_types_file").get(); } @@ -304,9 +313,11 @@ Config read_config_file(const fs::path& path_to_config_file, } // Then, update any config overrides - lg::info("Config Overide: '{}'", override_json); - auto cfg_override = parse_commented_json(override_json, ""); - json.update(cfg_override); + if (override_json != "{}" && !override_json.empty()) { + lg::info("Config Overide: '{}'", override_json); + auto cfg_override = parse_commented_json(override_json, ""); + json.update(cfg_override); + } // debugging, dump the JSON config to a file // fs::path debug_path = path_to_config_file.parent_path() / "config-debug.jsonc"; diff --git a/decompiler/config.h b/decompiler/config.h index 3f4406b73..b7f302115 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -112,10 +112,13 @@ struct Config { bool process_game_text = false; bool process_game_count = false; bool process_art_groups = false; + bool process_subtitle_text = false; + bool process_subtitle_images = false; bool dump_art_group_info = false; bool rip_levels = false; bool extract_collision = false; bool find_functions = false; + bool read_spools = false; bool write_hex_near_instructions = false; bool hexdump_code = false; diff --git a/decompiler/config/jak2/all-types.gc b/decompiler/config/jak2/all-types.gc index 0fc3bcd13..879a66def 100644 --- a/decompiler/config/jak2/all-types.gc +++ b/decompiler/config/jak2/all-types.gc @@ -3094,7 +3094,7 @@ (tex-all-map 319) ;; tex (progress 320) ;; hud | progress (screen-filter 321) ;; hud letterbox, no zbuf - (bucket-322 322) ;; hud + (subtitle 322) ;; hud (bucket-323 323) ;; hud (debug2 324) ;; debug (debug-no-zbuf2 325) ;; debug diff --git a/decompiler/config/jak2/jak2_config.jsonc b/decompiler/config/jak2/jak2_config.jsonc index fce60192f..f8a788308 100644 --- a/decompiler/config/jak2/jak2_config.jsonc +++ b/decompiler/config/jak2/jak2_config.jsonc @@ -43,6 +43,13 @@ // write out a json file containing the art info mapping, run this with all objects allowed "dump_art_group_info": false, + // set to false to skip adding .STR files to the decompiler database + "read_spools": false, + // write out spool subtitle text, implies read_spools + "process_subtitle_text": true, + // write out spool subtitle images, implies read_spools + "process_subtitle_images": true, + /////////////////////////// // WEIRD OPTIONS /////////////////////////// diff --git a/decompiler/config/jak2/ntsc_v1/inputs.jsonc b/decompiler/config/jak2/ntsc_v1/inputs.jsonc index 751523a3f..6fe420f29 100644 --- a/decompiler/config/jak2/ntsc_v1/inputs.jsonc +++ b/decompiler/config/jak2/ntsc_v1/inputs.jsonc @@ -164,7 +164,247 @@ ], // some objects are part of STR files (streaming data). - "str_file_names": [], + "str_file_names": [ + "STR/AT1INT.STR", + "STR/AT1RES.STR", + "STR/AT2INTRO.STR", + "STR/AT3INTRO.STR", + "STR/ATSA.STR", + "STR/ATSARA.STR", + "STR/ATSARB.STR", + "STR/ATSB.STR", + "STR/ATSC.STR", + "STR/ATSD.STR", + "STR/ATSE.STR", + "STR/ATSINTRO.STR", + "STR/ATSTANK.STR", + "STR/BACONSIT.STR", + "STR/BASQUID.STR", + "STR/BAWIDOW.STR", + "STR/CAATIN.STR", + "STR/CAATOUT.STR", + "STR/CAIIINTR.STR", + "STR/CAIIRES.STR", + "STR/CAKBFINT.STR", + "STR/CAKBFRES.STR", + "STR/CASEXPLO.STR", + "STR/CIADOFF.STR", + "STR/CIATICAS.STR", + "STR/CIATINES.STR", + "STR/CIATOUT.STR", + "STR/CIC1RIA.STR", + "STR/CIC1RIB.STR", + "STR/CIC1RRES.STR", + "STR/CIC2RINT.STR", + "STR/CIC2RRES.STR", + "STR/CIC3RINT.STR", + "STR/CIC3RRES.STR", + "STR/CIDGVINT.STR", + "STR/CIDSINTR.STR", + "STR/CIDSRES.STR", + "STR/CIECINTR.STR", + "STR/CIECRES.STR", + "STR/CIEKINTR.STR", + "STR/CIGDGUN.STR", + "STR/CIGHOVER.STR", + "STR/CIGYGUN.STR", + "STR/CIHKINTR.STR", + "STR/CIHKRESO.STR", + "STR/CIIDINTR.STR", + "STR/CIIHCINT.STR", + "STR/CIIHCRES.STR", + "STR/CIITINTR.STR", + "STR/CIITRES.STR", + "STR/CIKCINTR.STR", + "STR/CIKCRES.STR", + "STR/CIKDINTR.STR", + "STR/CIMBINTR.STR", + "STR/CIMBRES.STR", + "STR/CIOINTRO.STR", + "STR/CIOL0.STR", + "STR/CIOL1.STR", + "STR/CIOL2.STR", + "STR/CIOL3.STR", + "STR/CIPHOVER.STR", + "STR/CIPOGINT.STR", + "STR/CIPOGRES.STR", + "STR/CIPSINTR.STR", + "STR/CISBBINT.STR", + "STR/CISLINTR.STR", + "STR/CISOPINT.STR", + "STR/CISUINTR.STR", + "STR/CIWAMINT.STR", + "STR/CIWAMRES.STR", + "STR/COFBRES.STR", + "STR/CRINTRO.STR", + "STR/CRVICTOR.STR", + "STR/DAMOLE.STR", + "STR/DEDINTRO.STR", + // "STR/DESCREEN.STR", + "STR/DIDEXPLO.STR", + "STR/DIFTINTR.STR", + "STR/DIFTRES.STR", + "STR/DIKDSINT.STR", + "STR/DRBSBREA.STR", + "STR/DRCBREAK.STR", + "STR/DRDCTINT.STR", + "STR/DRDSINTR.STR", + "STR/DRKMHINT.STR", + "STR/DRTEXPLO.STR", + "STR/DRW1.STR", + "STR/DRW2.STR", + "STR/ECINTRO.STR", + "STR/ECVICTOR.STR", + "STR/FO2INTRO.STR", + "STR/FOBUARA.STR", + "STR/FOBUARB.STR", + "STR/FOCMHINT.STR", + "STR/FOFA.STR", + "STR/FOFB.STR", + "STR/FOHCMHIN.STR", + "STR/FOPSIA.STR", + "STR/FOPSIB.STR", + "STR/FOPSRES.STR", + "STR/FOSFIA.STR", + "STR/FOSFRES.STR", + "STR/GRMANIMS.STR", + "STR/INCSQUAR.STR", + "STR/INPRISON.STR", + "STR/INSHUT.STR", + "STR/INVORTEX.STR", + "STR/JAA1.STR", + "STR/JAA2.STR", + "STR/JAA3.STR", + "STR/JAA4.STR", + "STR/JAA5.STR", + "STR/JAA6.STR", + "STR/JAA7.STR", + "STR/JABOARD.STR", + "STR/JACARRY.STR", + "STR/JAD1.STR", + "STR/JAD2.STR", + "STR/JAD3.STR", + "STR/JAD4.STR", + "STR/JAD5.STR", + "STR/JADARK.STR", + "STR/JADON.STR", + "STR/JADUMMY.STR", + "STR/JAFLUT.STR", + "STR/JAGUN.STR", + "STR/JAICE.STR", + "STR/JAINDAX.STR", + "STR/JAMECH.STR", + "STR/JAPEGASU.STR", + "STR/JAPIDAX.STR", + "STR/JAPILOT.STR", + "STR/JAPOLE.STR", + "STR/JARACER.STR", + "STR/JASWIM.STR", + "STR/JATUBE.STR", + "STR/JATURRET.STR", + "STR/KEANIM.STR", + "STR/KEGARAGE.STR", + "STR/KILTRNKR.STR", + "STR/KILYSKDC.STR", + "STR/KINESTB.STR", + "STR/KITOMBD.STR", + "STR/KRDRES.STR", + "STR/MOFINTRO.STR", + "STR/MOGRES.STR", + "STR/MOLRES.STR", + "STR/MOSRES.STR", + "STR/MTAR1.STR", + "STR/MTPBRA.STR", + "STR/MTSPRA.STR", + "STR/MTSPRB.STR", + "STR/MTSPRC.STR", + "STR/NEATIN.STR", + "STR/NEATOUT.STR", + "STR/NEBBRES.STR", + "STR/NEKBFIB.STR", + "STR/NEKBFMID.STR", + "STR/ONGAME.STR", + "STR/OUHIPHOG.STR", + "STR/OUNEST.STR", + "STR/OUPALACE.STR", + "STR/OUPORT.STR", + "STR/PABRES.STR", + "STR/PAOWRB.STR", + "STR/PAOWRES.STR", + "STR/PASIRES.STR", + // "STR/PRMINIMA.STR", + "STR/RHW1.STR", + "STR/RHW2.STR", + "STR/RUB1.STR", + "STR/RUBW1.STR", + "STR/RUBW2.STR", + "STR/RUBW3.STR", + "STR/RUBW4.STR", + "STR/RUBW5.STR", + "STR/RUBW6.STR", + "STR/RUDPA1.STR", + "STR/RUDPB1.STR", + "STR/RUDPC1.STR", + "STR/RUGTHRES.STR", + "STR/RUPC1.STR", + "STR/RUPC2.STR", + "STR/RUPC3.STR", + "STR/RUSINTRO.STR", + "STR/RUSVICTO.STR", + "STR/RUTINTRO.STR", + "STR/RUTVICTO.STR", + "STR/SALSAMER.STR", + // "STR/SCBOOK.STR", + "STR/SE1INTRO.STR", + "STR/SE1RES.STR", + "STR/SE2INTRO.STR", + "STR/SEBUSINT.STR", + "STR/SEBUSRES.STR", + "STR/SEC1.STR", + "STR/SEDRES.STR", + "STR/SEHOSEHE.STR", + "STR/SESGRUNT.STR", + "STR/SEW1.STR", + "STR/SEW2.STR", + "STR/TELHIPHO.STR", + "STR/TELTRNTE.STR", + "STR/TELWHACK.STR", + "STR/TESCENE.STR", + "STR/TIDINTRO.STR", + "STR/TOBBA.STR", + "STR/TOBBB.STR", + "STR/TOBINTRO.STR", + "STR/TOBOPEN.STR", + "STR/TOBRES.STR", + "STR/TOBSTART.STR", + "STR/TOFTINTR.STR", + "STR/TOSC0.STR", + "STR/TOSC1.STR", + "STR/TOSC2.STR", + "STR/TOSSCARE.STR", + "STR/TOTURRET.STR", + "STR/TOUPOLES.STR", + "STR/TOUSTART.STR", + "STR/TOUWATER.STR", + "STR/UNBD1.STR", + "STR/UNBD2.STR", + "STR/UNBD3.STR", + "STR/UNBD4.STR", + "STR/UNCONE.STR", + "STR/UNCTHREE.STR", + "STR/UNCTWO.STR", + "STR/UNFSRES.STR", + "STR/UNGSORES.STR", + "STR/VIRESCUE.STR", + "STR/VIRINTRO.STR", + // "STR/WOMAP.STR", + "STR/YOFOREST.STR", + "STR/YOLTRNYS.STR", + "STR/YOLYSAMS.STR", + "STR/YOLYSKDC.STR", + "STR/YOONINTE.STR", + "STR/YOTOMBD.STR" + ], // some objects are directly stored as files on the DVD. This is just text files. "object_file_names": [ diff --git a/decompiler/data/StrFileReader.cpp b/decompiler/data/StrFileReader.cpp index 9a4320b70..4bca7ec12 100644 --- a/decompiler/data/StrFileReader.cpp +++ b/decompiler/data/StrFileReader.cpp @@ -13,8 +13,23 @@ #include "game/common/overlord_common.h" #include "game/common/str_rpc_types.h" +#include "third-party/fmt/format.h" + namespace decompiler { -StrFileReader::StrFileReader(const fs::path& file_path) { +StrFileReader::StrFileReader(const fs::path& file_path, GameVersion version) : m_version(version) { + switch (version) { + case GameVersion::Jak1: + init_jak1(file_path); + break; + case GameVersion::Jak2: + init_jak2(file_path); + break; + default: + throw std::runtime_error("[StrFileReader] NYI game version"); + } +} + +void StrFileReader::init_jak1(const fs::path& file_path) { auto data = file_util::read_binary_file(file_path); ASSERT(data.size() >= SECTOR_SIZE); // must have at least the header sector ASSERT(data.size() % SECTOR_SIZE == 0); // should be multiple of the sector size. @@ -62,6 +77,49 @@ StrFileReader::StrFileReader(const fs::path& file_path) { } } +void StrFileReader::init_jak2(const fs::path& file_path) { + auto data = file_util::read_binary_file(file_path); + ASSERT(data.size() >= SECTOR_SIZE); // must have at least the header sector + ASSERT(data.size() % SECTOR_SIZE == 0); // should be multiple of the sector size. + int end_sector = int(data.size()) / SECTOR_SIZE; + + auto* header = (StrFileHeaderJ2*)data.data(); + + bool got_zero = false; + for (int i = 0; i < SECTOR_TABLE_SIZE_J2; i++) { + // the chunk is from sector to next_sector + int sector = header->sectors[i]; + // assume this chunk continues to the end... + int next_sector = end_sector; + // unless there's another chunk. + if (i + 1 < SECTOR_TABLE_SIZE_J2 && header->sectors[i + 1]) { + next_sector = header->sectors[i + 1]; + } + if (sector) { + ASSERT(!got_zero); // shouldn't have a non-zero after a zero! + ASSERT(next_sector > sector); // should have a positive size. + ASSERT(next_sector * SECTOR_SIZE <= int(data.size())); // check for overflowing the file + // get chunk data. + std::vector chunk; + chunk.insert(chunk.end(), data.begin() + sector * SECTOR_SIZE, + data.begin() + next_sector * SECTOR_SIZE); + m_chunks.emplace_back(std::move(chunk)); + } else { + got_zero = true; + } + } + + // check our sizes are accurate. Will make sure that we include all data, as our m_chunks + // are sized assuming they are packed in order and dense (sectors); + for (int i = 0; i < SECTOR_TABLE_SIZE_J2; i++) { + if (header->sectors[i]) { + ASSERT(header->sizes[i] == m_chunks.at(i).size()); + } else { + ASSERT(header->sizes[i] == 0); + } + } +} + int StrFileReader::chunk_count() const { return m_chunks.size(); } @@ -138,7 +196,7 @@ std::string StrFileReader::get_full_name(const std::string& short_name) const { bool done_first = false; // this string is part of the file info struct and the stuff after it is the file name. - const std::string file_info_string = "/src/next/data/art-group6/"; + const auto& file_info_string = get_file_info_string(); // it should occur in each chunk. int chunk_id = 0; @@ -150,7 +208,7 @@ std::string StrFileReader::get_full_name(const std::string& short_name) const { if (find_string_in_data(chunk.data(), int(chunk.size()), file_info_string, &offset)) { offset += file_info_string.length(); } else { - ASSERT(false); + ASSERT_MSG(false, fmt::format("did not find string '{}'", file_info_string)); } // extract the name info as a "name" + "chunk id" + "-ag.go" format. diff --git a/decompiler/data/StrFileReader.h b/decompiler/data/StrFileReader.h index 2ff542d08..3d16de1ff 100644 --- a/decompiler/data/StrFileReader.h +++ b/decompiler/data/StrFileReader.h @@ -9,17 +9,37 @@ #include #include "common/common_types.h" +#include "common/util/Assert.h" #include "common/util/FileUtil.h" +#include "common/versions.h" namespace decompiler { class StrFileReader { public: - explicit StrFileReader(const fs::path& file_path); + explicit StrFileReader(const fs::path& file_path, GameVersion version); int chunk_count() const; const std::vector& get_chunk(int idx) const; std::string get_full_name(const std::string& short_name) const; private: + void init_jak1(const fs::path& file_path); + void init_jak2(const fs::path& file_path); + + GameVersion m_version; + const std::string get_file_info_string() const { + switch (m_version) { + case GameVersion::Jak1: + return "/src/next/data/art-group6/"; + break; + case GameVersion::Jak2: + return "/src/jak2/final/art-group7/"; + break; + default: + ASSERT_MSG(false, "NYI get_file_info_string version"); + break; + } + } + std::vector> m_chunks; }; } // namespace decompiler diff --git a/decompiler/data/game_subs.h b/decompiler/data/game_subs.h new file mode 100644 index 000000000..3d9f0c3c8 --- /dev/null +++ b/decompiler/data/game_subs.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "common/util/FileUtil.h" +#include "common/util/FontUtils.h" + +namespace decompiler { +struct ObjectFileData; + +struct SpoolSubtitleMessage { + enum class Kind { NIL = 0, STRING, IMAGE } kind; + std::string text; + + s16 w, h; + std::array palette; + std::vector data; +}; + +struct SpoolSubtitleRange { + float start_frame; + float end_frame; + SpoolSubtitleMessage message[8]; + + bool operator==(const SpoolSubtitleRange& other) const { + if (start_frame != other.start_frame || end_frame != other.end_frame) + return false; + for (int i = 0; i < 8; ++i) { + if (message[i].kind != other.message[i].kind) + return false; + if (message[i].kind == SpoolSubtitleMessage::Kind::IMAGE) { + if (message[i].data != other.message[i].data || + message[i].palette != other.message[i].palette || message[i].w != other.message[i].w || + message[i].h != other.message[i].h) + return false; + } else { + if (message[i].text != other.message[i].text) + return false; + } + } + return true; + } + bool operator!=(const SpoolSubtitleRange& other) const { return !(*this == other); } +}; + +std::vector process_spool_subtitles(ObjectFileData& data, + GameTextVersion version); +std::string write_spool_subtitles( + GameTextVersion version, + const fs::path& image_out, + const std::unordered_map>& data); +} // namespace decompiler diff --git a/decompiler/data/game_text.cpp b/decompiler/data/game_text.cpp index c34b0785e..7bac33d1b 100644 --- a/decompiler/data/game_text.cpp +++ b/decompiler/data/game_text.cpp @@ -5,9 +5,12 @@ #include #include +#include "game_subs.h" + #include "common/goos/Reader.h" #include "common/util/BitUtils.h" #include "common/util/FontUtils.h" +#include "common/util/print_float.h" #include "decompiler/ObjectFile/ObjectFileDB.h" @@ -33,6 +36,9 @@ static const std::unordered_map> sTextCredi {GameTextVersion::JAK1_V1, {0xb00, 0xf00}}, {GameTextVersion::JAK1_V2, {0xb00, 0xf00}}}; +bool word_is_type(const LinkedWord& word, const std::string& type_name) { + return word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == type_name; +} } // namespace /* @@ -241,4 +247,245 @@ std::string write_game_text( return result; } + +/* +(defun unpack-comp-rle ((arg0 (pointer int8)) (arg1 (pointer int8))) + (local-vars (v1-2 int) (v1-3 uint)) + (nop!) + (loop + (loop + (set! v1-2 (-> arg1 0)) + (set! arg1 (&-> arg1 1)) + (b! (<= v1-2 0) cfg-5 :delay (nop!)) + (let ((a2-0 (-> arg1 0))) + (set! arg1 (&-> arg1 1)) + (label cfg-3) + (nop!) + (nop!) + (nop!) + (nop!) + (set! (-> arg0 0) a2-0) + ) + (set! arg0 (&-> arg0 1)) + (b! (> v1-2 0) cfg-3 :delay (set! v1-2 (+ v1-2 -1))) + ) + (label cfg-5) + (b! (zero? v1-2) cfg-8 :delay (set! v1-3 (the-as uint (- v1-2)))) + (label cfg-6) + (let ((a2-1 (-> arg1 0))) + (set! arg1 (&-> arg1 1)) + (nop!) + (nop!) + (set! (-> arg0 0) a2-1) + ) + (+! v1-3 -1) + (b! (> (the-as int v1-3) 0) cfg-6 :delay (set! arg0 (&-> arg0 1))) + ) + (label cfg-8) + (none) + ) +*/ +void unpack_comp_rle(s8* dst, s8* src) { + while (true) { + int amt; + while (true) { + // how many bytes to duplicate? + amt = *(src++); + // no more data or copy from src + if (amt <= 0) + break; + // the byte to duplicate + int dup = *(src++); + do { + *(dst++) = dup; + } while (amt-- > 0); + } + // no more data + if (amt == 0) + return; + // how many bytes to simply copy? + int copy = -amt; + do { + *(dst++) = *(src++); + } while (--copy > 0); + } +} + +/* +(deftype subtitle-range (basic) + ((start-frame float :offset-assert 4) + (end-frame float :offset-assert 8) + (message basic 8 :offset-assert 12) + ) + :method-count-assert 9 + :size-assert #x2c + :flag-assert #x90000002c + ;; Failed to read fields. + ) + */ + +std::vector process_spool_subtitles(ObjectFileData& data, + GameTextVersion version) { + std::vector result; + auto& words = data.linked_data.words_by_seg.at(0); + + // type tag for art-group and get art count + ASSERT(word_is_type(words.at(0), "art-group")); + auto elt_count = get_word(words.at(3)); + + for (int i = 0; i < elt_count; ++i) { + auto art_label = get_label(data, words.at(8 + i)); + auto art_word_ofs = art_label.offset / 4; + if (word_is_type(words.at(art_word_ofs - 1), "art-joint-anim")) { + auto lump_for_art_word = words.at(art_word_ofs + 3); + if (lump_for_art_word.kind() == LinkedWord::PTR) { + // got lump + auto lump_word_ofs = get_label(data, lump_for_art_word).offset / 4; + ASSERT(word_is_type(words.at(lump_word_ofs - 1), "res-lump")); + auto tag_amt = get_word(words.at(lump_word_ofs)); + auto data_base = get_label(data, words.at(lump_word_ofs + 2)).offset / 4; + auto tag_ofs = get_label(data, words.at(lump_word_ofs + 6)).offset / 4; + + for (int t = 0; t < tag_amt; ++t) { + // check for a specific kind of res-tag (name, frame, type and length) + if (words.at(tag_ofs).symbol_name() == "subtitle-range" && + get_word(words.at(tag_ofs + 1)) == -1000000000.0f && + word_is_type(words.at(tag_ofs + 2), "array") && + (get_word(words.at(tag_ofs + 3)) >> 16) == 0x1) { + auto data_ofs = get_word(words.at(tag_ofs + 3)) & 0xffff; + data_ofs = data_base + align4(data_ofs) / 4; + // the res will be a (array subtitle-range) + auto subtitle_array_ofs = get_label(data, words.at(data_ofs)).offset / 4; + ASSERT(word_is_type(words.at(subtitle_array_ofs + 2), "subtitle-range")); + auto subtitle_amount = get_word(words.at(subtitle_array_ofs)); + + for (int s = 0; s < subtitle_amount; ++s) { + auto subtitle_lbl = get_label(data, words.at(subtitle_array_ofs + 3 + s)); + auto subtitle_ofs = subtitle_lbl.offset / 4; + // add our new subtitle range + auto& subtitles = result.emplace_back(); + subtitles.start_frame = get_word(words.at(subtitle_ofs)); + subtitles.end_frame = get_word(words.at(subtitle_ofs + 1)); + + for (int m = 0; m < 8; ++m) { + // process the message for each language + auto& msg = subtitles.message[m]; + const auto& msg_ptr = words.at(subtitle_ofs + 2 + m); + if (msg_ptr.kind() == LinkedWord::Kind::SYM_PTR && msg_ptr.symbol_name() == "#f") { + msg.kind = SpoolSubtitleMessage::Kind::NIL; + } else { + auto m_lbl = get_label(data, msg_ptr); + auto m_ofs = m_lbl.offset / 4; + auto m_type_w = words.at(m_ofs - 1); + ASSERT(m_type_w.kind() == LinkedWord::TYPE_PTR); + if (m_type_w.symbol_name() == "string") { + msg.kind = SpoolSubtitleMessage::Kind::STRING; + auto text = data.linked_data.get_goal_string_by_label(m_lbl); + // escape characters + if (font_bank_exists(version)) { + msg.text = get_font_bank(version)->convert_game_to_utf8(text.c_str()); + } else { + msg.text = goos::get_readable_string(text.c_str()); // HACK! + } + } else if (m_type_w.symbol_name() == "subtitle-image") { + msg.kind = SpoolSubtitleMessage::Kind::IMAGE; + auto wh = get_word(words.at(m_ofs)); + msg.w = wh; + msg.h = (wh >> 16) + 1; + msg.h--; // correct height + for (int p = 0; p < 16; ++p) { + msg.palette[p] = get_word(words.at(m_ofs + 3 + p)); + } + + // unpack the image data. we have no idea what the input size is, + // so we just copy all of the plain data after this. + auto img_data_ofs = m_ofs + 3 + 16; + int img_data_top = words.size(); + for (int check = img_data_ofs; check < img_data_top; ++check) { + if (words.at(check).kind() != LinkedWord::Kind::PLAIN_DATA) { + img_data_top = check; + break; + } + } + // it's a 4-bit image, meaning 2 pixels per byte, so we round up. + msg.data.resize(align2(msg.w * msg.h / 2)); + std::vector input_data; + input_data.resize((img_data_top - img_data_ofs) * 4); + for (int copy = 0; copy < (img_data_top - img_data_ofs); ++copy) { + *((u32*)(input_data.data() + copy * 4)) = + get_word(words.at(img_data_ofs + copy)); + } + // unpack now! hopefully there was enough input data... + unpack_comp_rle((s8*)msg.data.data(), (s8*)input_data.data()); + } else { + ASSERT_MSG(false, fmt::format("unknown subtitle message type {}", + m_type_w.symbol_name())); + } + } + } + } + } + tag_ofs += 4; + } + } + } + } + + return result; +} + +std::string write_spool_subtitles( + GameTextVersion version, + const fs::path& image_out, + const std::unordered_map>& data) { + // write! + std::string result; // = "\xEF\xBB\xBF"; // UTF-8 encode (don't need this anymore) + + bool dump_images = !image_out.empty(); + if (dump_images) { + file_util::create_dir_if_needed(image_out); + } + + for (auto& [spool_name, subs] : data) { + int image_count = 0; + result += "(\"" + spool_name + "\"\n"; + for (auto& sub : subs) { + std::string temp_for_indent = fmt::format(" (({} {}) (", float_to_string(sub.start_frame), + float_to_string(sub.end_frame)); + auto indent = temp_for_indent.length(); + result += temp_for_indent; + for (int i = 0; i < 8; ++i) { + const auto& msg = sub.message[i]; + if (i > 0) { + result += "\n" + std::string(indent, ' '); + } + if (msg.kind == SpoolSubtitleMessage::Kind::NIL) { + result += "#f"; + } else { + result += "("; + if (msg.kind == SpoolSubtitleMessage::Kind::IMAGE) { + auto img_name = fmt::format("{}-{}-{}.png", spool_name, i, image_count++); + result += "image " + img_name; + if (dump_images) { + std::vector rgba_out; + rgba_out.resize(msg.w * msg.h); + for (int px = 0; px < rgba_out.size(); ++px) { + int idx = px & 1 ? msg.data[px / 2] >> 4 : msg.data[px / 2] & 0xf; + rgba_out.at(px) = msg.palette[idx]; + } + file_util::write_rgba_png(image_out / img_name, rgba_out.data(), msg.w, msg.h); + } + } else if (msg.kind == SpoolSubtitleMessage::Kind::STRING) { + result += "\"" + msg.text + "\""; + } + result += ")"; + } + } + result += ")\n )\n"; + } + result += " )\n\n"; + } + + return result; +} } // namespace decompiler diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 16f652413..abf940ec7 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -21,6 +21,13 @@ #include "third-party/CLI11.hpp" #include "third-party/json.hpp" +template +static void mem_log(const std::string& format, Args&&... args) { +#ifndef _WIN32 + lg::info("[Mem] " + format, std::forward(args)...); +#endif +} + int main(int argc, char** argv) { ArgumentGuard u8_guard(argc, argv); @@ -80,6 +87,9 @@ int main(int argc, char** argv) { return 1; } + // these options imply read_spools + config.read_spools |= config.process_subtitle_text || config.process_subtitle_images; + // Check if any banned objects are also in the allowed objects list // if so, throw an error as this can be a confusing situation auto intersection = set_util::intersection(config.allowed_objects, config.banned_objects); @@ -112,11 +122,11 @@ int main(int argc, char** argv) { Timer decomp_timer; - lg::info("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024)); + mem_log("Top of main: {} MB\n", get_peak_rss() / (1024 * 1024)); init_opcode_info(); - lg::info("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024)); + mem_log("After init: {} MB\n", get_peak_rss() / (1024 * 1024)); std::vector dgos, objs, strs; for (const auto& dgo_name : config.dgo_names) { @@ -131,7 +141,7 @@ int main(int argc, char** argv) { strs.push_back(in_folder / str_name); } - lg::info("[Mem] After config read: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After config read: {} MB", get_peak_rss() / (1024 * 1024)); // build file database lg::info("Setting up object file DB..."); @@ -152,7 +162,7 @@ int main(int argc, char** argv) { } } - lg::info("[Mem] After DB setup: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After DB setup: {} MB", get_peak_rss() / (1024 * 1024)); // write out DGO file info file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing()); @@ -169,10 +179,10 @@ int main(int argc, char** argv) { // process files (required for all analysis) db.process_link_data(config); - lg::info("[Mem] After link data: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After link data: {} MB", get_peak_rss() / (1024 * 1024)); db.find_code(config); db.process_labels(); - lg::info("[Mem] After code: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After code: {} MB", get_peak_rss() / (1024 * 1024)); // top level decompile (do this before printing asm so we get function names) if (config.find_functions) { @@ -216,7 +226,7 @@ int main(int argc, char** argv) { config.hacks.types_with_bad_inspect_methods); } - lg::info("[Mem] After decomp: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After decomp: {} MB", get_peak_rss() / (1024 * 1024)); // write out all symbols file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types()); @@ -242,7 +252,17 @@ int main(int argc, char** argv) { } } - lg::info("[Mem] After text: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After text: {} MB", get_peak_rss() / (1024 * 1024)); + + if (config.process_subtitle_text || config.process_subtitle_images) { + auto result = db.process_all_spool_subtitles( + config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : ""); + if (!result.empty()) { + file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result); + } + } + + mem_log("After spool handling: {} MB", get_peak_rss() / (1024 * 1024)); decompiler::TextureDB tex_db; if (config.process_tpages || config.levels_extract) { @@ -256,7 +276,8 @@ int main(int argc, char** argv) { } } - lg::info("[Mem] After textures: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After textures: {} MB", get_peak_rss() / (1024 * 1024)); + auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements"; if (fs::exists(replacements_path)) { tex_db.replace_textures(replacements_path); @@ -277,7 +298,7 @@ int main(int argc, char** argv) { config.rip_levels, config.extract_collision, level_out_path); } - lg::info("[Mem] After extraction: {} MB", get_peak_rss() / (1024 * 1024)); + mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024)); if (!config.audio_dir_file_name.empty()) { auto streaming_audio_in = in_folder / "VAG"; diff --git a/game/graphics/opengl_renderer/OpenGLRenderer.cpp b/game/graphics/opengl_renderer/OpenGLRenderer.cpp index 532a84b75..0a49ec699 100644 --- a/game/graphics/opengl_renderer/OpenGLRenderer.cpp +++ b/game/graphics/opengl_renderer/OpenGLRenderer.cpp @@ -205,7 +205,7 @@ void OpenGLRenderer::init_bucket_renderers_jak2() { init_bucket_renderer("tex-all-sprite", BucketCategory::TEX, BucketId::TEX_ALL_SPRITE); init_bucket_renderer("particles", BucketCategory::SPRITE, BucketId::PARTICLES); - init_bucket_renderer("lightning", BucketCategory::OTHER, BucketId::EFFECTS); + init_bucket_renderer("effects", BucketCategory::OTHER, BucketId::EFFECTS); init_bucket_renderer("tex-all-warp", BucketCategory::TEX, BucketId::TEX_ALL_WARP); init_bucket_renderer("debug-no-zbuf1", BucketCategory::OTHER, @@ -217,7 +217,7 @@ void OpenGLRenderer::init_bucket_renderers_jak2() { 0x8000); init_bucket_renderer("screen-filter", BucketCategory::OTHER, BucketId::SCREEN_FILTER, 256); - init_bucket_renderer("bucket-322", BucketCategory::OTHER, BucketId::BUCKET_322, + init_bucket_renderer("subtitle", BucketCategory::OTHER, BucketId::SUBTITLE, 0x8000); init_bucket_renderer("debug2", BucketCategory::OTHER, BucketId::DEBUG2, 0x8000); init_bucket_renderer("debug-no-zbuf2", BucketCategory::OTHER, diff --git a/game/graphics/opengl_renderer/buckets.h b/game/graphics/opengl_renderer/buckets.h index 7ebc7d9ee..5601dab09 100644 --- a/game/graphics/opengl_renderer/buckets.h +++ b/game/graphics/opengl_renderer/buckets.h @@ -403,7 +403,7 @@ enum class BucketId { TEX_ALL_MAP, PROGRESS, SCREEN_FILTER, - BUCKET_322, + SUBTITLE, BUCKET_323, DEBUG2, DEBUG_NO_ZBUF2, diff --git a/goal_src/jak2/engine/ambient/ambient.gc b/goal_src/jak2/engine/ambient/ambient.gc index 5db047f6f..9805e0cb8 100644 --- a/goal_src/jak2/engine/ambient/ambient.gc +++ b/goal_src/jak2/engine/ambient/ambient.gc @@ -330,7 +330,7 @@ a2-3 a3-2 (if (or (= v1-31 (gui-channel notice)) (= v1-31 (gui-channel notice-low)) (= v1-31 (gui-channel subtitle))) - (bucket-id bucket-322) + (bucket-id subtitle) (bucket-id progress) ) ) diff --git a/goal_src/jak2/engine/gfx/hw/display-h.gc b/goal_src/jak2/engine/gfx/hw/display-h.gc index 00592d0f0..64ac03b56 100644 --- a/goal_src/jak2/engine/gfx/hw/display-h.gc +++ b/goal_src/jak2/engine/gfx/hw/display-h.gc @@ -150,4 +150,4 @@ At any point in time, there are 3 frames in progress: `(the int (* ,frac 512))) (defmacro get-screen-y (frac) - `(the int (* ,frac 224))) + `(the int (* ,frac 416))) diff --git a/goal_src/jak2/engine/gfx/vu1-user-h.gc b/goal_src/jak2/engine/gfx/vu1-user-h.gc index e7e838cf6..6faf4b56b 100644 --- a/goal_src/jak2/engine/gfx/vu1-user-h.gc +++ b/goal_src/jak2/engine/gfx/vu1-user-h.gc @@ -405,7 +405,7 @@ for example: tie-s-l1-tfrag (tex-all-map) ;; tex (progress) ;; hud | progress (screen-filter) ;; hud letterbox, no zbuf - (bucket-322) ;; hud + (subtitle) ;; hud (bucket-323) ;; hud (debug2) ;; debug (debug-no-zbuf2) ;; debug diff --git a/goal_src/jak2/engine/scene/scene.gc b/goal_src/jak2/engine/scene/scene.gc index 2ddbcb9f0..7fa584897 100644 --- a/goal_src/jak2/engine/scene/scene.gc +++ b/goal_src/jak2/engine/scene/scene.gc @@ -763,7 +763,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s4-1 (the-as (pointer dma-tag) a3-15) ) @@ -822,7 +822,7 @@ (set! (-> s2-0 origin y) (+ 1.0 (-> s2-0 origin y))) (set! (-> s2-0 color) (font-color default)) (set! (-> s2-0 flags) (font-flags shadow kerning middle left large)) - (print-game-text (the-as string s3-0) s2-0 #f 44 (bucket-id bucket-322)) + (print-game-text (the-as string s3-0) s2-0 #f 44 (bucket-id subtitle)) (gui-control-method-12 *gui-control* self @@ -1463,7 +1463,7 @@ gp-6 #f 44 - (bucket-id bucket-322) + (bucket-id subtitle) ) ) ) diff --git a/goal_src/jak2/levels/intro/intro-scenes.gc b/goal_src/jak2/levels/intro/intro-scenes.gc index 1b2c06eb3..6802d29f4 100644 --- a/goal_src/jak2/levels/intro/intro-scenes.gc +++ b/goal_src/jak2/levels/intro/intro-scenes.gc @@ -2329,7 +2329,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s3-0 (the-as (pointer dma-tag) a3-0) ) @@ -2456,7 +2456,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s2-0 (the-as (pointer dma-tag) a3-0) ) diff --git a/goal_src/jak2/pc/util/font-encode-test.gc b/goal_src/jak2/pc/util/font-encode-test.gc new file mode 100644 index 000000000..3c39fc66c --- /dev/null +++ b/goal_src/jak2/pc/util/font-encode-test.gc @@ -0,0 +1,110 @@ +;;-*-Lisp-*- +(in-package goal) + + +;; This file is used for debugging and testing the large font encoding. +;; This file should *not* be included as part of any packages, it should be manually loaded by the user. + +;; To run this: + +#| +(mng) ;; build the engine +(lt) ;; connect to the runtime +(ml "goal_src/jak2/pc/util/font-encode-test.gc") ;; build and load this file. +|# + +(declare-file (debug)) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; constants +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defconstant FONT_ENCODE_TEXT_LEFT (get-screen-x 0.35)) +(defconstant FONT_ENCODE_TEXT_Y (get-screen-y 0.4)) + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; functions +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(define *font-string* (new 'global 'string 64 (the string #f))) +(define *font-string-val* #x96) +(define *text-id-val* #x100) +(define *music-to-be-less-annoying* 'finboss1) + +(defun font-encode-test-stop () + "stop the encode test proc" + + (kill-by-name "font-encode" *active-pool*) + ) + +(defun font-encode-test-start () + "start the encode test proc" + + (font-encode-test-stop) + (process-spawn-function process :name "font-encode" + (lambda :behavior process () + (let ((fnt (new 'stack 'font-context *font-default-matrix* FONT_ENCODE_TEXT_LEFT FONT_ENCODE_TEXT_Y 0.0 + (font-color red) (font-flags shadow kerning large middle))) + ) + + (set-width! fnt (get-screen-x 0.8)) + (set-height! fnt (get-screen-y 0.1)) + + (loop + (suspend) + + (set-setting! 'music *music-to-be-less-annoying* 0 0) + (cond + ((not (cpad-hold? 0 r3)) + (if (or (cpad-pressed? 0 left) (cpad-hold? 0 l1)) + (-! *font-string-val* 1)) + (if (or (cpad-pressed? 0 right) (cpad-hold? 0 r1)) + (+! *font-string-val* 1)) + ) + (else + (if (or (cpad-pressed? 0 left) (cpad-hold? 0 l1)) + (-! *text-id-val* 1)) + (if (or (cpad-pressed? 0 right) (cpad-hold? 0 r1)) + (+! *text-id-val* 1)) + ) + ) + (minmax! *font-string-val* 1 #xfff) + + (let ((chars (1+ (/ (log2 *font-string-val*) 8)))) + (dotimes (i chars) + (set! (-> *font-string* data (- chars i 1)) (ash *font-string-val* (* i -8)))) + (set! (-> *font-string* data (1+ (/ (log2 *font-string-val*) 8))) 0) + ) + + (with-dma-buffer-add-bucket ((buf (-> (current-frame) debug-buf)) + (bucket-id debug-no-zbuf2)) + (set-scale! fnt 1.0) + (set-origin! fnt (get-screen-x 0.5) (- FONT_ENCODE_TEXT_Y 25)) + (set-flags! fnt (font-flags shadow kerning middle)) + (draw-string (string-format "#x~x (~x ~x ~x ~x)" *font-string-val* + (-> *font-string* data 0) + (-> *font-string* data 1) + (-> *font-string* data 2) + (-> *font-string* data 3)) + buf fnt) + (set-origin! fnt (get-screen-x 0.5) (+ FONT_ENCODE_TEXT_Y 50)) + (draw-string (string-format "#x~x" *text-id-val*) buf fnt) + (set-flags! fnt (font-flags shadow kerning large middle)) + (set-origin! fnt (get-screen-x 0.5) FONT_ENCODE_TEXT_Y) + (draw-string *font-string* buf fnt) + (set-origin! fnt (get-screen-x 0.5) (+ FONT_ENCODE_TEXT_Y 75)) + (set-scale! fnt 0.75) + (draw-string (lookup-text! *common-text* (the game-text-id *text-id-val*) #f) buf fnt) + ) + ) + ) + + ) + ) + ) + +(font-encode-test-start) + diff --git a/test/decompiler/reference/jak2/engine/ambient/ambient_REF.gc b/test/decompiler/reference/jak2/engine/ambient/ambient_REF.gc index 7e3eed73b..ef9509309 100644 --- a/test/decompiler/reference/jak2/engine/ambient/ambient_REF.gc +++ b/test/decompiler/reference/jak2/engine/ambient/ambient_REF.gc @@ -345,7 +345,7 @@ a2-3 a3-2 (if (or (= v1-31 (gui-channel notice)) (= v1-31 (gui-channel notice-low)) (= v1-31 (gui-channel subtitle))) - (bucket-id bucket-322) + (bucket-id subtitle) (bucket-id progress) ) ) diff --git a/test/decompiler/reference/jak2/engine/scene/scene_REF.gc b/test/decompiler/reference/jak2/engine/scene/scene_REF.gc index 7fbf16441..7b2f37aed 100644 --- a/test/decompiler/reference/jak2/engine/scene/scene_REF.gc +++ b/test/decompiler/reference/jak2/engine/scene/scene_REF.gc @@ -925,7 +925,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s4-1 (the-as (pointer dma-tag) a3-15) ) @@ -986,7 +986,7 @@ (set! (-> s2-0 origin y) (+ 1.0 (-> s2-0 origin y))) (set! (-> s2-0 color) (font-color default)) (set! (-> s2-0 flags) (font-flags shadow kerning middle left large)) - (print-game-text (the-as string s3-0) s2-0 #f 44 (bucket-id bucket-322)) + (print-game-text (the-as string s3-0) s2-0 #f 44 (bucket-id subtitle)) (gui-control-method-12 *gui-control* self @@ -1629,7 +1629,7 @@ gp-6 #f 44 - (bucket-id bucket-322) + (bucket-id subtitle) ) ) ) diff --git a/test/decompiler/reference/jak2/levels/intro/intro-scenes_REF.gc b/test/decompiler/reference/jak2/levels/intro/intro-scenes_REF.gc index 8cce22a21..1b6a18bbe 100644 --- a/test/decompiler/reference/jak2/levels/intro/intro-scenes_REF.gc +++ b/test/decompiler/reference/jak2/levels/intro/intro-scenes_REF.gc @@ -2470,7 +2470,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s3-0 (the-as (pointer dma-tag) a3-0) ) @@ -2597,7 +2597,7 @@ ) (dma-bucket-insert-tag (-> *display* frames (-> *display* on-screen) bucket-group) - (bucket-id bucket-322) + (bucket-id subtitle) s2-0 (the-as (pointer dma-tag) a3-0) )