diff --git a/common/type_system/Type.cpp b/common/type_system/Type.cpp index 4b06f394d..9573d73d5 100644 --- a/common/type_system/Type.cpp +++ b/common/type_system/Type.cpp @@ -32,7 +32,8 @@ std::string reg_kind_to_string(RegClass kind) { */ bool MethodInfo::operator==(const MethodInfo& other) const { return id == other.id && name == other.name && type == other.type && - defined_in_type == other.defined_in_type && other.no_virtual == no_virtual; + defined_in_type == other.defined_in_type && other.no_virtual == no_virtual && + other.overrides_method_type_of_parent == overrides_method_type_of_parent; } std::string MethodInfo::diff(const MethodInfo& other) const { @@ -56,6 +57,11 @@ std::string MethodInfo::diff(const MethodInfo& other) const { if (no_virtual != other.no_virtual) { result += fmt::format("no_virtual: {} vs. {}\n", no_virtual, other.no_virtual); } + + if (overrides_method_type_of_parent != other.overrides_method_type_of_parent) { + result += fmt::format("overrides_method_type_of_parent: {} vs. {}\n", + overrides_method_type_of_parent, other.overrides_method_type_of_parent); + } return result; } @@ -311,9 +317,11 @@ bool Type::get_my_method(int id, MethodInfo* out) const { * defined specifically for this type or not. */ bool Type::get_my_last_method(MethodInfo* out) const { - if (!m_methods.empty()) { - *out = m_methods.back(); - return true; + for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) { + if (!it->overrides_method_type_of_parent) { + *out = *it; + return true; + } } return false; } @@ -334,9 +342,13 @@ bool Type::get_my_new_method(MethodInfo* out) const { * Add a method defined specifically for this type. */ const MethodInfo& Type::add_method(const MethodInfo& info) { - if (!m_methods.empty()) { - assert(m_methods.back().id + 1 == info.id); + for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) { + if (!it->overrides_method_type_of_parent) { + assert(it->id + 1 == info.id); + break; + } } + m_methods.push_back(info); return m_methods.back(); } diff --git a/common/type_system/Type.h b/common/type_system/Type.h index 544e1e7b9..d1075bfec 100644 --- a/common/type_system/Type.h +++ b/common/type_system/Type.h @@ -21,6 +21,7 @@ struct MethodInfo { TypeSpec type; std::string defined_in_type; bool no_virtual = false; + bool overrides_method_type_of_parent = false; bool operator==(const MethodInfo& other) const; bool operator!=(const MethodInfo& other) const { return !((*this) == other); } diff --git a/common/type_system/TypeFieldLookup.cpp b/common/type_system/TypeFieldLookup.cpp index c14abd455..9735bf158 100644 --- a/common/type_system/TypeFieldLookup.cpp +++ b/common/type_system/TypeFieldLookup.cpp @@ -81,6 +81,13 @@ void try_reverse_lookup_other(const FieldReverseLookupInput& input, FieldReverseMultiLookupOutput* output, int max_count); +std::vector parent_to_vector(const ReverseLookupNode* parent) { + if (!parent) { + return {}; + } + return parent->to_vector(); +} + /*! * Handle a dereference of a pointer/boxed array. This can be: * - just dereferencing a pointer @@ -188,7 +195,7 @@ void try_reverse_lookup_array_like(const FieldReverseLookupInput& input, // also just return the array if (elt_idx == 0) { if (boxed_array) { - auto vec = parent->to_vector(); + auto vec = parent_to_vector(parent); FieldReverseLookupOutput::Token tok; tok.kind = FieldReverseLookupOutput::Token::Kind::FIELD; tok.field_score = 0.0; // don't bother @@ -196,7 +203,7 @@ void try_reverse_lookup_array_like(const FieldReverseLookupInput& input, vec.push_back(tok); output->results.emplace_back(false, array_data_type, vec); } else { - auto parent_vector = parent->to_vector(); + auto parent_vector = parent_to_vector(parent); if (!parent_vector.empty()) { output->results.emplace_back(false, input.base_type, parent_vector); } @@ -280,9 +287,9 @@ void try_reverse_lookup_inline_array(const FieldReverseLookupInput& input, // can we just return the array? if (expected_offset_into_elt == offset_into_elt && !input.deref.has_value() && elt_idx == 0) { - auto parent_vec = parent->to_vector(); + auto parent_vec = parent_to_vector(parent); if (!parent_vec.empty()) { - output->results.emplace_back(false, input.base_type, parent->to_vector()); + output->results.emplace_back(false, input.base_type, parent_to_vector(parent)); } if ((int)output->results.size() >= max_count) { diff --git a/common/type_system/TypeSystem.cpp b/common/type_system/TypeSystem.cpp index 89bb97321..3c0aa0912 100644 --- a/common/type_system/TypeSystem.cpp +++ b/common/type_system/TypeSystem.cpp @@ -477,8 +477,10 @@ int TypeSystem::get_load_size_allow_partial_def(const TypeSpec& ts) const { MethodInfo TypeSystem::declare_method(const std::string& type_name, const std::string& method_name, bool no_virtual, - const TypeSpec& ts) { - return declare_method(lookup_type(make_typespec(type_name)), method_name, no_virtual, ts); + const TypeSpec& ts, + bool override_type) { + return declare_method(lookup_type(make_typespec(type_name)), method_name, no_virtual, ts, + override_type); } /*! @@ -493,8 +495,12 @@ MethodInfo TypeSystem::declare_method(const std::string& type_name, MethodInfo TypeSystem::declare_method(Type* type, const std::string& method_name, bool no_virtual, - const TypeSpec& ts) { + const TypeSpec& ts, + bool override_type) { if (method_name == "new") { + if (override_type) { + throw_typesystem_error("Cannot use :replace option with a new method."); + } return add_new_method(type, ts); } @@ -502,34 +508,48 @@ MethodInfo TypeSystem::declare_method(Type* type, MethodInfo existing_info; bool got_existing = try_lookup_method(type, method_name, &existing_info); - if (got_existing) { - // make sure we aren't changing anything. - if (!existing_info.type.is_compatible_child_method(ts, type->get_name())) { + if (override_type) { + if (!got_existing) { throw_typesystem_error( - "The method {} of type {} was originally declared as {}, but has been " - "redeclared as {}\n", - method_name, type->get_name(), existing_info.type.print(), ts.print()); + "Cannot use :replace on method {} of {} because this method was not previously declared " + "in a parent.", + method_name, type->get_name()); } - if ((existing_info.no_virtual || no_virtual) && - existing_info.defined_in_type != type->get_name()) { - throw_typesystem_error( - "Cannot define method {} in type {} when it was defined as no_virtual in parent type {}", - method_name, type->get_name(), existing_info.defined_in_type); - } - - if (no_virtual != existing_info.no_virtual) { - throw_typesystem_error( - "The method {} of type {} was originally declared with no_virtual = {}, but has been " - "redeclared as {}", - method_name, type->get_name(), existing_info.no_virtual, no_virtual); - } - - return existing_info; - } else { - // add a new method! + // use the existing ID. return type->add_method( - {get_next_method_id(type), method_name, ts, type->get_name(), no_virtual}); + {existing_info.id, method_name, ts, type->get_name(), no_virtual, true}); + } else { + if (got_existing) { + // make sure we aren't changing anything. + if (!existing_info.type.is_compatible_child_method(ts, type->get_name())) { + throw_typesystem_error( + "The method {} of type {} was originally declared as {}, but has been " + "redeclared as {}\n", + method_name, type->get_name(), existing_info.type.print(), ts.print()); + } + + if ((existing_info.no_virtual || no_virtual) && + existing_info.defined_in_type != type->get_name()) { + throw_typesystem_error( + "Cannot define method {} in type {} when it was defined as no_virtual in parent type " + "{}", + method_name, type->get_name(), existing_info.defined_in_type); + } + + if (no_virtual != existing_info.no_virtual) { + throw_typesystem_error( + "The method {} of type {} was originally declared with no_virtual = {}, but has been " + "redeclared as {}", + method_name, type->get_name(), existing_info.no_virtual, no_virtual); + } + + return existing_info; + } else { + // add a new method! + return type->add_method( + {get_next_method_id(type), method_name, ts, type->get_name(), no_virtual, false}); + } } } @@ -955,23 +975,25 @@ void TypeSystem::add_builtin_types() { // OBJECT declare_method(obj_type, "new", false, - make_function_typespec({"symbol", "type", "int"}, "_type_")); - declare_method(obj_type, "delete", false, make_function_typespec({"_type_"}, "none")); - declare_method(obj_type, "print", false, make_function_typespec({"_type_"}, "_type_")); - declare_method(obj_type, "inspect", false, make_function_typespec({"_type_"}, "_type_")); - declare_method(obj_type, "length", false, - make_function_typespec({"_type_"}, "int")); // todo - this integer type? - declare_method(obj_type, "asize-of", false, make_function_typespec({"_type_"}, "int")); - declare_method(obj_type, "copy", false, make_function_typespec({"_type_", "symbol"}, "_type_")); - declare_method(obj_type, "relocate", false, make_function_typespec({"_type_", "int"}, "_type_")); + make_function_typespec({"symbol", "type", "int"}, "_type_"), false); + declare_method(obj_type, "delete", false, make_function_typespec({"_type_"}, "none"), false); + declare_method(obj_type, "print", false, make_function_typespec({"_type_"}, "_type_"), false); + declare_method(obj_type, "inspect", false, make_function_typespec({"_type_"}, "_type_"), false); + declare_method(obj_type, "length", false, make_function_typespec({"_type_"}, "int"), + false); // todo - this integer type? + declare_method(obj_type, "asize-of", false, make_function_typespec({"_type_"}, "int"), false); + declare_method(obj_type, "copy", false, make_function_typespec({"_type_", "symbol"}, "_type_"), + false); + declare_method(obj_type, "relocate", false, make_function_typespec({"_type_", "int"}, "_type_"), + false); declare_method(obj_type, "mem-usage", false, - make_function_typespec({"_type_", "memory-usage-block", "int"}, "_type_")); + make_function_typespec({"_type_", "memory-usage-block", "int"}, "_type_"), false); // STRUCTURE // structure new doesn't support dynamic sizing, which is kinda weird - it grabs the size from // the type. Dynamic structures use new-dynamic-structure, which is used exactly once ever. - declare_method(structure_type, "new", false, - make_function_typespec({"symbol", "type"}, "_type_")); + declare_method(structure_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"), + false); // structure_type is a field-less StructureType, so we have to do this to match the runtime. // structure_type->override_size_in_memory(4); @@ -980,18 +1002,19 @@ void TypeSystem::add_builtin_types() { add_field_to_type(basic_type, "type", make_typespec("type")); // the default new basic doesn't support dynamic sizing. anything dynamic will override this // and then call (method object new) to do the dynamically-sized allocation. - declare_method(basic_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_")); + declare_method(basic_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"), + false); // SYMBOL builtin_structure_inherit(symbol_type); add_field_to_type(symbol_type, "value", make_typespec("object")); // a new method which returns type none means new is illegal. - declare_method(symbol_type, "new", false, make_function_typespec({}, "none")); + declare_method(symbol_type, "new", false, make_function_typespec({}, "none"), false); // TYPE builtin_structure_inherit(type_type); declare_method(type_type, "new", false, - make_function_typespec({"symbol", "type", "int"}, "_type_")); + make_function_typespec({"symbol", "type", "int"}, "_type_"), false); add_field_to_type(type_type, "symbol", make_typespec("symbol")); add_field_to_type(type_type, "parent", make_typespec("type")); add_field_to_type(type_type, "size", make_typespec("uint16")); // actually u16 @@ -1008,7 +1031,7 @@ void TypeSystem::add_builtin_types() { // string is never deftype'd for the decompiler, so we need to manually give the constructor // type here. declare_method(string_type, "new", false, - make_function_typespec({"symbol", "type", "int", "string"}, "_type_")); + make_function_typespec({"symbol", "type", "int", "string"}, "_type_"), false); // FUNCTION builtin_structure_inherit(function_type); @@ -1037,7 +1060,7 @@ void TypeSystem::add_builtin_types() { // todo builtin_structure_inherit(array_type); declare_method(array_type, "new", false, - make_function_typespec({"symbol", "type", "type", "int"}, "_type_")); + make_function_typespec({"symbol", "type", "type", "int"}, "_type_"), false); // array has: number, number, type add_field_to_type(array_type, "length", make_typespec("int32")); add_field_to_type(array_type, "allocated-length", make_typespec("int32")); @@ -1047,7 +1070,7 @@ void TypeSystem::add_builtin_types() { // pair pair_type->override_offset(2); declare_method(pair_type, "new", false, - make_function_typespec({"symbol", "type", "object", "object"}, "_type_")); + make_function_typespec({"symbol", "type", "object", "object"}, "_type_"), false); add_field_to_type(pair_type, "car", make_typespec("object")); add_field_to_type(pair_type, "cdr", make_typespec("object")); @@ -1065,7 +1088,7 @@ void TypeSystem::add_builtin_types() { add_field_to_type(file_stream_type, "name", make_typespec("string")); add_field_to_type(file_stream_type, "file", make_typespec("uint32")); declare_method(file_stream_type, "new", false, - make_function_typespec({"symbol", "type", "string", "basic"}, "_type_")); + make_function_typespec({"symbol", "type", "string", "basic"}, "_type_"), false); } /*! @@ -1590,6 +1613,10 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const { methods_string.append(":no-virtual "); } + if (info.overrides_method_type_of_parent) { + methods_string.append(":replace "); + } + methods_string.append(fmt::format("{})\n ", info.id)); } diff --git a/common/type_system/TypeSystem.h b/common/type_system/TypeSystem.h index 7a3c685e8..a66da2de1 100644 --- a/common/type_system/TypeSystem.h +++ b/common/type_system/TypeSystem.h @@ -159,12 +159,13 @@ class TypeSystem { MethodInfo declare_method(const std::string& type_name, const std::string& method_name, bool no_virtual, - const TypeSpec& ts); + const TypeSpec& ts, + bool override_type); MethodInfo declare_method(Type* type, const std::string& method_name, bool no_virtual, - const TypeSpec& ts); - + const TypeSpec& ts, + bool override_type); MethodInfo define_method(const std::string& type_name, const std::string& method_name, const TypeSpec& ts); diff --git a/common/type_system/deftype.cpp b/common/type_system/deftype.cpp index d2653fddf..c87875714 100644 --- a/common/type_system/deftype.cpp +++ b/common/type_system/deftype.cpp @@ -188,12 +188,18 @@ void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def obj = cdr(obj); bool no_virtual = false; + bool replace_method = false; if (!obj->is_empty_list() && car(obj).is_symbol(":no-virtual")) { obj = cdr(obj); no_virtual = true; } + if (!obj->is_empty_list() && car(obj).is_symbol(":replace")) { + obj = cdr(obj); + replace_method = true; + } + int id = -1; if (!obj->is_empty_list() && car(obj).is_int()) { auto& id_obj = car(obj); @@ -212,7 +218,8 @@ void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def }); function_typespec.add_arg(parse_typespec(type_system, return_type)); - auto info = type_system->declare_method(type, method_name, no_virtual, function_typespec); + auto info = type_system->declare_method(type, method_name, no_virtual, function_typespec, + replace_method); // check the method assert if (id != -1) { diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 9f786e240..4a9ed5f51 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -15,6 +15,7 @@ add_library( analysis/type_analysis.cpp analysis/variable_naming.cpp + data/dir_tpages.cpp data/game_count.cpp data/game_text.cpp data/StrFileReader.cpp diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index c14d07ba3..e9fcdfb2c 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -15,6 +15,7 @@ #include "decompiler/data/tpage.h" #include "decompiler/data/game_text.h" #include "decompiler/data/StrFileReader.h" +#include "decompiler/data/dir_tpages.h" #include "decompiler/data/game_count.h" #include "LinkedObjectFileCreation.h" #include "decompiler/config.h" @@ -556,20 +557,34 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) { lg::info(" Total {:.3f} ms\n", timer.getMs()); } -void ObjectFileDB::process_tpages() { +std::string ObjectFileDB::process_tpages() { lg::info("- Finding textures in tpages..."); std::string tpage_string = "tpage-"; int total = 0, success = 0; + int tpage_dir_count = 0; Timer timer; + + std::string result; for_each_obj([&](ObjectFileData& data) { if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) { auto statistics = process_tpage(data); total += statistics.total_textures; success += statistics.successful_textures; + } else if (data.name_in_dgo == "dir-tpages") { + result = process_dir_tpages(data).to_source(); + tpage_dir_count++; } }); + + assert(tpage_dir_count <= 1); + + if (tpage_dir_count == 0) { + lg::warn("Did not find tpage-dir."); + } + lg::info("Processed {} / {} textures {:.2f}% in {:.2f} ms", success, total, 100.f * float(success) / float(total), timer.getMs()); + return result; } std::string ObjectFileDB::process_game_text_files() { diff --git a/decompiler/ObjectFile/ObjectFileDB.h b/decompiler/ObjectFile/ObjectFileDB.h index a52207590..b4e842749 100644 --- a/decompiler/ObjectFile/ObjectFileDB.h +++ b/decompiler/ObjectFile/ObjectFileDB.h @@ -84,7 +84,7 @@ class ObjectFileDB { std::string ir2_final_out(ObjectFileData& data, const std::unordered_set& skip_functions = {}); - void process_tpages(); + std::string process_tpages(); std::string process_game_count_file(); std::string process_game_text_files(); diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index 16de45526..fc16c3348 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -4272,6 +4272,7 @@ :size-assert #x80 :flag-assert #xf00000080 (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (remove-from-heap (_type_ kheap) _type_ 9) (get-leftover-block-count (_type_ int int) int 10) (dummy-11 () none 11) @@ -4313,6 +4314,7 @@ (entries texture-page-dir-entry 1 :inline) ) (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (dummy-9 (_type_ kheap) int 9) ) :flag-assert #xa00000014 @@ -11265,7 +11267,7 @@ :size-assert #x190 :flag-assert #x1400000190 (:methods - (relocate (_type_ int) _type_ 7) + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (dummy-17 () none 17) (dummy-18 (_type_) none 18) (dummy-19 (_type_) none 19) diff --git a/decompiler/config/jak1_ntsc_black_label.jsonc b/decompiler/config/jak1_ntsc_black_label.jsonc index d3c4a896a..64fea8785 100644 --- a/decompiler/config/jak1_ntsc_black_label.jsonc +++ b/decompiler/config/jak1_ntsc_black_label.jsonc @@ -3,7 +3,7 @@ // if you want to filter to only some object names. // it will make the decompiler much faster. - "allowed_objects": ["vol-h"], + "allowed_objects": [], //////////////////////////// // CODE ANALYSIS OPTIONS diff --git a/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc b/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc index 469ffdcf3..31893d949 100644 --- a/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc @@ -787,27 +787,27 @@ ], //"bg": [[[25, 52], "a0", "string"]], - "(anon-function 29 process-drawable)": [[[0, 99999], "s6", "process-drawable"]], - "ja-done?": [[[0, 99999], "s6", "process-drawable"]], - "ja-min?": [[[0, 99999], "s6", "process-drawable"]], - "ja-max?": [[[0, 99999], "s6", "process-drawable"]], - "ja-num-frames": [[[0, 99999], "s6", "process-drawable"]], - "ja-frame-num": [[[0, 99999], "s6", "process-drawable"]], - "ja-aframe-num": [[[0, 99999], "s6", "process-drawable"]], - "ja-aframe": [[[0, 99999], "s6", "process-drawable"]], - "ja-step": [[[0, 99999], "s6", "process-drawable"]], - "ja-channel-set!": [[[0, 99999], "s6", "process-drawable"]], - "ja-channel-push!": [[[0, 99999], "s6", "process-drawable"]], - "ja-group-size": [[[0, 99999], "s6", "process-drawable"]], - "ja-eval": [[[0, 99999], "s6", "process-drawable"]], - "ja-blend-eval": [[[0, 99999], "s6", "process-drawable"]], - "ja-post": [[[0, 99999], "s6", "process-drawable"], [54, "a1", "process"]], - "transform-post": [[[0, 99999], "s6", "process-drawable"]], - "rider-trans": [[[0, 99999], "s6", "process-drawable"]], - "rider-post": [[[0, 99999], "s6", "process-drawable"]], - "pusher-post": [[[0, 99999], "s6", "process-drawable"]], - "process-drawable-delay-player": [[[0, 99999], "s6", "process-drawable"]], - "init-target": [[[0, 99999], "s6", "target"]], + "(anon-function 29 process-drawable)": [[[0, 999], "s6", "process-drawable"]], + "ja-done?": [[[0, 999], "s6", "process-drawable"]], + "ja-min?": [[[0, 999], "s6", "process-drawable"]], + "ja-max?": [[[0, 999], "s6", "process-drawable"]], + "ja-num-frames": [[[0, 999], "s6", "process-drawable"]], + "ja-frame-num": [[[0, 999], "s6", "process-drawable"]], + "ja-aframe-num": [[[0, 999], "s6", "process-drawable"]], + "ja-aframe": [[[0, 999], "s6", "process-drawable"]], + "ja-step": [[[0, 999], "s6", "process-drawable"]], + "ja-channel-set!": [[[0, 999], "s6", "process-drawable"]], + "ja-channel-push!": [[[0, 999], "s6", "process-drawable"]], + "ja-group-size": [[[0, 999], "s6", "process-drawable"]], + "ja-eval": [[[0, 999], "s6", "process-drawable"]], + "ja-blend-eval": [[[0, 999], "s6", "process-drawable"]], + "ja-post": [[[0, 999], "s6", "process-drawable"], [54, "a1", "process"]], + "transform-post": [[[0, 999], "s6", "process-drawable"]], + "rider-trans": [[[0, 999], "s6", "process-drawable"]], + "rider-post": [[[0, 999], "s6", "process-drawable"]], + "pusher-post": [[[0, 999], "s6", "process-drawable"]], + "process-drawable-delay-player": [[[0, 999], "s6", "process-drawable"]], + "init-target": [[[0, 999], "s6", "target"]], "upload-generic-shrub": [ [[3, 13], "t0", "dma-packet"], diff --git a/decompiler/config/jak1_ntsc_black_label/var_names.jsonc b/decompiler/config/jak1_ntsc_black_label/var_names.jsonc index 3657ba19e..feaa762df 100644 --- a/decompiler/config/jak1_ntsc_black_label/var_names.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/var_names.jsonc @@ -1252,6 +1252,19 @@ } }, + "texture-page-dir-inspect": { + "args":["dir", "mode"], + "vars": { + "v1-0":"pool", + "s4-0":"level-idx", + "a1-3":"lev", + "s4-1":"entry-idx", + "s3-0":"entry-page", + "s2-0":"entry-link", + "s1-0":"entry-list-length" + } + }, + "texture-page-size-check": { "args": ["pool", "level", "hide-prints"], "vars": { diff --git a/decompiler/data/dir_tpages.cpp b/decompiler/data/dir_tpages.cpp new file mode 100644 index 000000000..9d967d3fd --- /dev/null +++ b/decompiler/data/dir_tpages.cpp @@ -0,0 +1,51 @@ +#include "third-party/fmt/core.h" +#include "decompiler/ObjectFile/ObjectFileDB.h" +#include "dir_tpages.h" + +namespace decompiler { +std::string DirTpageResult::to_source() const { + std::string result; + int i = 0; + for (auto len : lengths) { + result += fmt::format(" {:6s} ;; entry {}\n", fmt::format("#x{:x}", len), i); + i++; + } + return result; +} + +DirTpageResult process_dir_tpages(ObjectFileData& data) { + DirTpageResult result; + + auto& words = data.linked_data.words_by_seg.at(0); + + int word_idx = 0; + // first is type + assert(words.at(word_idx).kind == LinkedWord::TYPE_PTR); + assert(words.at(word_idx).symbol_name == "texture-page-dir"); + word_idx++; + // next is length + assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA); + int dir_length = words.at(word_idx).data; + fmt::print("length: {}\n", dir_length); + word_idx++; + + for (int i = 0; i < dir_length; i++) { + assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA); + u32 entry = words.at(word_idx).data; + assert((entry & 0xffff7000) == 0); // 7 checks for sign bit. + word_idx++; + result.lengths.push_back(entry & 0xffff); + + assert(words.at(word_idx).kind == LinkedWord::SYM_PTR); + assert(words.at(word_idx).symbol_name == "#f"); + word_idx++; + assert(words.at(word_idx).kind == LinkedWord::SYM_PTR); + assert(words.at(word_idx).symbol_name == "#f"); + word_idx++; + } + + assert(word_idx == (int)words.size()); + + return result; +} +} // namespace decompiler diff --git a/decompiler/data/dir_tpages.h b/decompiler/data/dir_tpages.h new file mode 100644 index 000000000..ddf6034f8 --- /dev/null +++ b/decompiler/data/dir_tpages.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace decompiler { + +struct ObjectFileData; + +struct DirTpageResult { + std::vector lengths; + + std::string to_source() const; +}; + +DirTpageResult process_dir_tpages(ObjectFileData& data); +} // namespace decompiler \ No newline at end of file diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 4e08ff4b4..a904a604d 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -104,7 +104,8 @@ int main(int argc, char** argv) { } if (config.process_tpages) { - db.process_tpages(); + auto result = db.process_tpages(); + file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), result); } if (config.process_game_count) { diff --git a/docs/markdown/progress-notes/changelog.md b/docs/markdown/progress-notes/changelog.md index 7d74e5c08..455bfa04a 100644 --- a/docs/markdown/progress-notes/changelog.md +++ b/docs/markdown/progress-notes/changelog.md @@ -171,4 +171,5 @@ - Forward declared basics can be used in more places - You can now set a field which has a forward declared structure or basic type - `cdr` now returns an object of type `pair`. -- `lambda`s can now be used inside of a static object definition. \ No newline at end of file +- `lambda`s can now be used inside of a static object definition. +- Methods can now be `:replace`d to override their type from their parent. Use this with extreme care. \ No newline at end of file diff --git a/goal_src/engine/gfx/texture-h.gc b/goal_src/engine/gfx/texture-h.gc index 7ee5e1c30..90bd5e68b 100644 --- a/goal_src/engine/gfx/texture-h.gc +++ b/goal_src/engine/gfx/texture-h.gc @@ -133,6 +133,7 @@ :size-assert #x80 :flag-assert #xf00000080 (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (remove-from-heap (_type_ kheap) _type_ 9) (get-leftover-block-count (_type_ int int) int 10) (dummy-11 () none 11) @@ -174,8 +175,9 @@ (entries texture-page-dir-entry 1 :inline) ) (:methods - (dummy-9 (_type_ kheap) int 9) - ) + (relocate (_type_ kheap (pointer uint8)) none :replace 7) + (dummy-9 (_type_ kheap) int 9) + ) :flag-assert #xa00000014 ) diff --git a/goal_src/engine/gfx/texture.gc b/goal_src/engine/gfx/texture.gc index 59b4afbf4..a139aaabe 100644 --- a/goal_src/engine/gfx/texture.gc +++ b/goal_src/engine/gfx/texture.gc @@ -13,7 +13,7 @@ ;; (common textures for Jak, crates, etc) ;; - "global common" textures that are uploaded as needed, but aren't level specific ;; (the start menu hud) -;; - "level common" textures that are uploaded as needed +;; - "level common" textures that are uploaded as needed, and are specific to a level. ;; - "level near" textures. Part of these textures remain in VRAM always (don't live in RAM) and ;; part is reuploaded as needed. This is for TFRAG stuff near the camera @@ -1662,7 +1662,120 @@ (define *texture-pool* (new 'global 'texture-pool)) ;; temp hack for loading: -(defmethod relocate texture-page ((obj texture-page) (offset int)) +(defmethod relocate texture-page ((obj texture-page) (heap kheap) (ptr (pointer uint8))) (format #t "HACK! removing texture page ~A from level heap~%" (-> obj name)) - (remove-from-heap obj loading-level) + (remove-from-heap obj loading-level) + (none) + ) + + +(defmethod asize-of texture-page-dir ((obj texture-page-dir)) + (the-as int + (+ (-> texture-page-dir size) (the-as uint (* 12 (+ (-> obj length) -1)))) + ) + ) + +(defmethod length texture-page-dir ((obj texture-page-dir)) + (-> obj length) + ) + +(defmethod relocate texture-page-dir ((obj texture-page-dir) (arg0 kheap) (arg1 (pointer uint8))) + (set! *texture-page-dir* obj) + (none) + ) + +(defun-debug texture-page-dir-inspect ((dir texture-page-dir) (mode symbol)) + (format #t "[~8x] ~A~%" dir (-> dir type)) + (let ((pool *texture-pool*)) + (format + #t + "~Ttexture pool (~DK used, ~DK free)~%" + (/ (- (-> pool cur) (-> pool top)) 256) + (/ (- #xa0000 (-> pool cur)) 256) + ) + ) + (dotimes (level-idx (-> *level* length)) + (let ((lev (-> *level* level level-idx))) + (if (= (-> lev status) 'active) + (texture-page-size-check *texture-pool* lev #f) + ) + ) + ) + (format #t "~Tlength: ~D~%" (-> dir length)) + (format #t "~Tdata[~D]: @ #x~X~%" (-> dir length) (-> dir entries)) + (dotimes (entry-idx (-> dir length)) + (let ((entry-page (-> dir entries entry-idx page)) + (entry-link (-> dir entries entry-idx link)) + ) + (cond + (entry-page + (format #t "~T [~3D] loaded ~S ~A~%" entry-idx (if entry-link + " linked" + "unlinked" + ) + entry-page + ) + ) + (else + (if (= mode 'full) + (format + #t + "~T [~3D] unloaded ~S #~%" + entry-idx + (if entry-link + " linked" + "unlinked" + ) + (-> dir entries entry-idx length) + ) + ) + ) + ) + (when (and (or entry-page entry-link) mode) + (dotimes (entry-list-length (-> dir entries entry-idx length)) + (cond + ((not entry-link) + (format #t "~T [~3D] unlinked" entry-list-length) + ) + ((zero? (-> entry-link next entry-list-length shader)) + (format #t "~T [~3D] UNUSED " entry-list-length) + ) + (else + (let ((t9-9 format) + (a0-12 #t) + (a1-10 "~T [~3D] ~3D links ") + (a2-11 entry-list-length) + (a3-7 0) + ) + (let + ((v1-40 + (the-as object (* (-> entry-link next entry-list-length shader) 16)) + ) + ) + (while (nonzero? (the-as int v1-40)) + (nop!) + (+! a3-7 1) + (set! v1-40 (* (-> (the-as adgif-shader v1-40) next shader) 16)) + ) + ) + (t9-9 a0-12 a1-10 a2-11 a3-7) + ) + ) + ) + (cond + ((not entry-page) + (format #t " unloaded~%") + ) + ((not (-> entry-page data entry-list-length)) + (format #t " empty~%") + ) + (else + (format #t " ~A~%" (-> entry-page data entry-list-length)) + ) + ) + ) + ) + ) + ) + (none) ) diff --git a/goal_src/engine/gfx/vis/bsp-h.gc b/goal_src/engine/gfx/vis/bsp-h.gc index c9b4c946e..3ee7d4083 100644 --- a/goal_src/engine/gfx/vis/bsp-h.gc +++ b/goal_src/engine/gfx/vis/bsp-h.gc @@ -53,11 +53,11 @@ :size-assert #x190 :flag-assert #x1400000190 (:methods - (relocate (_type_ int) _type_ 7) - (dummy-17 () none 17) - (dummy-18 (_type_) none 18) - (dummy-19 (_type_) none 19) - ) + (relocate (_type_ kheap (pointer uint8)) none :replace 7) + (dummy-17 () none 17) + (dummy-18 (_type_) none 18) + (dummy-19 (_type_) none 19) + ) ) (deftype game-level (basic) diff --git a/goal_src/engine/level/level.gc b/goal_src/engine/level/level.gc index 143ac1138..edbeb9d1e 100644 --- a/goal_src/engine/level/level.gc +++ b/goal_src/engine/level/level.gc @@ -93,7 +93,7 @@ ) ;; relocate bsp-header -(defmethod relocate bsp-header ((obj bsp-header) (arg0 int)) +(defmethod relocate bsp-header ((obj bsp-header) (dest-heap kheap) (name (pointer uint8))) (let ((s5-0 (-> *level* unknown-level-2))) (if s5-0 (cond @@ -130,6 +130,7 @@ ) ) ) + (none) ) (defmethod dummy-24 level ((obj level)) diff --git a/goal_src/goal-lib.gc b/goal_src/goal-lib.gc index 8d7dc1e54..3c01a6d46 100644 --- a/goal_src/goal-lib.gc +++ b/goal_src/goal-lib.gc @@ -60,6 +60,7 @@ `(begin (asm-data-file game-text "assets/game_text.txt") (asm-data-file game-count "assets/game_count.txt") + (asm-data-file dir-tpages "assets/tpage-dir.txt") ) ) diff --git a/goalc/CMakeLists.txt b/goalc/CMakeLists.txt index c2781ebdc..c5b6a9c00 100644 --- a/goalc/CMakeLists.txt +++ b/goalc/CMakeLists.txt @@ -27,6 +27,7 @@ add_library(compiler compiler/compilation/Static.cpp compiler/Util.cpp data_compiler/game_text.cpp + data_compiler/dir_tpages.cpp data_compiler/DataObjectGenerator.cpp data_compiler/game_count.cpp debugger/Debugger.cpp diff --git a/goalc/compiler/StaticObject.cpp b/goalc/compiler/StaticObject.cpp index 6df8ef3d2..967d31de6 100644 --- a/goalc/compiler/StaticObject.cpp +++ b/goalc/compiler/StaticObject.cpp @@ -145,10 +145,10 @@ void StaticStructure::add_type_record(std::string name, int offset) { } void StaticStructure::add_function_record(const FunctionEnv* function, int offset) { - FunctionRecord rec; - rec.func = function; - rec.offset_in_this = offset; - functions.push_back(rec); + FunctionRecord frec; + frec.func = function; + frec.offset_in_this = offset; + functions.push_back(frec); } /////////////////// diff --git a/goalc/compiler/compilation/CompilerControl.cpp b/goalc/compiler/compilation/CompilerControl.cpp index c5cbac077..2a945be96 100644 --- a/goalc/compiler/compilation/CompilerControl.cpp +++ b/goalc/compiler/compilation/CompilerControl.cpp @@ -11,6 +11,7 @@ #include "common/util/FileUtil.h" #include "goalc/data_compiler/game_text.h" #include "goalc/data_compiler/game_count.h" +#include "goalc/data_compiler/dir_tpages.h" #include "common/goos/ReplUtils.h" #include #include @@ -67,6 +68,8 @@ Val* Compiler::compile_asm_data_file(const goos::Object& form, const goos::Objec compile_game_text(as_string(args.unnamed.at(1))); } else if (kind == "game-count") { compile_game_count(as_string(args.unnamed.at(1))); + } else if (kind == "dir-tpages") { + compile_dir_tpages(as_string(args.unnamed.at(1))); } else { throw_compiler_error(form, "The option {} was not recognized for asm-data-file.", kind); } diff --git a/goalc/data_compiler/DataObjectGenerator.cpp b/goalc/data_compiler/DataObjectGenerator.cpp index 6c463dd95..9d5545d4e 100644 --- a/goalc/data_compiler/DataObjectGenerator.cpp +++ b/goalc/data_compiler/DataObjectGenerator.cpp @@ -194,8 +194,24 @@ std::vector DataObjectGenerator::generate_link_table() { } push_variable_length_integer(0, &link); - // todo symbols - assert(m_symbol_links.empty()); + for (auto& sl : m_symbol_links) { + // insert name. first char won't have the highest bit set + for (auto c : sl.first) { + link.push_back(c); + } + link.push_back(0); + std::sort(sl.second.begin(), sl.second.end()); + int prev = 0; + + for (auto& x : sl.second) { + int diff = x - prev; + assert(diff >= 0); + push_better_variable_length_integer(diff * 4, &link); + m_words.at(x) = 0xffffffff; + prev = x; + } + link.push_back(0); + } // types for (auto& tl : m_type_links) { diff --git a/goalc/data_compiler/dir_tpages.cpp b/goalc/data_compiler/dir_tpages.cpp new file mode 100644 index 000000000..37dca4041 --- /dev/null +++ b/goalc/data_compiler/dir_tpages.cpp @@ -0,0 +1,37 @@ +#include "dir_tpages.h" + +#include "DataObjectGenerator.h" +#include "common/goos/Reader.h" +#include "common/goos/ParseHelpers.h" +#include "common/util/FileUtil.h" + +void compile_dir_tpages(const std::string& filename) { + std::vector lengths; + + printf("[Build tpage dir] %s\n", filename.c_str()); + + goos::Reader reader; + auto code = reader.read_from_file({filename}); + std::string err; + + goos::for_each_in_list(code.as_pair()->cdr, [&](const goos::Object& obj) { + if (!obj.is_int()) { + throw std::runtime_error("Invalid tpage dir file"); + } + lengths.push_back(obj.as_int()); + }); + + DataObjectGenerator gen; + gen.add_type_tag("texture-page-dir"); + gen.add_word(lengths.size()); + for (auto len : lengths) { + gen.add_word(len); + gen.add_symbol_link("#f"); + gen.add_symbol_link("#f"); + } + + auto data = gen.generate_v4(); + file_util::create_dir_if_needed(file_util::get_file_path({"out", "obj"})); + file_util::write_binary_file(file_util::get_file_path({"out", "obj", "dir-tpages.go"}), + data.data(), data.size()); +} \ No newline at end of file diff --git a/goalc/data_compiler/dir_tpages.h b/goalc/data_compiler/dir_tpages.h new file mode 100644 index 000000000..faf291917 --- /dev/null +++ b/goalc/data_compiler/dir_tpages.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +void compile_dir_tpages(const std::string& filename); \ No newline at end of file diff --git a/goalc/data_compiler/game_text.cpp b/goalc/data_compiler/game_text.cpp index d938671bc..92fad52a6 100644 --- a/goalc/data_compiler/game_text.cpp +++ b/goalc/data_compiler/game_text.cpp @@ -173,6 +173,9 @@ std::vector> parse(const goos::Object& data */ void compile(const std::vector>& text, const std::string& group_name) { + if (text.empty()) { + return; + } // get all text ID's we know std::vector add_order; add_order.reserve(text.front().size()); diff --git a/out/hash.md5 b/out/hash.md5 index f4e9cb0e6..88c760d42 100644 --- a/out/hash.md5 +++ b/out/hash.md5 @@ -6,3 +6,4 @@ abcc25e5d7469dd6a572dc53dbb9671c iso/3COMMON.TXT 5d62de2c78b4cf102b9a78f3aa96c8c9 iso/5COMMON.TXT 9495f80955e6782513fe12f6539fc8e7 iso/6COMMON.TXT 9765bdc3add08cb06fd3e87ebd5713aa obj/game-cnt.go +5033811c685e1bde4411d396b1d4ffba obj/dir-tpages.go diff --git a/test/decompiler/reference/engine/gfx/texture-h_REF.gc b/test/decompiler/reference/engine/gfx/texture-h_REF.gc index 3c71e58aa..54ec91640 100644 --- a/test/decompiler/reference/engine/gfx/texture-h_REF.gc +++ b/test/decompiler/reference/engine/gfx/texture-h_REF.gc @@ -190,6 +190,7 @@ :size-assert #x80 :flag-assert #xf00000080 (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (remove-from-heap (_type_ kheap) _type_ 9) (get-leftover-block-count (_type_ int int) int 10) (dummy-11 () none 11) @@ -271,6 +272,7 @@ :size-assert #x14 :flag-assert #xa00000014 (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (dummy-9 (_type_ kheap) int 9) ) ) diff --git a/test/decompiler/reference/engine/gfx/vis/bsp-h_REF.gc b/test/decompiler/reference/engine/gfx/vis/bsp-h_REF.gc index 582712df9..369f34c3a 100644 --- a/test/decompiler/reference/engine/gfx/vis/bsp-h_REF.gc +++ b/test/decompiler/reference/engine/gfx/vis/bsp-h_REF.gc @@ -60,6 +60,7 @@ :size-assert #x190 :flag-assert #x1400000190 (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) (dummy-18 (_type_) none 18) (dummy-19 (_type_) none 19) ) diff --git a/test/goalc/source_templates/with_game/test-method-replace.gc b/test/goalc/source_templates/with_game/test-method-replace.gc new file mode 100644 index 000000000..fe707fe57 --- /dev/null +++ b/test/goalc/source_templates/with_game/test-method-replace.gc @@ -0,0 +1,15 @@ +(deftype type-with-weird-relocate (basic) + ((foo int)) + (:methods + (relocate (_type_ kheap (pointer uint8)) none :replace 7) + ) + ) + +(defmethod relocate type-with-weird-relocate ((obj type-with-weird-relocate) (dest-heap kheap) (name (pointer uint8))) + (format #t "relocate! foo: ~D heap: ~D name: ~D~%" (-> obj foo) dest-heap name) + (none) + ) + +(let ((obj (new 'static 'type-with-weird-relocate :foo 123))) + (relocate obj (the kheap 1) (the (pointer uint8) 2)) + ) diff --git a/test/goalc/test_with_game.cpp b/test/goalc/test_with_game.cpp index 812055015..18080ecb1 100644 --- a/test/goalc/test_with_game.cpp +++ b/test/goalc/test_with_game.cpp @@ -775,6 +775,11 @@ TEST_F(WithGameTests, StaticLambda) { runner.run_static_test(env, testCategory, "test-static-lambda.gc", {"Add: 30 sub: -10\n0\n"}); } +TEST_F(WithGameTests, MethodReplace) { + runner.run_static_test(env, testCategory, "test-method-replace.gc", + {"relocate! foo: 123 heap: 1 name: 2\n0\n"}); +} + TEST(TypeConsistency, TypeConsistency) { Compiler compiler; compiler.enable_throw_on_redefines(); diff --git a/test/test_type_system.cpp b/test/test_type_system.cpp index 40ff67278..d08a02c85 100644 --- a/test/test_type_system.cpp +++ b/test/test_type_system.cpp @@ -231,27 +231,28 @@ TEST(TypeSystem, AddMethodAndLookupMethod) { ts.add_builtin_types(); auto parent_info = ts.declare_method(ts.lookup_type("structure"), "test-method-1", false, - ts.make_function_typespec({"integer"}, "string")); + ts.make_function_typespec({"integer"}, "string"), false); // when trying to add the same method to a child, should return the parent's method auto child_info_same = ts.declare_method(ts.lookup_type("basic"), "test-method-1", false, - ts.make_function_typespec({"integer"}, "string")); + ts.make_function_typespec({"integer"}, "string"), false); EXPECT_EQ(parent_info.id, child_info_same.id); EXPECT_EQ(parent_info.id, GOAL_MEMUSAGE_METHOD + 1); // any amount of fiddling with method types should cause an error EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false, - ts.make_function_typespec({"integer"}, "integer"))); + ts.make_function_typespec({"integer"}, "integer"), false)); EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false, - ts.make_function_typespec({}, "string"))); + ts.make_function_typespec({}, "string"), false)); EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false, - ts.make_function_typespec({"integer", "string"}, "string"))); + ts.make_function_typespec({"integer", "string"}, "string"), + false)); EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false, - ts.make_function_typespec({"string"}, "string"))); + ts.make_function_typespec({"string"}, "string"), false)); ts.declare_method(ts.lookup_type("basic"), "test-method-2", false, - ts.make_function_typespec({"integer"}, "string")); + ts.make_function_typespec({"integer"}, "string"), false); EXPECT_EQ(parent_info.id, ts.lookup_method("basic", "test-method-1").id); EXPECT_EQ(parent_info.id, ts.lookup_method("structure", "test-method-1").id); @@ -274,10 +275,10 @@ TEST(TypeSystem, NewMethod) { ts.add_builtin_types(); ts.add_type("test-1", std::make_unique("basic", "test-1", false, 0)); ts.declare_method(ts.lookup_type("test-1"), "new", false, - ts.make_function_typespec({"symbol", "string"}, "test-1")); + ts.make_function_typespec({"symbol", "string"}, "test-1"), false); ts.add_type("test-2", std::make_unique("test-1", "test-2", false, 0)); ts.declare_method(ts.lookup_type("test-2"), "new", false, - ts.make_function_typespec({"symbol", "string", "symbol"}, "test-2")); + ts.make_function_typespec({"symbol", "string", "symbol"}, "test-2"), false); EXPECT_EQ(ts.lookup_method("test-1", "new").type.print(), "(function symbol string test-1)"); EXPECT_EQ(ts.lookup_method("test-2", "new").type.print(), @@ -296,7 +297,7 @@ TEST(TypeSystem, MethodSubstitute) { ts.add_builtin_types(); ts.add_type("test-1", std::make_unique("basic", "test-1", false, 0)); ts.declare_method(ts.lookup_type("test-1"), "new", false, - ts.make_function_typespec({"symbol", "string", "_type_"}, "_type_")); + ts.make_function_typespec({"symbol", "string", "_type_"}, "_type_"), false); auto final_type = ts.lookup_method("test-1", "new").type.substitute_for_method_call("test-1"); EXPECT_EQ(final_type.print(), "(function symbol string test-1 test-1)"); diff --git a/tools/MemoryDumpTool/main.cpp b/tools/MemoryDumpTool/main.cpp index f26825350..ceb9faf44 100644 --- a/tools/MemoryDumpTool/main.cpp +++ b/tools/MemoryDumpTool/main.cpp @@ -249,7 +249,6 @@ void inspect_basics(const Ram& ram, bool goal_array = field.type() == TypeSpec("array", {TypeSpec("basic")}); std::unordered_map type_frequency; - int array_max_elts = 0; for (auto base_addr : basics.at(name)) { for (int elt_idx = 0; elt_idx < array_size; elt_idx++) {