Game-Decompilation/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project.git synced 2024-11-23 06:09:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support dir tpages (#671)

Browse Source 
* support dir tpages

* fix warnings and bad return

* one more try

* revive the offline test script

* fix this null bug
This commit is contained in:
water111 2021-07-02 14:50:58 -04:00 committed by GitHub
parent b96d865e2b
commit 6366068bc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 477 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
common/type_system/Type.cpp
View File

@ -32,7 +32,8 @@ std::string reg_kind_to_string(RegClass kind) {
*/
bool MethodInfo::operator==(const MethodInfo& other) const {
return id == other.id && name == other.name && type == other.type &&
defined_in_type == other.defined_in_type && other.no_virtual == no_virtual;
defined_in_type == other.defined_in_type && other.no_virtual == no_virtual &&
other.overrides_method_type_of_parent == overrides_method_type_of_parent;
}
std::string MethodInfo::diff(const MethodInfo& other) const {
@ -56,6 +57,11 @@ std::string MethodInfo::diff(const MethodInfo& other) const {
if (no_virtual != other.no_virtual) {
result += fmt::format("no_virtual: {} vs. {}\n", no_virtual, other.no_virtual);
}
if (overrides_method_type_of_parent != other.overrides_method_type_of_parent) {
result += fmt::format("overrides_method_type_of_parent: {} vs. {}\n",
overrides_method_type_of_parent, other.overrides_method_type_of_parent);
}
return result;
}
@ -311,9 +317,11 @@ bool Type::get_my_method(int id, MethodInfo* out) const {
* defined specifically for this type or not.
*/
bool Type::get_my_last_method(MethodInfo* out) const {
if (!m_methods.empty()) {
*out = m_methods.back();
return true;
for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
if (!it->overrides_method_type_of_parent) {
*out = *it;
return true;
}
}
return false;
}
@ -334,9 +342,13 @@ bool Type::get_my_new_method(MethodInfo* out) const {
* Add a method defined specifically for this type.
*/
const MethodInfo& Type::add_method(const MethodInfo& info) {
if (!m_methods.empty()) {
assert(m_methods.back().id + 1 == info.id);
for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
if (!it->overrides_method_type_of_parent) {
assert(it->id + 1 == info.id);
break;
}
}
m_methods.push_back(info);
return m_methods.back();
}

1
common/type_system/Type.h
View File

@ -21,6 +21,7 @@ struct MethodInfo {
TypeSpec type;
std::string defined_in_type;
bool no_virtual = false;
bool overrides_method_type_of_parent = false;
bool operator==(const MethodInfo& other) const;
bool operator!=(const MethodInfo& other) const { return !((*this) == other); }

15
common/type_system/TypeFieldLookup.cpp
View File

@ -81,6 +81,13 @@ void try_reverse_lookup_other(const FieldReverseLookupInput& input,
FieldReverseMultiLookupOutput* output,
int max_count);
std::vector<FieldReverseLookupOutput::Token> parent_to_vector(const ReverseLookupNode* parent) {
if (!parent) {
return {};
}
return parent->to_vector();
}
/*!
* Handle a dereference of a pointer/boxed array. This can be:
* - just dereferencing a pointer
@ -188,7 +195,7 @@ void try_reverse_lookup_array_like(const FieldReverseLookupInput& input,
// also just return the array
if (elt_idx == 0) {
if (boxed_array) {
auto vec = parent->to_vector();
auto vec = parent_to_vector(parent);
FieldReverseLookupOutput::Token tok;
tok.kind = FieldReverseLookupOutput::Token::Kind::FIELD;
tok.field_score = 0.0; // don't bother
@ -196,7 +203,7 @@ void try_reverse_lookup_array_like(const FieldReverseLookupInput& input,
vec.push_back(tok);
output->results.emplace_back(false, array_data_type, vec);
} else {
auto parent_vector = parent->to_vector();
auto parent_vector = parent_to_vector(parent);
if (!parent_vector.empty()) {
output->results.emplace_back(false, input.base_type, parent_vector);
}
@ -280,9 +287,9 @@ void try_reverse_lookup_inline_array(const FieldReverseLookupInput& input,
// can we just return the array?
if (expected_offset_into_elt == offset_into_elt && !input.deref.has_value() && elt_idx == 0) {
auto parent_vec = parent->to_vector();
auto parent_vec = parent_to_vector(parent);
if (!parent_vec.empty()) {
output->results.emplace_back(false, input.base_type, parent->to_vector());
output->results.emplace_back(false, input.base_type, parent_to_vector(parent));
}
if ((int)output->results.size() >= max_count) {

119
common/type_system/TypeSystem.cpp
View File

@ -477,8 +477,10 @@ int TypeSystem::get_load_size_allow_partial_def(const TypeSpec& ts) const {
MethodInfo TypeSystem::declare_method(const std::string& type_name,
const std::string& method_name,
bool no_virtual,
const TypeSpec& ts) {
return declare_method(lookup_type(make_typespec(type_name)), method_name, no_virtual, ts);
const TypeSpec& ts,
bool override_type) {
return declare_method(lookup_type(make_typespec(type_name)), method_name, no_virtual, ts,
override_type);
}
/*!
@ -493,8 +495,12 @@ MethodInfo TypeSystem::declare_method(const std::string& type_name,
MethodInfo TypeSystem::declare_method(Type* type,
const std::string& method_name,
bool no_virtual,
const TypeSpec& ts) {
const TypeSpec& ts,
bool override_type) {
if (method_name == "new") {
if (override_type) {
throw_typesystem_error("Cannot use :replace option with a new method.");
}
return add_new_method(type, ts);
}
@ -502,34 +508,48 @@ MethodInfo TypeSystem::declare_method(Type* type,
MethodInfo existing_info;
bool got_existing = try_lookup_method(type, method_name, &existing_info);
if (got_existing) {
// make sure we aren't changing anything.
if (!existing_info.type.is_compatible_child_method(ts, type->get_name())) {
if (override_type) {
if (!got_existing) {
throw_typesystem_error(
"The method {} of type {} was originally declared as {}, but has been "
"redeclared as {}\n",
method_name, type->get_name(), existing_info.type.print(), ts.print());
"Cannot use :replace on method {} of {} because this method was not previously declared "
"in a parent.",
method_name, type->get_name());
}
if ((existing_info.no_virtual || no_virtual) &&
existing_info.defined_in_type != type->get_name()) {
throw_typesystem_error(
"Cannot define method {} in type {} when it was defined as no_virtual in parent type {}",
method_name, type->get_name(), existing_info.defined_in_type);
}
if (no_virtual != existing_info.no_virtual) {
throw_typesystem_error(
"The method {} of type {} was originally declared with no_virtual = {}, but has been "
"redeclared as {}",
method_name, type->get_name(), existing_info.no_virtual, no_virtual);
}
return existing_info;
} else {
// add a new method!
// use the existing ID.
return type->add_method(
{get_next_method_id(type), method_name, ts, type->get_name(), no_virtual});
{existing_info.id, method_name, ts, type->get_name(), no_virtual, true});
} else {
if (got_existing) {
// make sure we aren't changing anything.
if (!existing_info.type.is_compatible_child_method(ts, type->get_name())) {
throw_typesystem_error(
"The method {} of type {} was originally declared as {}, but has been "
"redeclared as {}\n",
method_name, type->get_name(), existing_info.type.print(), ts.print());
}
if ((existing_info.no_virtual || no_virtual) &&
existing_info.defined_in_type != type->get_name()) {
throw_typesystem_error(
"Cannot define method {} in type {} when it was defined as no_virtual in parent type "
"{}",
method_name, type->get_name(), existing_info.defined_in_type);
}
if (no_virtual != existing_info.no_virtual) {
throw_typesystem_error(
"The method {} of type {} was originally declared with no_virtual = {}, but has been "
"redeclared as {}",
method_name, type->get_name(), existing_info.no_virtual, no_virtual);
}
return existing_info;
} else {
// add a new method!
return type->add_method(
{get_next_method_id(type), method_name, ts, type->get_name(), no_virtual, false});
}
}
}
@ -955,23 +975,25 @@ void TypeSystem::add_builtin_types() {
// OBJECT
declare_method(obj_type, "new", false,
make_function_typespec({"symbol", "type", "int"}, "_type_"));
declare_method(obj_type, "delete", false, make_function_typespec({"_type_"}, "none"));
declare_method(obj_type, "print", false, make_function_typespec({"_type_"}, "_type_"));
declare_method(obj_type, "inspect", false, make_function_typespec({"_type_"}, "_type_"));
declare_method(obj_type, "length", false,
make_function_typespec({"_type_"}, "int")); // todo - this integer type?
declare_method(obj_type, "asize-of", false, make_function_typespec({"_type_"}, "int"));
declare_method(obj_type, "copy", false, make_function_typespec({"_type_", "symbol"}, "_type_"));
declare_method(obj_type, "relocate", false, make_function_typespec({"_type_", "int"}, "_type_"));
make_function_typespec({"symbol", "type", "int"}, "_type_"), false);
declare_method(obj_type, "delete", false, make_function_typespec({"_type_"}, "none"), false);
declare_method(obj_type, "print", false, make_function_typespec({"_type_"}, "_type_"), false);
declare_method(obj_type, "inspect", false, make_function_typespec({"_type_"}, "_type_"), false);
declare_method(obj_type, "length", false, make_function_typespec({"_type_"}, "int"),
false); // todo - this integer type?
declare_method(obj_type, "asize-of", false, make_function_typespec({"_type_"}, "int"), false);
declare_method(obj_type, "copy", false, make_function_typespec({"_type_", "symbol"}, "_type_"),
false);
declare_method(obj_type, "relocate", false, make_function_typespec({"_type_", "int"}, "_type_"),
false);
declare_method(obj_type, "mem-usage", false,
make_function_typespec({"_type_", "memory-usage-block", "int"}, "_type_"));
make_function_typespec({"_type_", "memory-usage-block", "int"}, "_type_"), false);
// STRUCTURE
// structure new doesn't support dynamic sizing, which is kinda weird - it grabs the size from
// the type. Dynamic structures use new-dynamic-structure, which is used exactly once ever.
declare_method(structure_type, "new", false,
make_function_typespec({"symbol", "type"}, "_type_"));
declare_method(structure_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"),
false);
// structure_type is a field-less StructureType, so we have to do this to match the runtime.
// structure_type->override_size_in_memory(4);
@ -980,18 +1002,19 @@ void TypeSystem::add_builtin_types() {
add_field_to_type(basic_type, "type", make_typespec("type"));
// the default new basic doesn't support dynamic sizing. anything dynamic will override this
// and then call (method object new) to do the dynamically-sized allocation.
declare_method(basic_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"));
declare_method(basic_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"),
false);
// SYMBOL
builtin_structure_inherit(symbol_type);
add_field_to_type(symbol_type, "value", make_typespec("object"));
// a new method which returns type none means new is illegal.
declare_method(symbol_type, "new", false, make_function_typespec({}, "none"));
declare_method(symbol_type, "new", false, make_function_typespec({}, "none"), false);
// TYPE
builtin_structure_inherit(type_type);
declare_method(type_type, "new", false,
make_function_typespec({"symbol", "type", "int"}, "_type_"));
make_function_typespec({"symbol", "type", "int"}, "_type_"), false);
add_field_to_type(type_type, "symbol", make_typespec("symbol"));
add_field_to_type(type_type, "parent", make_typespec("type"));
add_field_to_type(type_type, "size", make_typespec("uint16")); // actually u16
@ -1008,7 +1031,7 @@ void TypeSystem::add_builtin_types() {
// string is never deftype'd for the decompiler, so we need to manually give the constructor
// type here.
declare_method(string_type, "new", false,
make_function_typespec({"symbol", "type", "int", "string"}, "_type_"));
make_function_typespec({"symbol", "type", "int", "string"}, "_type_"), false);
// FUNCTION
builtin_structure_inherit(function_type);
@ -1037,7 +1060,7 @@ void TypeSystem::add_builtin_types() {
// todo
builtin_structure_inherit(array_type);
declare_method(array_type, "new", false,
make_function_typespec({"symbol", "type", "type", "int"}, "_type_"));
make_function_typespec({"symbol", "type", "type", "int"}, "_type_"), false);
// array has: number, number, type
add_field_to_type(array_type, "length", make_typespec("int32"));
add_field_to_type(array_type, "allocated-length", make_typespec("int32"));
@ -1047,7 +1070,7 @@ void TypeSystem::add_builtin_types() {
// pair
pair_type->override_offset(2);
declare_method(pair_type, "new", false,
make_function_typespec({"symbol", "type", "object", "object"}, "_type_"));
make_function_typespec({"symbol", "type", "object", "object"}, "_type_"), false);
add_field_to_type(pair_type, "car", make_typespec("object"));
add_field_to_type(pair_type, "cdr", make_typespec("object"));
@ -1065,7 +1088,7 @@ void TypeSystem::add_builtin_types() {
add_field_to_type(file_stream_type, "name", make_typespec("string"));
add_field_to_type(file_stream_type, "file", make_typespec("uint32"));
declare_method(file_stream_type, "new", false,
make_function_typespec({"symbol", "type", "string", "basic"}, "_type_"));
make_function_typespec({"symbol", "type", "string", "basic"}, "_type_"), false);
}
/*!
@ -1590,6 +1613,10 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
methods_string.append(":no-virtual ");
}
if (info.overrides_method_type_of_parent) {
methods_string.append(":replace ");
}
methods_string.append(fmt::format("{})\n ", info.id));
}

7
common/type_system/TypeSystem.h
View File

@ -159,12 +159,13 @@ class TypeSystem {
MethodInfo declare_method(const std::string& type_name,
const std::string& method_name,
bool no_virtual,
const TypeSpec& ts);
const TypeSpec& ts,
bool override_type);
MethodInfo declare_method(Type* type,
const std::string& method_name,
bool no_virtual,
const TypeSpec& ts);
const TypeSpec& ts,
bool override_type);
MethodInfo define_method(const std::string& type_name,
const std::string& method_name,
const TypeSpec& ts);

9
common/type_system/deftype.cpp
View File

@ -188,12 +188,18 @@ void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def
obj = cdr(obj);
bool no_virtual = false;
bool replace_method = false;
if (!obj->is_empty_list() && car(obj).is_symbol(":no-virtual")) {
obj = cdr(obj);
no_virtual = true;
}
if (!obj->is_empty_list() && car(obj).is_symbol(":replace")) {
obj = cdr(obj);
replace_method = true;
}
int id = -1;
if (!obj->is_empty_list() && car(obj).is_int()) {
auto& id_obj = car(obj);
@ -212,7 +218,8 @@ void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def
});
function_typespec.add_arg(parse_typespec(type_system, return_type));
auto info = type_system->declare_method(type, method_name, no_virtual, function_typespec);
auto info = type_system->declare_method(type, method_name, no_virtual, function_typespec,
replace_method);
// check the method assert
if (id != -1) {

1
decompiler/CMakeLists.txt
View File

@ -15,6 +15,7 @@ add_library(
analysis/type_analysis.cpp
analysis/variable_naming.cpp
data/dir_tpages.cpp
data/game_count.cpp
data/game_text.cpp
data/StrFileReader.cpp

17
decompiler/ObjectFile/ObjectFileDB.cpp
View File

@ -15,6 +15,7 @@
#include "decompiler/data/tpage.h"
#include "decompiler/data/game_text.h"
#include "decompiler/data/StrFileReader.h"
#include "decompiler/data/dir_tpages.h"
#include "decompiler/data/game_count.h"
#include "LinkedObjectFileCreation.h"
#include "decompiler/config.h"
@ -556,20 +557,34 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
lg::info(" Total {:.3f} ms\n", timer.getMs());
}
void ObjectFileDB::process_tpages() {
std::string ObjectFileDB::process_tpages() {
lg::info("- Finding textures in tpages...");
std::string tpage_string = "tpage-";
int total = 0, success = 0;
int tpage_dir_count = 0;
Timer timer;
std::string result;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) {
auto statistics = process_tpage(data);
total += statistics.total_textures;
success += statistics.successful_textures;
} else if (data.name_in_dgo == "dir-tpages") {
result = process_dir_tpages(data).to_source();
tpage_dir_count++;
}
});
assert(tpage_dir_count <= 1);
if (tpage_dir_count == 0) {
lg::warn("Did not find tpage-dir.");
}
lg::info("Processed {} / {} textures {:.2f}% in {:.2f} ms", success, total,
100.f * float(success) / float(total), timer.getMs());
return result;
}
std::string ObjectFileDB::process_game_text_files() {

2
decompiler/ObjectFile/ObjectFileDB.h
View File

@ -84,7 +84,7 @@ class ObjectFileDB {
std::string ir2_final_out(ObjectFileData& data,
const std::unordered_set<std::string>& skip_functions = {});
void process_tpages();
std::string process_tpages();
std::string process_game_count_file();
std::string process_game_text_files();

4
decompiler/config/all-types.gc
View File

@ -4272,6 +4272,7 @@
:size-assert #x80
:flag-assert #xf00000080
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(remove-from-heap (_type_ kheap) _type_ 9)
(get-leftover-block-count (_type_ int int) int 10)
(dummy-11 () none 11)
@ -4313,6 +4314,7 @@
(entries texture-page-dir-entry 1 :inline)
)
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-9 (_type_ kheap) int 9)
)
:flag-assert #xa00000014
@ -11265,7 +11267,7 @@
:size-assert #x190
:flag-assert #x1400000190
(:methods
(relocate (_type_ int) _type_ 7)
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-17 () none 17)
(dummy-18 (_type_) none 18)
(dummy-19 (_type_) none 19)

2
decompiler/config/jak1_ntsc_black_label.jsonc
View File

@ -3,7 +3,7 @@
// if you want to filter to only some object names.
// it will make the decompiler much faster.
"allowed_objects": ["vol-h"],
"allowed_objects": [],
////////////////////////////
// CODE ANALYSIS OPTIONS

42
decompiler/config/jak1_ntsc_black_label/type_casts.jsonc
View File

@ -787,27 +787,27 @@
],
//"bg": [[[25, 52], "a0", "string"]],
"(anon-function 29 process-drawable)": [[[0, 99999], "s6", "process-drawable"]],
"ja-done?": [[[0, 99999], "s6", "process-drawable"]],
"ja-min?": [[[0, 99999], "s6", "process-drawable"]],
"ja-max?": [[[0, 99999], "s6", "process-drawable"]],
"ja-num-frames": [[[0, 99999], "s6", "process-drawable"]],
"ja-frame-num": [[[0, 99999], "s6", "process-drawable"]],
"ja-aframe-num": [[[0, 99999], "s6", "process-drawable"]],
"ja-aframe": [[[0, 99999], "s6", "process-drawable"]],
"ja-step": [[[0, 99999], "s6", "process-drawable"]],
"ja-channel-set!": [[[0, 99999], "s6", "process-drawable"]],
"ja-channel-push!": [[[0, 99999], "s6", "process-drawable"]],
"ja-group-size": [[[0, 99999], "s6", "process-drawable"]],
"ja-eval": [[[0, 99999], "s6", "process-drawable"]],
"ja-blend-eval": [[[0, 99999], "s6", "process-drawable"]],
"ja-post": [[[0, 99999], "s6", "process-drawable"], [54, "a1", "process"]],
"transform-post": [[[0, 99999], "s6", "process-drawable"]],
"rider-trans": [[[0, 99999], "s6", "process-drawable"]],
"rider-post": [[[0, 99999], "s6", "process-drawable"]],
"pusher-post": [[[0, 99999], "s6", "process-drawable"]],
"process-drawable-delay-player": [[[0, 99999], "s6", "process-drawable"]],
"init-target": [[[0, 99999], "s6", "target"]],
"(anon-function 29 process-drawable)": [[[0, 999], "s6", "process-drawable"]],
"ja-done?": [[[0, 999], "s6", "process-drawable"]],
"ja-min?": [[[0, 999], "s6", "process-drawable"]],
"ja-max?": [[[0, 999], "s6", "process-drawable"]],
"ja-num-frames": [[[0, 999], "s6", "process-drawable"]],
"ja-frame-num": [[[0, 999], "s6", "process-drawable"]],
"ja-aframe-num": [[[0, 999], "s6", "process-drawable"]],
"ja-aframe": [[[0, 999], "s6", "process-drawable"]],
"ja-step": [[[0, 999], "s6", "process-drawable"]],
"ja-channel-set!": [[[0, 999], "s6", "process-drawable"]],
"ja-channel-push!": [[[0, 999], "s6", "process-drawable"]],
"ja-group-size": [[[0, 999], "s6", "process-drawable"]],
"ja-eval": [[[0, 999], "s6", "process-drawable"]],
"ja-blend-eval": [[[0, 999], "s6", "process-drawable"]],
"ja-post": [[[0, 999], "s6", "process-drawable"], [54, "a1", "process"]],
"transform-post": [[[0, 999], "s6", "process-drawable"]],
"rider-trans": [[[0, 999], "s6", "process-drawable"]],
"rider-post": [[[0, 999], "s6", "process-drawable"]],
"pusher-post": [[[0, 999], "s6", "process-drawable"]],
"process-drawable-delay-player": [[[0, 999], "s6", "process-drawable"]],
"init-target": [[[0, 999], "s6", "target"]],
"upload-generic-shrub": [
[[3, 13], "t0", "dma-packet"],

13
decompiler/config/jak1_ntsc_black_label/var_names.jsonc
View File

@ -1252,6 +1252,19 @@
}
},
"texture-page-dir-inspect": {
"args":["dir", "mode"],
"vars": {
"v1-0":"pool",
"s4-0":"level-idx",
"a1-3":"lev",
"s4-1":"entry-idx",
"s3-0":"entry-page",
"s2-0":"entry-link",
"s1-0":"entry-list-length"
}
},
"texture-page-size-check": {
"args": ["pool", "level", "hide-prints"],
"vars": {

51
decompiler/data/dir_tpages.cpp Normal file
View File

@ -0,0 +1,51 @@
#include "third-party/fmt/core.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "dir_tpages.h"
namespace decompiler {
std::string DirTpageResult::to_source() const {
std::string result;
int i = 0;
for (auto len : lengths) {
result += fmt::format(" {:6s} ;; entry {}\n", fmt::format("#x{:x}", len), i);
i++;
}
return result;
}
DirTpageResult process_dir_tpages(ObjectFileData& data) {
DirTpageResult result;
auto& words = data.linked_data.words_by_seg.at(0);
int word_idx = 0;
// first is type
assert(words.at(word_idx).kind == LinkedWord::TYPE_PTR);
assert(words.at(word_idx).symbol_name == "texture-page-dir");
word_idx++;
// next is length
assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA);
int dir_length = words.at(word_idx).data;
fmt::print("length: {}\n", dir_length);
word_idx++;
for (int i = 0; i < dir_length; i++) {
assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA);
u32 entry = words.at(word_idx).data;
assert((entry & 0xffff7000) == 0); // 7 checks for sign bit.
word_idx++;
result.lengths.push_back(entry & 0xffff);
assert(words.at(word_idx).kind == LinkedWord::SYM_PTR);
assert(words.at(word_idx).symbol_name == "#f");
word_idx++;
assert(words.at(word_idx).kind == LinkedWord::SYM_PTR);
assert(words.at(word_idx).symbol_name == "#f");
word_idx++;
}
assert(word_idx == (int)words.size());
return result;
}
} // namespace decompiler

17
decompiler/data/dir_tpages.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include <vector>
#include <string>
namespace decompiler {
struct ObjectFileData;
struct DirTpageResult {
std::vector<int> lengths;
std::string to_source() const;
};
DirTpageResult process_dir_tpages(ObjectFileData& data);
} // namespace decompiler

3
decompiler/main.cpp
View File

@ -104,7 +104,8 @@ int main(int argc, char** argv) {
}
if (config.process_tpages) {
db.process_tpages();
auto result = db.process_tpages();
file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), result);
}
if (config.process_game_count) {

3
docs/markdown/progress-notes/changelog.md
View File

@ -171,4 +171,5 @@
- Forward declared basics can be used in more places
- You can now set a field which has a forward declared structure or basic type
- `cdr` now returns an object of type `pair`.
- `lambda`s can now be used inside of a static object definition.
- `lambda`s can now be used inside of a static object definition.
- Methods can now be `:replace`d to override their type from their parent. Use this with extreme care.

6
goal_src/engine/gfx/texture-h.gc
View File

@ -133,6 +133,7 @@
:size-assert #x80
:flag-assert #xf00000080
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(remove-from-heap (_type_ kheap) _type_ 9)
(get-leftover-block-count (_type_ int int) int 10)
(dummy-11 () none 11)
@ -174,8 +175,9 @@
(entries texture-page-dir-entry 1 :inline)
)
(:methods
(dummy-9 (_type_ kheap) int 9)
)
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-9 (_type_ kheap) int 9)
)
:flag-assert #xa00000014
)

119
goal_src/engine/gfx/texture.gc
View File

@ -13,7 +13,7 @@
;; (common textures for Jak, crates, etc)
;; - "global common" textures that are uploaded as needed, but aren't level specific
;; (the start menu hud)
;; - "level common" textures that are uploaded as needed
;; - "level common" textures that are uploaded as needed, and are specific to a level.
;; - "level near" textures. Part of these textures remain in VRAM always (don't live in RAM) and
;; part is reuploaded as needed. This is for TFRAG stuff near the camera
@ -1662,7 +1662,120 @@
(define *texture-pool* (new 'global 'texture-pool))
;; temp hack for loading:
(defmethod relocate texture-page ((obj texture-page) (offset int))
(defmethod relocate texture-page ((obj texture-page) (heap kheap) (ptr (pointer uint8)))
(format #t "HACK! removing texture page ~A from level heap~%" (-> obj name))
(remove-from-heap obj loading-level)
(remove-from-heap obj loading-level)
(none)
)
(defmethod asize-of texture-page-dir ((obj texture-page-dir))
(the-as int
(+ (-> texture-page-dir size) (the-as uint (* 12 (+ (-> obj length) -1))))
)
)
(defmethod length texture-page-dir ((obj texture-page-dir))
(-> obj length)
)
(defmethod relocate texture-page-dir ((obj texture-page-dir) (arg0 kheap) (arg1 (pointer uint8)))
(set! *texture-page-dir* obj)
(none)
)
(defun-debug texture-page-dir-inspect ((dir texture-page-dir) (mode symbol))
(format #t "[~8x] ~A~%" dir (-> dir type))
(let ((pool *texture-pool*))
(format
#t
"~Ttexture pool (~DK used, ~DK free)~%"
(/ (- (-> pool cur) (-> pool top)) 256)
(/ (- #xa0000 (-> pool cur)) 256)
)
)
(dotimes (level-idx (-> *level* length))
(let ((lev (-> *level* level level-idx)))
(if (= (-> lev status) 'active)
(texture-page-size-check *texture-pool* lev #f)
)
)
)
(format #t "~Tlength: ~D~%" (-> dir length))
(format #t "~Tdata[~D]: @ #x~X~%" (-> dir length) (-> dir entries))
(dotimes (entry-idx (-> dir length))
(let ((entry-page (-> dir entries entry-idx page))
(entry-link (-> dir entries entry-idx link))
)
(cond
(entry-page
(format #t "~T [~3D] loaded ~S ~A~%" entry-idx (if entry-link
" linked"
"unlinked"
)
entry-page
)
)
(else
(if (= mode 'full)
(format
#t
"~T [~3D] unloaded ~S #<texture-page :length ~D>~%"
entry-idx
(if entry-link
" linked"
"unlinked"
)
(-> dir entries entry-idx length)
)
)
)
)
(when (and (or entry-page entry-link) mode)
(dotimes (entry-list-length (-> dir entries entry-idx length))
(cond
((not entry-link)
(format #t "~T [~3D] unlinked" entry-list-length)
)
((zero? (-> entry-link next entry-list-length shader))
(format #t "~T [~3D] UNUSED " entry-list-length)
)
(else
(let ((t9-9 format)
(a0-12 #t)
(a1-10 "~T [~3D] ~3D links ")
(a2-11 entry-list-length)
(a3-7 0)
)
(let
((v1-40
(the-as object (* (-> entry-link next entry-list-length shader) 16))
)
)
(while (nonzero? (the-as int v1-40))
(nop!)
(+! a3-7 1)
(set! v1-40 (* (-> (the-as adgif-shader v1-40) next shader) 16))
)
)
(t9-9 a0-12 a1-10 a2-11 a3-7)
)
)
)
(cond
((not entry-page)
(format #t " unloaded~%")
)
((not (-> entry-page data entry-list-length))
(format #t " empty~%")
)
(else
(format #t " ~A~%" (-> entry-page data entry-list-length))
)
)
)
)
)
)
(none)
)

10
goal_src/engine/gfx/vis/bsp-h.gc
View File

@ -53,11 +53,11 @@
:size-assert #x190
:flag-assert #x1400000190
(:methods
(relocate (_type_ int) _type_ 7)
(dummy-17 () none 17)
(dummy-18 (_type_) none 18)
(dummy-19 (_type_) none 19)
)
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-17 () none 17)
(dummy-18 (_type_) none 18)
(dummy-19 (_type_) none 19)
)
)
(deftype game-level (basic)

3
goal_src/engine/level/level.gc
View File

@ -93,7 +93,7 @@
)
;; relocate bsp-header
(defmethod relocate bsp-header ((obj bsp-header) (arg0 int))
(defmethod relocate bsp-header ((obj bsp-header) (dest-heap kheap) (name (pointer uint8)))
(let ((s5-0 (-> *level* unknown-level-2)))
(if s5-0
(cond
@ -130,6 +130,7 @@
)
)
)
(none)
)
(defmethod dummy-24 level ((obj level))

1
goal_src/goal-lib.gc
View File

@ -60,6 +60,7 @@
`(begin
(asm-data-file game-text "assets/game_text.txt")
(asm-data-file game-count "assets/game_count.txt")
(asm-data-file dir-tpages "assets/tpage-dir.txt")
)
)

1
goalc/CMakeLists.txt
View File

@ -27,6 +27,7 @@ add_library(compiler
compiler/compilation/Static.cpp
compiler/Util.cpp
data_compiler/game_text.cpp
data_compiler/dir_tpages.cpp
data_compiler/DataObjectGenerator.cpp
data_compiler/game_count.cpp
debugger/Debugger.cpp

8
goalc/compiler/StaticObject.cpp
View File

@ -145,10 +145,10 @@ void StaticStructure::add_type_record(std::string name, int offset) {
}
void StaticStructure::add_function_record(const FunctionEnv* function, int offset) {
FunctionRecord rec;
rec.func = function;
rec.offset_in_this = offset;
functions.push_back(rec);
FunctionRecord frec;
frec.func = function;
frec.offset_in_this = offset;
functions.push_back(frec);
}
///////////////////

3
goalc/compiler/compilation/CompilerControl.cpp
View File

@ -11,6 +11,7 @@
#include "common/util/FileUtil.h"
#include "goalc/data_compiler/game_text.h"
#include "goalc/data_compiler/game_count.h"
#include "goalc/data_compiler/dir_tpages.h"
#include "common/goos/ReplUtils.h"
#include <regex>
#include <stack>
@ -67,6 +68,8 @@ Val* Compiler::compile_asm_data_file(const goos::Object& form, const goos::Objec
compile_game_text(as_string(args.unnamed.at(1)));
} else if (kind == "game-count") {
compile_game_count(as_string(args.unnamed.at(1)));
} else if (kind == "dir-tpages") {
compile_dir_tpages(as_string(args.unnamed.at(1)));
} else {
throw_compiler_error(form, "The option {} was not recognized for asm-data-file.", kind);
}

20
goalc/data_compiler/DataObjectGenerator.cpp
View File

@ -194,8 +194,24 @@ std::vector<u8> DataObjectGenerator::generate_link_table() {
}
push_variable_length_integer(0, &link);
// todo symbols
assert(m_symbol_links.empty());
for (auto& sl : m_symbol_links) {
// insert name. first char won't have the highest bit set
for (auto c : sl.first) {
link.push_back(c);
}
link.push_back(0);
std::sort(sl.second.begin(), sl.second.end());
int prev = 0;
for (auto& x : sl.second) {
int diff = x - prev;
assert(diff >= 0);
push_better_variable_length_integer(diff * 4, &link);
m_words.at(x) = 0xffffffff;
prev = x;
}
link.push_back(0);
}
// types
for (auto& tl : m_type_links) {

37
goalc/data_compiler/dir_tpages.cpp Normal file
View File

@ -0,0 +1,37 @@
#include "dir_tpages.h"
#include "DataObjectGenerator.h"
#include "common/goos/Reader.h"
#include "common/goos/ParseHelpers.h"
#include "common/util/FileUtil.h"
void compile_dir_tpages(const std::string& filename) {
std::vector<int> lengths;
printf("[Build tpage dir] %s\n", filename.c_str());
goos::Reader reader;
auto code = reader.read_from_file({filename});
std::string err;
goos::for_each_in_list(code.as_pair()->cdr, [&](const goos::Object& obj) {
if (!obj.is_int()) {
throw std::runtime_error("Invalid tpage dir file");
}
lengths.push_back(obj.as_int());
});
DataObjectGenerator gen;
gen.add_type_tag("texture-page-dir");
gen.add_word(lengths.size());
for (auto len : lengths) {
gen.add_word(len);
gen.add_symbol_link("#f");
gen.add_symbol_link("#f");
}
auto data = gen.generate_v4();
file_util::create_dir_if_needed(file_util::get_file_path({"out", "obj"}));
file_util::write_binary_file(file_util::get_file_path({"out", "obj", "dir-tpages.go"}),
data.data(), data.size());
}

5
goalc/data_compiler/dir_tpages.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
#include <string>
void compile_dir_tpages(const std::string& filename);

3
goalc/data_compiler/game_text.cpp
View File

@ -173,6 +173,9 @@ std::vector<std::unordered_map<int, std::string>> parse(const goos::Object& data
*/
void compile(const std::vector<std::unordered_map<int, std::string>>& text,
const std::string& group_name) {
if (text.empty()) {
return;
}
// get all text ID's we know
std::vector<int> add_order;
add_order.reserve(text.front().size());

1
out/hash.md5
View File

@ -6,3 +6,4 @@ abcc25e5d7469dd6a572dc53dbb9671c iso/3COMMON.TXT
5d62de2c78b4cf102b9a78f3aa96c8c9 iso/5COMMON.TXT
9495f80955e6782513fe12f6539fc8e7 iso/6COMMON.TXT
9765bdc3add08cb06fd3e87ebd5713aa obj/game-cnt.go
5033811c685e1bde4411d396b1d4ffba obj/dir-tpages.go

2
test/decompiler/reference/engine/gfx/texture-h_REF.gc
View File

@ -190,6 +190,7 @@
:size-assert #x80
:flag-assert #xf00000080
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(remove-from-heap (_type_ kheap) _type_ 9)
(get-leftover-block-count (_type_ int int) int 10)
(dummy-11 () none 11)
@ -271,6 +272,7 @@
:size-assert #x14
:flag-assert #xa00000014
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-9 (_type_ kheap) int 9)
)
)

1
test/decompiler/reference/engine/gfx/vis/bsp-h_REF.gc
View File

@ -60,6 +60,7 @@
:size-assert #x190
:flag-assert #x1400000190
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
(dummy-18 (_type_) none 18)
(dummy-19 (_type_) none 19)
)

15
test/goalc/source_templates/with_game/test-method-replace.gc Normal file
View File

@ -0,0 +1,15 @@
(deftype type-with-weird-relocate (basic)
((foo int))
(:methods
(relocate (_type_ kheap (pointer uint8)) none :replace 7)
)
)
(defmethod relocate type-with-weird-relocate ((obj type-with-weird-relocate) (dest-heap kheap) (name (pointer uint8)))
(format #t "relocate! foo: ~D heap: ~D name: ~D~%" (-> obj foo) dest-heap name)
(none)
)
(let ((obj (new 'static 'type-with-weird-relocate :foo 123)))
(relocate obj (the kheap 1) (the (pointer uint8) 2))
)

5
test/goalc/test_with_game.cpp
View File

@ -775,6 +775,11 @@ TEST_F(WithGameTests, StaticLambda) {
runner.run_static_test(env, testCategory, "test-static-lambda.gc", {"Add: 30 sub: -10\n0\n"});
}
TEST_F(WithGameTests, MethodReplace) {
runner.run_static_test(env, testCategory, "test-method-replace.gc",
{"relocate! foo: 123 heap: 1 name: 2\n0\n"});
}
TEST(TypeConsistency, TypeConsistency) {
Compiler compiler;
compiler.enable_throw_on_redefines();

21
test/test_type_system.cpp
View File

@ -231,27 +231,28 @@ TEST(TypeSystem, AddMethodAndLookupMethod) {
ts.add_builtin_types();
auto parent_info = ts.declare_method(ts.lookup_type("structure"), "test-method-1", false,
ts.make_function_typespec({"integer"}, "string"));
ts.make_function_typespec({"integer"}, "string"), false);
// when trying to add the same method to a child, should return the parent's method
auto child_info_same = ts.declare_method(ts.lookup_type("basic"), "test-method-1", false,
ts.make_function_typespec({"integer"}, "string"));
ts.make_function_typespec({"integer"}, "string"), false);
EXPECT_EQ(parent_info.id, child_info_same.id);
EXPECT_EQ(parent_info.id, GOAL_MEMUSAGE_METHOD + 1);
// any amount of fiddling with method types should cause an error
EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false,
ts.make_function_typespec({"integer"}, "integer")));
ts.make_function_typespec({"integer"}, "integer"), false));
EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false,
ts.make_function_typespec({}, "string")));
ts.make_function_typespec({}, "string"), false));
EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false,
ts.make_function_typespec({"integer", "string"}, "string")));
ts.make_function_typespec({"integer", "string"}, "string"),
false));
EXPECT_ANY_THROW(ts.declare_method(ts.lookup_type("basic"), "test-method-1", false,
ts.make_function_typespec({"string"}, "string")));
ts.make_function_typespec({"string"}, "string"), false));
ts.declare_method(ts.lookup_type("basic"), "test-method-2", false,
ts.make_function_typespec({"integer"}, "string"));
ts.make_function_typespec({"integer"}, "string"), false);
EXPECT_EQ(parent_info.id, ts.lookup_method("basic", "test-method-1").id);
EXPECT_EQ(parent_info.id, ts.lookup_method("structure", "test-method-1").id);
@ -274,10 +275,10 @@ TEST(TypeSystem, NewMethod) {
ts.add_builtin_types();
ts.add_type("test-1", std::make_unique<BasicType>("basic", "test-1", false, 0));
ts.declare_method(ts.lookup_type("test-1"), "new", false,
ts.make_function_typespec({"symbol", "string"}, "test-1"));
ts.make_function_typespec({"symbol", "string"}, "test-1"), false);
ts.add_type("test-2", std::make_unique<BasicType>("test-1", "test-2", false, 0));
ts.declare_method(ts.lookup_type("test-2"), "new", false,
ts.make_function_typespec({"symbol", "string", "symbol"}, "test-2"));
ts.make_function_typespec({"symbol", "string", "symbol"}, "test-2"), false);
EXPECT_EQ(ts.lookup_method("test-1", "new").type.print(), "(function symbol string test-1)");
EXPECT_EQ(ts.lookup_method("test-2", "new").type.print(),
@ -296,7 +297,7 @@ TEST(TypeSystem, MethodSubstitute) {
ts.add_builtin_types();
ts.add_type("test-1", std::make_unique<BasicType>("basic", "test-1", false, 0));
ts.declare_method(ts.lookup_type("test-1"), "new", false,
ts.make_function_typespec({"symbol", "string", "_type_"}, "_type_"));
ts.make_function_typespec({"symbol", "string", "_type_"}, "_type_"), false);
auto final_type = ts.lookup_method("test-1", "new").type.substitute_for_method_call("test-1");
EXPECT_EQ(final_type.print(), "(function symbol string test-1 test-1)");

1
tools/MemoryDumpTool/main.cpp
View File

@ -249,7 +249,6 @@ void inspect_basics(const Ram& ram,
bool goal_array = field.type() == TypeSpec("array", {TypeSpec("basic")});
std::unordered_map<std::string, int> type_frequency;
int array_max_elts = 0;
for (auto base_addr : basics.at(name)) {
for (int elt_idx = 0; elt_idx < array_size; elt_idx++) {