Game-Decompilation/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project.git synced 2025-02-13 01:49:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

goalc: use iso_data build_info to inform custom level build process (#2959)

Browse Source
This commit is contained in:
Tyler Wilding 2023-09-03 16:17:35 -06:00 committed by GitHub
parent 46bdd109dd
commit 07d97bce8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 185 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
CMakePresets.json
View File

@ -58,6 +58,16 @@
"CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
}
},
{
"name": "base-linux-debug",
"hidden": true,
"inherits": "base",
"binaryDir": "${sourceDir}/build/Release/bin",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
}
},
{
"name": "base-macos-release",
"hidden": true,
@ -173,6 +183,16 @@
"ZYDIS_BUILD_SHARED_LIB": "OFF"
}
},
{
"name": "Debug-linux-clang-static",
"displayName": "Linux Static Release (clang)",
"description": "Build with Clang as Release without Debug Symbols but statically linked",
"inherits": ["base-linux-debug", "base-clang"],
"cacheVariables": {
"STATICALLY_LINK": "true",
"ZYDIS_BUILD_SHARED_LIB": "OFF"
}
},
{
"name": "Release-linux-gcc",
"displayName": "Linux Release (gcc)",

4
decompiler/CMakeLists.txt
View File

@ -35,6 +35,8 @@ add_library(
Disasm/OpcodeInfo.cpp
Disasm/Register.cpp
extractor/extractor_util.cpp
Function/BasicBlocks.cpp
Function/CfgVtx.cpp
Function/Function.cpp
@ -108,7 +110,7 @@ target_link_libraries(decompiler
add_executable(extractor
extractor/main.cpp extractor/extractor_util.hpp )
extractor/main.cpp extractor/extractor_util.cpp)
target_link_libraries(extractor
decomp

116
decompiler/extractor/extractor_util.hpp → decompiler/extractor/extractor_util.cpp
View File

@ -1,4 +1,4 @@
#pragma once
#include "extractor_util.h"
#include <optional>
#include <regex>
@ -15,94 +15,34 @@
#include "third-party/json.hpp"
#include "third-party/zstd/lib/common/xxhash.h"
enum class ExtractorErrorCode {
SUCCESS = 0,
INVALID_CLI_INPUT = 3990,
VALIDATION_CANT_LOCATE_ELF = 4000,
VALIDATION_SERIAL_MISSING_FROM_DB = 4001,
VALIDATION_ELF_MISSING_FROM_DB = 4002,
VALIDATION_BAD_ISO_CONTENTS = 4010,
VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011,
VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012,
VALIDATION_BAD_EXTRACTION = 4020,
DECOMPILATION_GENERIC_ERROR = 4030,
EXTRACTION_INVALID_ISO_PATH = 4040,
EXTRACTION_ISO_UNEXPECTED_SIZE = 4041,
COMPILATION_BAD_PROJECT_PATH = 4050,
};
enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
static const std::unordered_map<std::string, GameIsoFlags> sGameIsoFlagNames = {
const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
{"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
static const std::unordered_map<int, std::string> sGameIsoTerritoryMap = {
const std::unordered_map<int, std::string> game_iso_territory_map = {
{GAME_TERRITORY_SCEA, "NTSC-U"},
{GAME_TERRITORY_SCEE, "PAL"},
{GAME_TERRITORY_SCEI, "NTSC-J"},
{GAME_TERRITORY_SCEK, "NTSC-K"}};
std::string get_territory_name(int territory) {
ASSERT_MSG(sGameIsoTerritoryMap.count(territory),
ASSERT_MSG(game_iso_territory_map.count(territory),
fmt::format("territory {} not found in territory name map"));
return sGameIsoTerritoryMap.at(territory);
return game_iso_territory_map.at(territory);
}
// used for - decompiler_out/<jak1> and iso_data/<jak1>
std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
struct ISOMetadata {
std::string canonical_name;
int region; // territory code
int num_files;
uint64_t contents_hash;
std::string decomp_config_version;
std::string game_name;
std::vector<std::string> flags;
};
// This is all we need to re-fetch info from the database
// - if this changes such that we have a collision in the future,
// then the database isn't adequate and everything must change
struct BuildInfo {
std::string serial = "";
uint64_t elf_hash = 0;
};
void to_json(nlohmann::json& j, const BuildInfo& info) {
j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
}
void from_json(const nlohmann::json& j, BuildInfo& info) {
j[0].at("serial").get_to(info.serial);
j[0].at("elf_hash").get_to(info.elf_hash);
}
std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path) {
if (!fs::exists(iso_data_path / "buildinfo.json")) {
return {};
}
auto buildinfo_path = (iso_data_path / "buildinfo.json").string();
try {
return parse_commented_json(file_util::read_text_file(buildinfo_path), buildinfo_path)
.get<BuildInfo>();
} catch (std::exception& e) {
lg::error("JSON parsing error on buildinfo.json - {}", e.what());
return {};
}
}
static const ISOMetadata jak1_ntsc_black_label_info = {
"Jak & Daxter™: The Precursor Legacy (Black Label)",
GAME_TERRITORY_SCEA,
337,
11363853835861842434U,
"ntsc_v1",
"jak1",
{"jak1-black-label"}};
const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)",
GAME_TERRITORY_SCEA,
337,
11363853835861842434U,
"ntsc_v1",
"jak1",
{"jak1-black-label"}};
// { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> isoDatabase{
const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> iso_database = {
{"SCUS-97124",
{{7280758013604870207U, jak1_ntsc_black_label_info},
{744661860962747854,
@ -132,12 +72,36 @@ static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMet
"jak1",
{}}}}}};
void to_json(nlohmann::json& j, const BuildInfo& info) {
j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
}
void from_json(const nlohmann::json& j, BuildInfo& info) {
j[0].at("serial").get_to(info.serial);
j[0].at("elf_hash").get_to(info.elf_hash);
}
std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path) {
if (!fs::exists(iso_data_path / "buildinfo.json")) {
return {};
}
auto buildinfo_path = (iso_data_path / "buildinfo.json").string();
try {
const auto buildinfo_data = file_util::read_text_file(buildinfo_path);
lg::info("Found version info - {}", buildinfo_data);
return parse_commented_json(buildinfo_data, buildinfo_path).get<BuildInfo>();
} catch (std::exception& e) {
lg::error("JSON parsing error on buildinfo.json - {}", e.what());
return {};
}
}
std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& build_info) {
if (build_info.serial.empty() || build_info.elf_hash == 0) {
return {};
}
auto dbEntry = isoDatabase.find(build_info.serial);
if (dbEntry == isoDatabase.end()) {
auto dbEntry = iso_database.find(build_info.serial);
if (dbEntry == iso_database.end()) {
return {};
}

90
decompiler/extractor/extractor_util.h Normal file
View File

@ -0,0 +1,90 @@
#pragma once
#include <optional>
#include <regex>
#include <unordered_map>
#include "common/log/log.h"
#include "common/util/Assert.h"
#include "common/util/FileUtil.h"
#include "common/util/json_util.h"
#include "common/util/read_iso_file.h"
#include "game/kernel/common/kboot.h"
#include "third-party/json.hpp"
#include "third-party/zstd/lib/common/xxhash.h"
enum class ExtractorErrorCode {
SUCCESS = 0,
INVALID_CLI_INPUT = 3990,
VALIDATION_CANT_LOCATE_ELF = 4000,
VALIDATION_SERIAL_MISSING_FROM_DB = 4001,
VALIDATION_ELF_MISSING_FROM_DB = 4002,
VALIDATION_BAD_ISO_CONTENTS = 4010,
VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011,
VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012,
VALIDATION_BAD_EXTRACTION = 4020,
DECOMPILATION_GENERIC_ERROR = 4030,
EXTRACTION_INVALID_ISO_PATH = 4040,
EXTRACTION_ISO_UNEXPECTED_SIZE = 4041,
COMPILATION_BAD_PROJECT_PATH = 4050,
};
enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
extern const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names;
extern const std::unordered_map<int, std::string> game_iso_territory_map;
// used for - decompiler_out/<jak1> and iso_data/<jak1>
extern const std::unordered_map<std::string, std::string> data_subfolders;
std::string get_territory_name(int territory);
struct ISOMetadata {
std::string canonical_name;
int region; // territory code
int num_files;
uint64_t contents_hash;
std::string decomp_config_version;
std::string game_name;
std::vector<std::string> flags;
};
extern const ISOMetadata jak1_ntsc_black_label_info;
// { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
extern const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>
iso_database;
// This is all we need to re-fetch info from the database
// - if this changes such that we have a collision in the future,
// then the database isn't adequate and everything must change
struct BuildInfo {
std::string serial = "";
uint64_t elf_hash = 0;
};
void to_json(nlohmann::json& j, const BuildInfo& info);
void from_json(const nlohmann::json& j, BuildInfo& info);
std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path);
std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& build_info);
ISOMetadata get_version_info_or_default(const fs::path& iso_data_path);
std::tuple<std::optional<std::string>, std::optional<uint64_t>> findElfFile(
const fs::path& extracted_iso_path);
void log_potential_new_db_entry(ExtractorErrorCode error_code,
const std::string& serial,
const uint64_t elf_hash,
const int files_extracted,
const uint64_t contents_hash);
std::tuple<bool, ExtractorErrorCode> is_iso_file(fs::path path_to_supposed_iso);
std::tuple<uint64_t, int> calculate_extraction_hash(const IsoFile& iso_file);
std::tuple<uint64_t, int> calculate_extraction_hash(const fs::path& extracted_iso_path);

12
decompiler/extractor/main.cpp
View File

@ -2,7 +2,7 @@
#include <regex>
#include <unordered_map>
#include "extractor_util.hpp"
#include "extractor_util.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
@ -52,8 +52,8 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
}
// Find the game in our tracking database
auto dbEntry = isoDatabase.find(serial.value());
if (dbEntry == isoDatabase.end()) {
auto dbEntry = iso_database.find(serial.value());
if (dbEntry == iso_database.end()) {
lg::error("Serial '{}' not found in the validation database", serial.value());
log_potential_new_db_entry(ExtractorErrorCode::VALIDATION_SERIAL_MISSING_FROM_DB,
serial.value(), elf_hash.value(), expected_num_files, expected_hash);
@ -203,7 +203,7 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat
int flags = 0;
for (const auto& flag : version_info.flags) {
if (auto it = sGameIsoFlagNames.find(flag); it != sGameIsoFlagNames.end()) {
if (auto it = game_iso_flag_names.find(flag); it != game_iso_flag_names.end()) {
flags |= it->second;
}
}
@ -320,7 +320,7 @@ int main(int argc, char** argv) {
lg::error("Error: input game name '{}' is not valid", game_name);
return static_cast<int>(ExtractorErrorCode::INVALID_CLI_INPUT);
}
std::string data_subfolder = data_subfolders[game_name];
std::string data_subfolder = data_subfolders.at(game_name);
if (flag_extract) {
// we extract to a temporary location because we don't know what we're extracting yet!
@ -356,7 +356,7 @@ int main(int argc, char** argv) {
} else {
// We know the version since we just extracted it, so the user didn't need to provide this
// explicitly
data_subfolder = data_subfolders[version_info->game_name];
data_subfolder = data_subfolders.at(version_info->game_name);
iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
if (fs::exists(iso_data_path)) {
fs::remove_all(iso_data_path);

43
goalc/build_level/build_level.cpp
View File

@ -7,6 +7,7 @@
#include "common/util/json_util.h"
#include "common/util/string_util.h"
#include "decompiler/extractor/extractor_util.h"
#include "decompiler/level_extractor/extract_merc.h"
#include "goalc/build_level/Entity.h"
#include "goalc/build_level/FileInfo.h"
@ -144,37 +145,45 @@ bool run_build_level(const std::string& input_file,
// Add textures and models
// TODO remove hardcoded config settings
if (level_json.contains("art_groups") && !level_json.at("art_groups").empty()) {
fs::path iso_folder = "";
lg::info("Looking for ISO path...");
// TODO - add to file_util
for (const auto& entry :
fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) {
// TODO - hard-coded to jak 1
if (entry.is_directory() &&
entry.path().filename().string().find("jak1") != std::string::npos) {
lg::info("Found ISO path: {}", entry.path().string());
iso_folder = entry.path();
}
}
if (iso_folder.empty() || !fs::exists(iso_folder)) {
lg::warn("Could not locate ISO path!");
return false;
}
// Look for iso build info if it's available, otherwise default to ntsc_v1
const auto version_info = get_version_info_or_default(iso_folder);
decompiler::Config config;
try {
config = decompiler::read_config_file(
file_util::get_jak_project_dir() / "decompiler/config/jak1/jak1_config.jsonc", "ntsc_v1",
file_util::get_jak_project_dir() / "decompiler/config/jak1/jak1_config.jsonc",
version_info.decomp_config_version,
R"({"decompile_code": false, "find_functions": false, "levels_extract": true, "allowed_objects": []})");
} catch (const std::exception& e) {
lg::error("Failed to parse config: {}", e.what());
return false;
}
fs::path in_folder;
lg::info("Looking for ISO path...");
for (const auto& entry :
fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) {
if (entry.is_directory() &&
entry.path().filename().string().find("jak1") != std::string::npos) {
lg::info("Found ISO path: {}", entry.path().string());
in_folder = entry.path();
}
}
if (!fs::exists(in_folder)) {
lg::error("Could not find ISO path!");
return false;
}
std::vector<fs::path> dgos, objs;
for (const auto& dgo_name : config.dgo_names) {
dgos.push_back(in_folder / dgo_name);
dgos.push_back(iso_folder / dgo_name);
}
for (const auto& obj_name : config.object_file_names) {
objs.push_back(in_folder / obj_name);
objs.push_back(iso_folder / obj_name);
}
decompiler::ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, {},