[Tools] Add DGO packer and unpacker (#219)

* add dgo tools

* make codacy happy
This commit is contained in:
water111 2021-01-27 20:46:58 -05:00 committed by GitHub
parent 3ea8cbea6f
commit 51f70b6f4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 261 additions and 75 deletions

View File

@ -76,6 +76,9 @@ add_subdirectory(game)
# build the compiler
add_subdirectory(goalc)
# build standalone tools
add_subdirectory(tools)
# build the gtest libraries
add_subdirectory(third-party/googletest)

View File

@ -14,6 +14,8 @@ add_library(common
type_system/TypeFieldLookup.cpp
type_system/TypeSpec.cpp
type_system/TypeSystem.cpp
util/dgo_util.cpp
util/DgoReader.cpp
util/DgoWriter.cpp
util/FileUtil.cpp
util/Timer.cpp

View File

@ -11,32 +11,26 @@
class BinaryReader {
public:
BinaryReader(uint8_t* _buffer, uint32_t _size) : buffer(_buffer), size(_size) {}
explicit BinaryReader(std::vector<uint8_t>& _buffer)
: buffer((uint8_t*)_buffer.data()), size(_buffer.size()) {}
explicit BinaryReader(const std::vector<uint8_t>& _buffer) : buffer(_buffer) {}
template <typename T>
T read() {
assert(seek + sizeof(T) <= size);
T& obj = *(T*)(buffer + seek);
assert(seek + sizeof(T) <= buffer.size());
T& obj = *(T*)(buffer.data() + seek);
seek += sizeof(T);
return obj;
}
void ffwd(int amount) {
seek += amount;
assert(seek <= size);
assert(seek <= buffer.size());
}
uint32_t bytes_left() const { return size - seek; }
uint8_t* here() { return buffer + seek; }
uint32_t get_seek() { return seek; }
uint32_t bytes_left() const { return buffer.size() - seek; }
uint8_t* here() { return buffer.data() + seek; }
uint32_t get_seek() const { return seek; }
private:
uint8_t* buffer;
uint32_t size;
std::vector<u8> buffer;
uint32_t seek = 0;
};

54
common/util/DgoReader.cpp Normal file
View File

@ -0,0 +1,54 @@
#include <cstring>
#include <utility>
#include <unordered_set>
#include "DgoReader.h"
#include "BinaryReader.h"
#include "common/link_types.h"
#include "third-party/json.hpp"
#include "dgo_util.h"
DgoReader::DgoReader(std::string file_name, const std::vector<u8>& data)
: m_file_name(std::move(file_name)) {
BinaryReader reader(data);
auto header = reader.read<DgoHeader>();
m_internal_name = header.name;
std::unordered_set<std::string> all_unique_names;
// get all obj files...
for (uint32_t i = 0; i < header.object_count; i++) {
auto obj_header = reader.read<ObjectHeader>();
assert(reader.bytes_left() >= obj_header.size);
assert_string_empty_after(obj_header.name, 60);
DgoDataEntry entry;
entry.internal_name = obj_header.name;
entry.unique_name = get_object_file_name(entry.internal_name, reader.here(), obj_header.size);
all_unique_names.insert(entry.unique_name);
entry.data.resize(obj_header.size);
assert((reader.get_seek() % 16) == 0);
memcpy(entry.data.data(), reader.here(), obj_header.size);
m_entries.push_back(entry);
reader.ffwd(obj_header.size);
}
// check we're at the end
assert(0 == reader.bytes_left());
assert(all_unique_names.size() == m_entries.size());
}
std::string DgoReader::description_as_json() const {
using namespace nlohmann;
json j;
j["file_name"] = m_file_name;
j["internal_name"] = m_internal_name;
for (auto& entry : m_entries) {
json entry_desc;
entry_desc["unique_name"] = entry.unique_name;
entry_desc["internal_name"] = entry.internal_name;
j["objects"].push_back(entry_desc);
}
return j.dump(4);
}

22
common/util/DgoReader.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include <vector>
#include <string>
#include "common/common_types.h"
struct DgoDataEntry {
std::vector<u8> data;
std::string internal_name;
std::string unique_name;
};
class DgoReader {
public:
DgoReader(std::string file_name, const std::vector<u8>& data);
const std::vector<DgoDataEntry> entries() const { return m_entries; }
std::string description_as_json() const;
private:
std::vector<DgoDataEntry> m_entries;
std::string m_internal_name, m_file_name;
};

53
common/util/dgo_util.cpp Normal file
View File

@ -0,0 +1,53 @@
#include <cassert>
#include <cstring>
#include "dgo_util.h"
#include "common/versions.h"
#include "third-party/fmt/core.h"
/*!
* Assert false if the char[] has non-null data after the null terminated string.
* Used to sanity check the sizes of strings in DGO/object file headers.
*/
void assert_string_empty_after(const char* str, int size) {
auto ptr = str;
while (*ptr)
ptr++;
while (ptr - str < size) {
assert(!*ptr);
ptr++;
}
}
std::string get_object_file_name(const std::string& original_name, u8* data, int size) {
const std::string art_group_text =
fmt::format("/src/next/data/art-group{}/",
versions::ART_FILE_VERSION); // todo, this may change in other games
const std::string suffix = "-ag.go";
int len = int(art_group_text.length());
for (int start = 0; start < size; start++) {
bool failed = false;
for (int i = 0; i < len; i++) {
if (start + i >= size || data[start + i] != art_group_text[i]) {
failed = true;
break;
}
}
if (!failed) {
for (int i = 0; i < int(original_name.length()); i++) {
if (start + len + i >= size || data[start + len + i] != original_name[i]) {
assert(false);
}
}
assert(int(suffix.length()) + start + len + int(original_name.length()) < size);
assert(
!memcmp(data + start + len + original_name.length(), suffix.data(), suffix.length() + 1));
return original_name + "-ag";
}
}
return original_name;
}

7
common/util/dgo_util.h Normal file
View File

@ -0,0 +1,7 @@
#pragma once
#include <string>
#include "common/common_types.h"
void assert_string_empty_after(const char* str, int size);
std::string get_object_file_name(const std::string& original_name, u8* data, int size);

View File

@ -10,6 +10,8 @@
#include <set>
#include <cstring>
#include <map>
#include "common/link_types.h"
#include "common/util/dgo_util.h"
#include "decompiler/data/tpage.h"
#include "decompiler/data/game_text.h"
#include "decompiler/data/StrFileReader.h"
@ -180,62 +182,6 @@ void ObjectFileDB::load_map_file(const std::string& map_data) {
}
}
// Header for a DGO file
struct DgoHeader {
uint32_t size;
char name[60];
};
namespace {
/*!
* Assert false if the char[] has non-null data after the null terminated string.
* Used to sanity check the sizes of strings in DGO/object file headers.
*/
void assert_string_empty_after(const char* str, int size) {
auto ptr = str;
while (*ptr)
ptr++;
while (ptr - str < size) {
assert(!*ptr);
ptr++;
}
}
} // namespace
namespace {
std::string get_object_file_name(const std::string& original_name, uint8_t* data, int size) {
const char art_group_text[] =
"/src/next/data/art-group6/"; // todo, this may change in other games
const char suffix[] = "-ag.go";
int len = int(strlen(art_group_text));
for (int start = 0; start < size; start++) {
bool failed = false;
for (int i = 0; i < len; i++) {
if (start + i >= size || data[start + i] != art_group_text[i]) {
failed = true;
break;
}
}
if (!failed) {
for (int i = 0; i < int(original_name.length()); i++) {
if (start + len + i >= size || data[start + len + i] != original_name[i]) {
assert(false);
}
}
assert(int(strlen(suffix)) + start + len + int(original_name.length()) < size);
assert(!memcmp(data + start + len + original_name.length(), suffix, strlen(suffix) + 1));
return original_name + "-ag";
}
}
return original_name;
}
} // namespace
constexpr int MAX_CHUNK_SIZE = 0x8000;
/*!
* Load the objects stored in the given DGO into the ObjectFileDB
@ -303,9 +249,9 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
assert_string_empty_after(header.name, 60);
// get all obj files...
for (uint32_t i = 0; i < header.size; i++) {
for (uint32_t i = 0; i < header.object_count; i++) {
auto obj_header = reader.read<DgoHeader>();
assert(reader.bytes_left() >= obj_header.size);
assert(reader.bytes_left() >= obj_header.object_count);
assert_string_empty_after(obj_header.name, 60);
if (std::string(obj_header.name).find("-ag") != std::string::npos) {
@ -316,10 +262,10 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
assert(false);
}
auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.size);
auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.object_count);
add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.size, dgo_base_name);
reader.ffwd(obj_header.size);
add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.object_count, dgo_base_name);
reader.ffwd(obj_header.object_count);
}
// check we're at the end

7
tools/CMakeLists.txt Normal file
View File

@ -0,0 +1,7 @@
add_executable(dgo_unpacker
dgo_unpacker.cpp)
target_link_libraries(dgo_unpacker common)
add_executable(dgo_packer
dgo_packer.cpp)
target_link_libraries(dgo_packer common)

50
tools/dgo_packer.cpp Normal file
View File

@ -0,0 +1,50 @@
#include <cstdio>
#include "common/versions.h"
#include "common/util/FileUtil.h"
#include "common/util/BinaryWriter.h"
#include "third-party/json.hpp"
int main(int argc, char** argv) {
printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
printf("DGO Packing Tool\n");
if (argc < 3) {
printf("usage: dgo_packer <path> <dgo description file>\n");
return 1;
}
std::string out_path = argv[1];
for (int i = 2; i < argc; i++) {
std::string file_name = argv[i];
std::string file_text = file_util::read_text_file(file_name);
auto x = nlohmann::json::parse(file_text);
std::string out_file_name = x["file_name"];
std::string internal_name = x["internal_name"];
printf("Packing %s\n", internal_name.c_str());
BinaryWriter writer;
writer.add<u32>(x["objects"].size());
writer.add_cstr_len(x["internal_name"].get<std::string>().c_str(), 60);
for (auto& entry : x["objects"]) {
auto obj_data =
file_util::read_binary_file(file_util::combine_path(out_path, entry["unique_name"]));
// size
writer.add<uint32_t>(obj_data.size());
// name
writer.add_str_len(entry["internal_name"].get<std::string>().c_str(), 60);
// data
writer.add_data(obj_data.data(), obj_data.size());
// pad
while (writer.get_size() & 0xf) {
writer.add<uint8_t>(0);
}
}
writer.write_to_file(file_util::combine_path(out_path, "mod_" + out_file_name));
}
printf("Done\n");
return 0;
}

11
tools/dgo_tools.md Normal file
View File

@ -0,0 +1,11 @@
## DGO Tools
The DGO packer and unpacker can be used to extract and repack files in DGOs.
### Unpacking
Create a folder for the output, then run:
```tools/dgo_unpacker <path to output> <path to DGO files...>```
It will then place the files in the output folder. You may specify multiple DGO files, but this is not recommended because sometimes different DGO files have object files with the same name but different data. It will also create a DGO description file with the same name as the DGO file but with a `.txt` extension.
### Repacking
```tools/dgo_packer <path to folder with object files> <path to DGO description file>```
It will repack the DGO. The name will be the same as the original DGO, but with `mod_` in the front.

37
tools/dgo_unpacker.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <cstdio>
#include "common/versions.h"
#include "common/util/FileUtil.h"
#include "common/util/DgoReader.h"
int main(int argc, char** argv) {
printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
printf("DGO Unpacking Tool\n");
if (argc < 3) {
printf("usage: dgo_unpacker <output path> <dgo files>\n");
return 1;
}
std::string out_path = argv[1];
for (int i = 2; i < argc; i++) {
std::string file_name = argv[i];
std::string base = file_util::base_name(file_name);
printf("Unpacking %s\n", base.c_str());
// read the file
auto data = file_util::read_binary_file(file_name);
// read as a DGO
auto dgo = DgoReader(base, data);
// write dgo description
file_util::write_text_file(file_util::combine_path(out_path, base + ".txt"),
dgo.description_as_json());
// write files:
for (auto& entry : dgo.entries()) {
file_util::write_binary_file(file_util::combine_path(out_path, entry.unique_name),
(void*)entry.data.data(), entry.data.size());
}
}
printf("Done\n");
return 0;
}