Documentation cleanup and some feature improvements (#1155)

* ci: switch to codacy for coverage

* docs: update badges

* decomp: allow overriding config flags via CLI

* cleanup: top level file cleanup

* docs: big README overhaul

Attempt to close #1128 and #1086

* decomp: attempt to detect if `iso_data` is missing or wrongly extracted

* game: switch to `fpng` for screenshots, allow for compression

closes #1035

* game: switch vsync control to a checkbox

* lint: format cpp files

* lint: format json files

* docs/scripts: organize taskfile
This commit is contained in:
Tyler Wilding 2022-02-12 17:48:50 -05:00 committed by GitHub
parent 24578b64b9
commit ffb04ddd10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 4608 additions and 3170 deletions

View File

@ -77,10 +77,11 @@ jobs:
./test_code_coverage.sh
fi
- name: Coveralls
- name: Submit Coverage Report to Codacy
if: ${{ matrix.compiler }} != 'clang'
uses: coverallsapp/github-action@master
continue-on-error: true # Sometimes Coveralls has intermittent problems, and codecoverage isn't critical to our success
uses: codacy/codacy-coverage-reporter-action@v1
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: ./build/goalc-test_coverage.info
project-token: ${{ secrets.CODACY_PROJECT_KEY }}
# lcov report
coverage-reports: ./build/goalc-test_coverage.info

3
.gitignore vendored
View File

@ -8,6 +8,9 @@ build/*
decompiler_out/*
logs/*
# wsl apparently builds to here?
linux-default/
# for Nix
/result*

0
.gitmodules vendored
View File

View File

@ -67,7 +67,15 @@
"type" : "default",
"project" : "CMakeLists.txt",
"projectTarget" : "goalc.exe (bin\\goalc.exe)",
"name" : "Run - REPL"
"name" : "Run - REPL",
"args" : []
},
{
"type" : "default",
"project" : "CMakeLists.txt",
"projectTarget" : "goalc.exe (bin\\goalc.exe)",
"name" : "Run - REPL - Auto Listen",
"args" : [ "-auto-lt" ]
},
{
"type" : "default",

View File

@ -1,40 +0,0 @@
{
"configurations": [
{
"name": "Debug",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${configurationType}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"addressSanitizerEnabled": false
},
{
"name": "Release",
"generator": "Ninja",
"configurationType": "Release",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${configurationType}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"addressSanitizerEnabled": false
},
{
"name": "Debug (clang-cl)",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "clang_cl_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${configurationType}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"addressSanitizerEnabled": false
},
{
"name": "Release (clang-cl)",
"generator": "Ninja",
"configurationType": "Release",
"inheritEnvironments": [ "clang_cl_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${configurationType}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"addressSanitizerEnabled": false
}
]
}

View File

@ -1,6 +1,6 @@
ISC License
Copyright (c) 2020-2021 water111
Copyright (c) 2020-2021 OpenGOAL Team
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above

171
README.md
View File

@ -3,11 +3,11 @@
</p>
<p align="center">
<a href="https://water111.github.io/jak-project/" rel="nofollow"><img src="https://img.shields.io/badge/Documentation-Here-informational" alt="Documentation Badge" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/water111/jak-project/workflows/Linux/badge.svg"><img src="https://github.com/water111/jak-project/workflows/Linux/badge.svg" alt="Linux" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/water111/jak-project/workflows/Windows/badge.svg"><img src="https://github.com/water111/jak-project/workflows/Windows/badge.svg" alt="Windows" style="max-width:100%;"></a>
<a href="https://coveralls.io/github/water111/jak-project?branch=master" rel="nofollow"><img src="https://coveralls.io/repos/github/water111/jak-project/badge.svg?branch=master" alt="Coverage Status" style="max-width:100%;"></a>
<a href="https://www.codacy.com/gh/water111/jak-project/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=xTVaser/jak-project&amp;utm_campaign=Badge_Grade" rel="nofollow"><img src="https://app.codacy.com/project/badge/Grade/7c3cdc07523f43aca3433484ebc62ff9" alt="Codacy Badge" style="max-width:100%;"></a>
<a href="https://open-goal.github.io/" rel="nofollow"><img src="https://img.shields.io/badge/Documentation-Here-informational" alt="Documentation Badge" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/workflows/Linux/badge.svg"><img src="https://github.com/open-goal/jak-project/workflows/Linux/badge.svg" alt="Linux" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/workflows/Windows/badge.svg"><img src="https://github.com/open-goal/jak-project/workflows/Windows/badge.svg" alt="Windows" style="max-width:100%;"></a>
<a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&utm_medium=referral&utm_content=open-goal/jak-project&utm_campaign=Badge_Coverage" rel="nofollow"><img src="https://app.codacy.com/project/badge/Coverage/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
<a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=open-goal/jak-project&amp;utm_campaign=Badge_Grade" rel="nofollow"><img src="https://app.codacy.com/project/badge/Grade/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
<a href="https://discord.gg/V82sTJGEAs"><img src="https://img.shields.io/discord/756287461377703987" alt="Discord"></a>
<a href="https://makeapullrequest.com"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square" alt=PRs Welcome></a>
</p>
@ -20,17 +20,22 @@
- [Project Description](#project-description)
- [Current Status](#current-status)
- [What's Next](#whats-next)
- [Getting Started - Linux (Ubuntu)](#getting-started---linux-ubuntu)
- [Getting Started - Linux (Arch)](#getting-started---linux-arch)
- [Getting Started - Nixpkgs](#getting-started---nixpkgs)
- [Getting Started - Linux](#getting-started---linux)
- [Ubuntu (20.04)](#ubuntu-2004)
- [Arch](#arch)
- [With Nix](#with-nix)
- [Getting Started - Windows](#getting-started---windows)
- [Required Software](#required-software)
- [Setting up and Opening the Project](#setting-up-and-opening-the-project)
- [Building and Running the Game](#building-and-running-the-game)
- [Extract Assets](#extract-assets)
- [Build Game](#build-game)
- [Run Game](#run-game)
- [Build the Game](#build-the-game)
- [Run the Game](#run-the-game)
- [Connecting the REPL to the Game](#connecting-the-repl-to-the-game)
- [Running the Game Without Auto-Booting](#running-the-game-without-auto-booting)
- [Interacting with the Game](#interacting-with-the-game)
- [Project Layout](#project-layout)
- [Directory Layout](#directory-layout)
- [On Windows / Visual Studio](#on-windows--visual-studio)
<!-- tocstop -->
@ -53,6 +58,7 @@ We support both Linux and Windows on x86-64.
We have a Discord server where we discuss development. https://discord.gg/V82sTJGEAs
## Current Status
So far, we've decompiled around 400,000 lines of GOAL code, out of an estimated 500,000 total lines. We have a working OpenGL renderer which renders most of the game world and foreground. Levels are fully playable, and you can finish the game with 100% completion! There is currently *no* audio.
Here are some screenshots of the renderer:
@ -73,13 +79,15 @@ We don't save any assets from the game - you must bring your own copy of the gam
- Improve the decompiler. We are always finding new features and macros in the GOAL language.
- Investigate more complicated renderers. We have an in-progress port of the "merc" foreground renderer, shown in the screenshots above.
## Getting Started - Linux (Ubuntu)
## Getting Started - Linux
### Ubuntu (20.04)
Install packages and init repository:
```sh
sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev
git submodule update --init --recursive
sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev python
sudo snap install task --classic
```
Compile:
@ -95,21 +103,23 @@ Run tests:
```
Note: we have found that `clang` and `lld` are significantly faster to compile and link than `gcc`, generate faster code, and have better warning messages. To install these:
```sh
sudo apt install lld clang
```
and run `cmake` (in a fresh build directory) with:
```sh
cmake -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ ..
```
## Getting Started - Linux (Arch)
### Arch
Install packages and init repository:
```sh
sudo pacman -S gcc make cmake base-devel g++ nasm
git submodule update --init --recursive
sudo pacman -S gcc make cmake base-devel g++ nasm taskfile-git python
```
Compile:
@ -124,7 +134,7 @@ Run tests:
./test.sh
```
## Getting Started - Nixpkgs
### With Nix
If your Nix supports flakes:
@ -146,27 +156,35 @@ nix-build -A packages.x86_64-linux.jak-asan # package with Clang ASan build
## Getting Started - Windows
If you do not have it already, install Visual Studio 2019 or 2022 and get the `Desktop development with C++` workload during the installation process. If you already have Visual Studio installed and don't have this, open your `Visual Studio Installer` and modify the installation.
### Required Software
On Windows, it's recommended to get Scoop to use as a package manager, making the follow steps _much_ easier. Follow the steps on the bottom of the homepage here https://scoop.sh/
We primarily use Visual Studio for development on Windows for C++ development. Download the community addition from [here](https://visualstudio.microsoft.com/vs/)
Once Scoop is installed, run the following command:
You will require the `Desktop development with C++` workload. This can be selected during the installation, or after via the `Visual Studio Installer` and modifying the Visual Studio Installation.
On Windows, it's recommended to get Scoop to use as a package manager, making the following steps _much_ easier. Follow the steps on the bottom of the homepage [here](https://scoop.sh/)
Once Scoop is installed, run the following commands:
```sh
scoop install git llvm nasm
scoop install git llvm nasm python
scoop bucket add extras
scoop install task
```
Initialize the repository's third-party dependencies:
### Setting up and Opening the Project
Clone the repository by running the following command in your folder of choice.
```sh
git submodule update --init --recursive
git clone https://github.com/open-goal/jak-project.git
```
Open the project as a CMake project.
This will create a `jak-project` folder, we will open the project as a CMake project via Visual Studio.
![](./docs/markdown/imgs/windows/open-project.png)
Then build the entire project as "Release". You can also press Ctrl+Shift+B as a hotkey for Build All.
Then build the entire project as `Windows Release (clang-cl)`. You can also press Ctrl+Shift+B as a hotkey for Build All. We currently prefer `clang-cl` on Windows as opposed to `msvc`, though it should work as well!
![](./docs/markdown/imgs/windows/release-build.png)
![](./docs/markdown/imgs/windows/build-all.png)
@ -174,25 +192,94 @@ Then build the entire project as "Release". You can also press Ctrl+Shift+B as a
## Building and Running the Game
Getting a running game involves 4 steps:
1. Build C++ tools (follow steps above)
2. Extract assets from game
3. Build game
4. Run game
1. Build C++ tools (follow Getting Started steps above for your platform)
2. Extract assets from the game
3. Build the game
4. Run the game
### Extract Assets
Running the decompiler on the entire game is slow and not needed, so it is recommended to just run it on data. Edit `decompiler/config/jak1-ntsc_black_label.jsonc` and disable `decompile_code`.
```json
"decompile_code": false, // change this to false, don't decompile code
The first step is to extract your ISO file contents into the `iso_data/<game-name>` folder. In the case of jak this is `iso_data/jak1`.
Once this is done, open a terminal in the `jak-project` folder and run the following:
```sh
task extract-jak1
```
Place a copy of the game's files in `iso_data/jak1/`, then run the decompiler with the `scripts/shell/decomp.sh` (Linux) or `scripts/batch/decomp-jak1.bat` (Windows) script.
### Build the Game
### Build Game
Run the OpenGOAL compiler `out/build/goalc/goalc` (keep in mind that the build directories may vary by system, on Windows the equivalent is similar to `out/build/Release/bin/goalc`), or use one of the `gc` scripts. Enter `(mi)` to build the `"iso"` target, which contains everything we have placed in the build system so far.
The next step is to build the game itself. To do so, in the same terminal run the following:
### Run Game
In a separate terminal, start the runtime with `out/build/game/gk -boot -fakeiso -debug`, or instead run the `gk-display.bat` script (Windows). The game should boot up automatically! If you want to connect the REPL to the live game and use it for inspecting code or changing things around, rhen, in the OpenGOAL compiler window (REPL), run `(mi)` to create the data for the game and give the REPL information for running code and `(lt)` to connect. If you want to start the runtime but not boot up the game, remove the `-boot` argument or run the regular `gk.bat` script, and after connecting to the runtime from the REPL, run `(lg)` to load the game engine and `(test-play)` to start it. Assuming it all works right, it will look something like this:
```sh
task repl
```
You will be greeted with a prompt like so:
```sh
_____ _____ _____ _____ __
| |___ ___ ___| __| | _ | |
| | | . | -_| | | | | | | |__
|_____| _|___|_|_|_____|_____|__|__|_____|
|_|
Welcome to OpenGOAL 0.8!
Run (repl-help) for help with common commands and REPL usage.
Run (lt) to connect to the local target.
g >
```
Run the following to build the game:
```sh
g > (mi)
```
### Run the Game
Finally the game can be ran. Open a second terminal from the `jak-project` directly and run the following:
```sh
task boot-game
```
The game should boot automatically if everything was done correctly.
#### Connecting the REPL to the Game
Connecting the REPL to the game allows you to inspect and modify code or data while the game is running.
To do so, in the REPL after a successful `(mi)`, run the following:
```sh
g > (lt)
```
If successfuly your prompt should change to:
```sh
gc>
```
For example, running the following will print out some basic information about Jak:
```sh
gc> *target*
```
#### Running the Game Without Auto-Booting
You can also start up the game without booting. To do so run the following in one terminal
```sh
task run-game
```
And then in your REPL run the following (after a successful `(mi)`):
```sh
g > (lt)
[Listener] Socket connected established! (took 0 tries). Waiting for version...
Got version 0.8 OK!
@ -207,11 +294,15 @@ gc> (test-play)
gc>
```
### Interacting with the Game
In the graphics window, you can use the period key to bring up the debug menu. Controllers also work, using the same mapping as the original game.
Check out the `pc_debug`, `examples` and `engine/pc/` folders under `goal_src` for some examples of GOAL code we wrote. The debug files have instructions for how to run them if they are not loaded automatically by the engine.
## Project Layout
There are four main components to the project.
The first is `goalc`, which is a GOAL compiler for x86-64. Our implementation of GOAL is called OpenGOAL. All of the compiler source code is in `goalc`. To run the compiler on Linux, there is a script `gc.sh`. On Windows, there is a `gc.bat` scripts and a `gc-no-lt.bat` script, the latter of which will not attempt to automatically attach to a running target. The compiler is controlled through a prompt which can be used to enter commands to compile, connect to a running GOAL program for interaction, run the OpenGOAL debugger, or, if you are connected to a running GOAL program, can be used as a REPL to run code interactively. In addition to compiling code files, the compiler has features to pack and build data files.
@ -304,7 +395,3 @@ The final component is the "runtime", located in `game`. This is the part of the
- `json`: A JSON library.
- `replxx`: Used for the REPL input. Supports history and useful editing shortcuts.
- `svpng`: Save a PNG file.
### On Windows / Visual Studio
Until 16.9 Preview 4, when attaching a debugger to the ASan build, you must disable breaking on Win32 Access Violation exceptions. See the relevant section `Debugging - Exceptions` here https://devblogs.microsoft.com/cppblog/asan-for-windows-x64-and-debug-build-support/#known-issues

View File

@ -1,35 +1,43 @@
version: '3'
vars:
# make this OS agnostic eventually
RELEASE_PATH: './out/build/Release/bin'
includes:
build: ./scripts/tasks/Taskfile_{{OS}}.yml
tasks:
# GENERAL
extract-jak1:
cmds:
- '{{.RELEASE_PATH}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out" "decompile_code=false"'
boot-game:
cmds:
- "{{.RELEASE_PATH}}/gk -boot -fakeiso -debug -v"
run-game:
cmds:
- "{{.RELEASE_PATH}}/gk -fakeiso -debug -v"
repl:
env:
OPENGOAL_DECOMP_DIR: "jak1/"
cmds:
- "{{.RELEASE_PATH}}/goalc"
# DEVELOPMENT
format:
cmds:
- cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test -i
# npm install -g prettier
- cmd: prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
ignore_error: true
run-game:
cmds:
- "{{.RELEASE_PATH}}/gk.exe -fakeiso -debug -v"
run-game-headless:
cmds:
- "{{.RELEASE_PATH}}/gk.exe -fakeiso -debug -nodisplay"
repl:
env:
OPENGOAL_DECOMP_DIR: "jak1/"
cmds:
- "{{.RELEASE_PATH}}/goalc.exe"
- "{{.RELEASE_PATH}}/gk -fakeiso -debug -nodisplay"
repl-lt:
env:
OPENGOAL_DECOMP_DIR: "jak1/"
cmds:
- "{{.RELEASE_PATH}}/goalc.exe -auto-lt"
- "{{.RELEASE_PATH}}/goalc -auto-lt"
# DECOMPILING
decomp:
cmds:
- '{{.RELEASE_PATH}}/decompiler.exe "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out"'
- '{{.RELEASE_PATH}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out"'
decomp-clean:
cmds:
- rm ./decompiler_out/**/*.asm
@ -47,20 +55,10 @@ tasks:
decomp-watch:
cmds:
- watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/
clean-all-types:
cmds:
- python ./scripts/cleanup-all-types.py
analyze-ee-memory:
cmds:
- '{{.RELEASE_PATH}}/memory_dump_tool.exe "{{.FILE}}" ./ > ee-analysis.log'
watch-pcsx2:
cmds:
- watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
vars:
SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
# TESTS
offline-tests:
cmds:
- '{{.RELEASE_PATH}}/offline-test.exe "./iso_data/jak1"'
- '{{.RELEASE_PATH}}/offline-test "./iso_data/jak1"'
add-reference-test:
cmds:
- task: decomp-file
@ -73,7 +71,7 @@ tasks:
update-reference-tests:
cmds:
- cmd: python ./scripts/default-file-or-folder.py --path failures
- cmd: '{{.RELEASE_PATH}}/offline-test.exe "./iso_data/jak1" --dump-mode'
- cmd: '{{.RELEASE_PATH}}/offline-test "./iso_data/jak1" --dump-mode'
ignore_error: true
- python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/
- task: offline-tests
@ -84,11 +82,20 @@ tasks:
- python ./scripts/find-label-types.py --file "{{.FILES}}"
type-test:
cmds:
- cmd: '{{.RELEASE_PATH}}/goalc-test.exe --gtest_brief=0 --gtest_filter="*MANUAL_TEST_TypeConsistencyWithBuildFirst*"'
- cmd: '{{.RELEASE_PATH}}/goalc-test --gtest_brief=0 --gtest_filter="*MANUAL_TEST_TypeConsistencyWithBuildFirst*"'
ignore_error: true
check-gsrc-file:
cmds:
- python ./scripts/check-gsrc-file.py --files "{{.FILES}}"
# TOOLS
analyze-ee-memory:
cmds:
- '{{.RELEASE_PATH}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log'
watch-pcsx2:
cmds:
- watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
vars:
SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
update-gsrc:
cmds:
- python ./scripts/next-decomp-file.py --files "{{.FILES}}"
@ -100,3 +107,7 @@ tasks:
cast-repl:
cmds:
- cmd: python ./scripts/cast-repl.py
# Doesn't Currently Work
# clean-all-types:
# cmds:
# - python ./scripts/cleanup-all-types.py

View File

@ -13,7 +13,8 @@
#include "common/util/BinaryReader.h"
#include "BinaryWriter.h"
#include "common/common_types.h"
#include "third-party/svpng.h"
#include "third-party/fpng/fpng.cpp"
#include "third-party/fpng/fpng.h"
#include "third-party/fmt/core.h"
#include "third-party/lzokay/lzokay.hpp"
@ -99,15 +100,17 @@ void write_binary_file(const std::string& name, const void* data, size_t size) {
fclose(fp);
}
void write_rgba_png(const std::string& name, void* data, int w, int h) {
FILE* fp = fopen(name.c_str(), "wb");
if (!fp) {
throw std::runtime_error("couldn't open file " + name);
void write_rgba_png(const std::string& name, void* data, int w, int h, bool compress) {
auto flags = 0;
if (!compress) {
flags = fpng::FPNG_FORCE_UNCOMPRESSED;
}
svpng(fp, w, h, (const unsigned char*)data, 1);
auto ok = fpng::fpng_encode_image_to_file(name.c_str(), data, w, h, 4, flags);
fclose(fp);
if (!ok) {
throw std::runtime_error("couldn't save png file " + name);
}
}
void write_text_file(const std::string& file_name, const std::string& text) {
@ -174,6 +177,10 @@ std::string combine_path(const std::string& parent, const std::string& child) {
return parent + "/" + child;
}
bool file_exists(const std::string& path) {
return std::filesystem::exists(path);
}
std::string base_name(const std::string& filename) {
size_t pos = 0;
ASSERT(!filename.empty());

View File

@ -18,12 +18,13 @@ std::string get_project_path();
std::string get_file_path(const std::vector<std::string>& input);
bool create_dir_if_needed(const std::string& path);
void write_binary_file(const std::string& name, const void* data, size_t size);
void write_rgba_png(const std::string& name, void* data, int w, int h);
void write_rgba_png(const std::string& name, void* data, int w, int h, bool compress);
void write_text_file(const std::string& file_name, const std::string& text);
std::vector<uint8_t> read_binary_file(const std::string& filename);
std::string read_text_file(const std::string& path);
bool is_printable_char(char c);
std::string combine_path(const std::string& parent, const std::string& child);
bool file_exists(const std::string& path);
std::string base_name(const std::string& filename);
void init_crc();
uint32_t crc32(const uint8_t* data, size_t size);

View File

@ -305,4 +305,8 @@ std::string diff_strings(const std::string& lhs, const std::string& rhs) {
}
}
return "";
}
}
std::vector<std::string> split_string(const ::std::string& str, char delimiter) {
return SplitString(str, delimiter);
}

View File

@ -1,8 +1,11 @@
#pragma once
#include <string>
#include <vector>
/*!
* Diff two strings. This uses the code from gtest's diff implementation.
*/
std::string diff_strings(const std::string& lhs, const std::string& rhs);
std::string diff_strings(const std::string& lhs, const std::string& rhs);
std::vector<std::string> split_string(const ::std::string& str, char delimiter = '\n');

View File

@ -22,14 +22,24 @@ nlohmann::json read_json_file_from_config(const nlohmann::json& cfg, const std::
/*!
* Parse the main config file and return decompiler config.
*/
Config read_config_file(const std::string& path_to_config_file) {
Config read_config_file(const std::string& path_to_config_file,
const std::map<std::string, bool>& overrides) {
Config config;
auto config_str = file_util::read_text_file(path_to_config_file);
auto cfg = parse_commented_json(config_str, path_to_config_file);
// Override JSON
for (auto const& [key, val] : overrides) {
fmt::print("[Config] - Overwriting '{}' with '{}'\n", key, val);
cfg[key] = val;
}
config.game_version = cfg.at("game_version").get<int>();
config.text_version = cfg.at("text_version").get<GameTextVersion>();
config.game_name = cfg.at("game_name").get<std::string>();
if (cfg.contains("expected_elf_name")) {
config.expected_elf_name = cfg.at("expected_elf_name").get<std::string>();
}
auto inputs_json = read_json_file_from_config(cfg, "inputs_file");
config.dgo_names = inputs_json.at("dgo_names").get<std::vector<std::string>>();

View File

@ -112,6 +112,7 @@ struct Config {
bool is_pal = false;
std::string game_name;
std::string expected_elf_name;
GameTextVersion text_version = GameTextVersion::JAK1_V1;
std::unordered_set<std::string> allowed_objects;
@ -138,6 +139,7 @@ struct Config {
DecompileHacks hacks;
};
Config read_config_file(const std::string& path_to_config_file);
Config read_config_file(const std::string& path_to_config_file,
const std::map<std::string, bool>& overrides);
} // namespace decompiler

View File

@ -2,6 +2,7 @@
"game_version": 1,
"text_version": 10,
"game_name": "jak1",
"expected_elf_name": "SCUS_971.24",
// if you want to filter to only some object names.
// it will make the decompiler much faster.

View File

@ -808,32 +808,27 @@
[46, "(function float :behavior manipy)"]
],
"beach-obs": [
[8, "(function none :behavior camera-tracker)"]
],
"beach-obs": [[8, "(function none :behavior camera-tracker)"]],
"jungleb-obs": [
[4, "(function none :behavior camera-tracker)"]
],
"jungleb-obs": [[4, "(function none :behavior camera-tracker)"]],
"village2-obs": [
[4, "(function sparticle-launch-control :behavior ogreboss-village2)"]
],
"generic-obs": [
[5, "(function symbol :behavior touch-tracker)"]
],
"generic-obs": [[5, "(function symbol :behavior touch-tracker)"]],
"plant-boss": [
[22, "(function none :behavior camera-tracker)"]
],
"plant-boss": [[22, "(function none :behavior camera-tracker)"]],
"pelican": [
[1, "(function sparticle-launch-control :behavior pelican)"],
[5, "(function pelican int)"],
[6, "(function int)"],
[10, "(function int)"],
[15, "(function process int symbol event-message-block object :behavior pelican)"],
[
15,
"(function process int symbol event-message-block object :behavior pelican)"
],
[16, "(function none :behavior pelican)"],
[24, "(function pelican int)"],
[25, "(function int)"],
@ -852,9 +847,7 @@
[69, "(function part-tracker vector)"]
],
"collide-edge-grab": [
[0, "(function surface object object int float)"]
],
"collide-edge-grab": [[0, "(function surface object object int float)"]],
"target": [
[1, "(function none :behavior target)"],
@ -862,13 +855,9 @@
[16, "(function target float)"]
],
"water": [
[13, "(function water-vol object)"]
],
"water": [[13, "(function water-vol object)"]],
"rolling-lightning-mole": [
[38, "(function none :behavior lightning-mole)"]
],
"rolling-lightning-mole": [[38, "(function none :behavior lightning-mole)"]],
"target-flut": [
[23, "(function nav-enemy none :behavior target)"],
@ -890,22 +879,16 @@
[99, "(function symbol)"]
],
"swamp-blimp": [
[29, "(function int none)"]
],
"swamp-blimp": [[29, "(function int none)"]],
"jungle-mirrors": [
[14, "(function string none)"],
[32, "(function entity-actor (pointer symbol) object)"] // passed to `actor-link-info` function
],
"voicebox": [
[1, "(function none :behavior voicebox)"]
],
"voicebox": [[1, "(function none :behavior voicebox)"]],
"seagull": [
[30, "(function process symbol)"]
],
"seagull": [[30, "(function process symbol)"]],
"placeholder-do-not-add-below": []
}

View File

@ -232,14 +232,7 @@
"collide-probe-node", // CFG
// collide-edge-grab
// CFG
// CFG
"(method 15 collide-edge-work)", // CFG
// CFG
// CFG
// CFG
// CFG
"(method 12 collide-mesh)",
@ -250,29 +243,14 @@
// ambient
"ambient-inspect",
// target BUG
//"target-falling-anim-trans", // CFG resolution
// target2 BUG
"look-for-points-of-interest", // Failed to split nested sc - looks like dead code to me
// collide-cache
"(method 10 collide-puss-work)", // CFG
"(method 9 collide-puss-work)", // decompiler crash
// decompiler crash
// CFG
// CFG
// CFG
// "(method 14 collide-cache)", // CFG
// CFG
// "(method 26 collide-cache)", // CFG
//"(method 21 collide-cache)", // CFG
"(method 32 collide-cache)", // CFG
// memory-usage BUG
//"(method 14 level)",
// ocean
"draw-large-polygon-ocean", // CFG
@ -449,50 +427,57 @@
"draw-bones": [0, 1, 2, 8, 81],
"draw-bones-hud": [7, 8],
"(method 16 drawable-tree)": [7, 9, 10],
"(method 21 collide-cache)" : [3, 5, 19, 20,24,25,28,29],
"(method 14 collide-cache)" : [0, 1,2, 3, 4, 5],
"(method 9 collide-mesh-cache)" : [0, 1, 2, 5],
"(method 42 collide-shape)" : [0, 1, 2, 3, 4, 7],
"(method 23 collide-shape-prim-group)" : [1, 2, 3, 4, 5],
"(method 23 collide-shape-prim-mesh)" : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],
"(method 23 collide-shape-prim-sphere)" : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],
"(method 18 collide-shape-prim-sphere)" : [1, 3,4, 5, 7],
"(method 45 collide-shape)" : [0, 16, 42, 67, 92],
"(method 24 collide-shape-prim)" : [2, 3, 4, 5, 1],
"(method 20 collide-shape-prim-group)" : [11],
"(method 28 collide-shape-prim-mesh)" : [10],
"(method 40 collide-shape)": [0, 2,5,6,7, 11,12, 28, 43, 58, 63],
"(method 15 collide-shape-prim-group)" : [1, 2, 3, 4, 5, 6],
"(method 16 collide-shape-prim)" : [1, 2, 3, 4, 5, 6],
"(method 15 collide-shape-prim-sphere)" : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
"(method 15 collide-shape-prim-mesh)" : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 29],
"(method 35 collide-shape)" : [1],
"(method 22 collide-cache)":[2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 12 collide-shape-prim-sphere)" : [0, 1],
"(method 12 collide-shape-prim-group)" : [1, 2, 3, 4],
"(method 24 collide-cache)" : [2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 14 collide-shape-prim-sphere)" : [0, 1, 2, 3],
"(method 14 collide-shape-prim-group)" : [0, 1, 2, 3, 4],
"(method 23 collide-cache)" : [2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 13 collide-shape-prim-sphere)" : [0, 1, 2],
"(method 13 collide-shape-prim-group)" : [0, 1, 2,3,4],
"(method 19 collide-cache)" : [0, 1, 3, 4, 5, 18, 19],
"(method 10 collide-mesh)" : [1, 2, 4, 5],
"target-falling-anim-trans" : [5, 6],
"(method 19 process-drawable)" : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
"(anon-function 9 racer)" : [75],
"(method 13 collide-edge-work)" : [0, 2],
"(method 17 collide-edge-work)" : [0, 1, 2, 3, 4],
"(method 9 edge-grab-info)" : [15, 16, 18, 19, 21, 22, 24],
// "(method 18 collide-edge-work)" : [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24],
"target-falling-anim-trans" : [5, 6],
"(method 27 nav-mesh)" : [8],
"(method 21 collide-cache)": [3, 5, 19, 20, 24, 25, 28, 29],
"(method 14 collide-cache)": [0, 1, 2, 3, 4, 5],
"(method 9 collide-mesh-cache)": [0, 1, 2, 5],
"(method 42 collide-shape)": [0, 1, 2, 3, 4, 7],
"(method 23 collide-shape-prim-group)": [1, 2, 3, 4, 5],
"(method 23 collide-shape-prim-mesh)": [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14
],
"(method 23 collide-shape-prim-sphere)": [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14
],
"(method 18 collide-shape-prim-sphere)": [1, 3, 4, 5, 7],
"(method 45 collide-shape)": [0, 16, 42, 67, 92],
"(method 24 collide-shape-prim)": [2, 3, 4, 5, 1],
"(method 20 collide-shape-prim-group)": [11],
"(method 28 collide-shape-prim-mesh)": [10],
"(method 40 collide-shape)": [0, 2, 5, 6, 7, 11, 12, 28, 43, 58, 63],
"(method 15 collide-shape-prim-group)": [1, 2, 3, 4, 5, 6],
"(method 16 collide-shape-prim)": [1, 2, 3, 4, 5, 6],
"(method 15 collide-shape-prim-sphere)": [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19
],
"(method 15 collide-shape-prim-mesh)": [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 29
],
"(method 35 collide-shape)": [1],
"(method 22 collide-cache)": [2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 12 collide-shape-prim-sphere)": [0, 1],
"(method 12 collide-shape-prim-group)": [1, 2, 3, 4],
"(method 24 collide-cache)": [2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 14 collide-shape-prim-sphere)": [0, 1, 2, 3],
"(method 14 collide-shape-prim-group)": [0, 1, 2, 3, 4],
"(method 23 collide-cache)": [2, 3, 4, 13, 14, 15, 23, 24, 25, 33, 34, 35],
"(method 13 collide-shape-prim-sphere)": [0, 1, 2],
"(method 13 collide-shape-prim-group)": [0, 1, 2, 3, 4],
"(method 19 collide-cache)": [0, 1, 3, 4, 5, 18, 19],
"(method 10 collide-mesh)": [1, 2, 4, 5],
"(method 19 process-drawable)": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
"(anon-function 9 racer)": [75],
"(method 13 collide-edge-work)": [0, 2],
"(method 17 collide-edge-work)": [0, 1, 2, 3, 4],
"(method 9 edge-grab-info)": [15, 16, 18, 19, 21, 22, 24],
// "(method 18 collide-edge-work)" : [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24],
"target-falling-anim-trans": [5, 6],
"(method 27 nav-mesh)": [8],
"(method 32 nav-control)": [12, 21, 32],
"start-collect-nav": [0],
"end-collect-nav": [0],
"robotboss-always-trans": [9, 10, 11, 12, 13, 14, 15, 18, 29, 36, 45, 48],
"target-flut-falling-anim-trans" : [5, 6],
"(method 27 ropebridge)" : [5, 7],
"target-flut-falling-anim-trans": [5, 6],
"(method 27 ropebridge)": [5, 7],
"(anon-function 5 orbit-plat)": [4, 5, 11],
"(anon-function 21 plant-boss)": [0, 1, 3, 4],
"(method 55 ram-boss)": [5],
@ -586,16 +571,23 @@
],
"mips2c_jump_table_functions": {
"decompress-fixed-data-to-accumulator" : [108, 199, 233, 286, 301, 366, 387, 100, 155, 199, 261, 286, 335, 366, 402, 100],
"decompress-frame-data-to-accumulator" : [84, 92, 119, 140, 205, 220, 273, 307, 84, 107, 119, 174, 205, 248, 273, 354],
"decompress-frame-data-pair-to-accumulator" : [117, 125, 169, 197, 293, 318, 408, 459, 117, 150, 169, 248, 293, 366, 408, 533]
"decompress-fixed-data-to-accumulator": [
108, 199, 233, 286, 301, 366, 387, 100, 155, 199, 261, 286, 335, 366, 402,
100
],
"decompress-frame-data-to-accumulator": [
84, 92, 119, 140, 205, 220, 273, 307, 84, 107, 119, 174, 205, 248, 273,
354
],
"decompress-frame-data-pair-to-accumulator": [
117, 125, 169, 197, 293, 318, 408, 459, 117, 150, 169, 248, 293, 366, 408,
533
]
},
// there are some missing textures. I don't know what the game actually does here.
// the format for entries is [level, tpage, index]
"missing_textures": [
["finalboss", 1419, 3]
],
"missing_textures": [["finalboss", 1419, 3]],
// some object files have garbage pad data at the end which makes the decompiler
// assume they must be different files, such as the art group for orb-cache-top.
@ -631,5 +623,3 @@
"plat-ag"
]
}

View File

@ -263,8 +263,8 @@
"audio_dir_file_name": "",
"streamed_audio_file_names": ["VAGWAD.ENG", "VAGWAD.JAP"],
"levels_to_extract":[
"levels_to_extract": [
"BEA.DGO",
"CIT.DGO",
"DAR.DGO",

View File

@ -954,9 +954,7 @@
["L314", "float", true]
],
"seagull": [
["L226", "(inline-array air-box)", 10]
],
"seagull": [["L226", "(inline-array air-box)", 10]],
"rolling-race-ring": [
["L94", "vector"],
@ -1172,21 +1170,13 @@
["L101", "attack-info"]
],
"citadel-obs": [
["L265", "attack-info"]
],
"citadel-obs": [["L265", "attack-info"]],
"jungleb-obs": [
["L34", "vector"]
],
"jungleb-obs": [["L34", "vector"]],
"jungle-obs": [
["L164", "attack-info"]
],
"jungle-obs": [["L164", "attack-info"]],
"misty-obs": [
["L135", "rigid-body-platform-constants"]
],
"misty-obs": [["L135", "rigid-body-platform-constants"]],
"village2-obs": [
["L274", "rigid-body-platform-constants"],
@ -1230,9 +1220,7 @@
["L257", "vector"]
],
"firecanyon-obs": [
["L57", "attack-info"]
],
"firecanyon-obs": [["L57", "attack-info"]],
"ogre-obs": [
["L141", "rigid-body-platform-constants"],
@ -1240,9 +1228,7 @@
["L156", "attack-info"]
],
"snow-obs": [
["L201", "vector"]
],
"snow-obs": [["L201", "vector"]],
"lavatube-obs": [
["L162", "attack-info"],
@ -1448,13 +1434,9 @@
["L128", "attack-info"]
],
"billy": [
["L234", "vector"]
],
"billy": [["L234", "vector"]],
"lurkerworm": [
["L56", "attack-info"]
],
"lurkerworm": [["L56", "attack-info"]],
"pelican": [
["L199", "vector"],
@ -1473,13 +1455,9 @@
["L14", "attack-info"]
],
"steam-cap": [
["L51", "attack-info"]
],
"steam-cap": [["L51", "attack-info"]],
"sidekick": [
["L29", "vector"]
],
"sidekick": [["L29", "vector"]],
"target": [
["L720", "float", true],
@ -1603,9 +1581,7 @@
["L150", "attack-info"]
],
"racer": [
["L106", "vector"]
],
"racer": [["L106", "vector"]],
"target-racer-FIC-LAV-MIS-OGR-ROL": [
["L237", "attack-info"],
@ -1977,13 +1953,9 @@
["L303", "vector4w"]
],
"load-boundary-data": [
["L2", "(array array)"]
],
"load-boundary-data": [["L2", "(array array)"]],
"tfrag-near": [
["L1", "vu-function"]
],
"tfrag-near": [["L1", "vu-function"]],
"snow-ram-boss": [
["L220", "attack-info"],
@ -1993,13 +1965,9 @@
["L225", "attack-info"]
],
"snow-ram": [
["L73", "attack-info"]
],
"snow-ram": [["L73", "attack-info"]],
"snow-bumper": [
["L54", "attack-info"]
],
"snow-bumper": [["L54", "attack-info"]],
"snow-ball": [
["L86", "(pointer float)", 8],
@ -2007,22 +1975,16 @@
["L89", "attack-info"]
],
"puffer": [
["L170", "attack-info"]
],
"puffer": [["L170", "attack-info"]],
"driller-lurker": [
["L174", "attack-info"],
["L175", "attack-info"]
],
"dark-crystal": [
["L36", "attack-info"]
],
"dark-crystal": [["L36", "attack-info"]],
"kermit": [
["L164", "attack-info"]
],
"kermit": [["L164", "attack-info"]],
"swamp-blimp": [
["L177", "vector"],
@ -2035,22 +1997,16 @@
["L209", "tetherrock-info"]
],
"mistycannon": [
["L199", "attack-info"]
],
"mistycannon": [["L199", "attack-info"]],
"darkvine": [
["L41", "attack-info"]
],
"darkvine": [["L41", "attack-info"]],
"jungle-mirrors": [
["L325", "vector"],
["L326", "vector4w-4"]
],
"quicksandlurker": [
["L111", "attack-info"]
],
"quicksandlurker": [["L111", "attack-info"]],
"voicebox": [
["L47", "vector"],
@ -2080,21 +2036,13 @@
["L162", "vu-function"]
],
"tie": [
["L43", "vu-function"]
],
"tie": [["L43", "vu-function"]],
"tie-near": [
["L91", "vu-function"]
],
"sequence-a-village1": [
["L38", "vector"]
],
"tie-near": [["L91", "vu-function"]],
"depth-cue": [
["L17", "depth-cue-work"]
],
"sequence-a-village1": [["L38", "vector"]],
"depth-cue": [["L17", "depth-cue-work"]],
"navigate": [
["L402", "vector"],
@ -2117,9 +2065,7 @@
["L523", "float", true] // TODO - meters
],
"snow-bunny": [
["L190", "attack-info"]
],
"snow-bunny": [["L190", "attack-info"]],
// please do not add things after this entry! git is dumb.
"object-file-that-doesnt-actually-exist-and-i-just-put-this-here-to-prevent-merge-conflicts-with-this-file": []

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -521,7 +521,7 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
fmt::format(file_util::get_file_path(
{"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
data.name_in_dgo, tex.name, tex.w, tex.h),
out.data(), tex.w, tex.h);
out.data(), tex.w, tex.h, false);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name);
stats.successful_textures++;
@ -569,7 +569,7 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
fmt::format(file_util::get_file_path(
{"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
data.name_in_dgo, tex.name, tex.w, tex.h),
out.data(), tex.w, tex.h);
out.data(), tex.w, tex.h, false);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name);
stats.successful_textures++;
@ -599,7 +599,7 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
fmt::format(file_util::get_file_path(
{"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
data.name_in_dgo, tex.name, tex.w, tex.h),
out.data(), tex.w, tex.h);
out.data(), tex.w, tex.h, false);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name);
stats.successful_textures++;
@ -645,7 +645,7 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
fmt::format(file_util::get_file_path(
{"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
data.name_in_dgo, tex.name, tex.w, tex.h),
out.data(), tex.w, tex.h);
out.data(), tex.w, tex.h, false);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name);
stats.successful_textures++;
@ -691,7 +691,7 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
fmt::format(file_util::get_file_path(
{"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
data.name_in_dgo, tex.name, tex.w, tex.h),
out.data(), tex.w, tex.h);
out.data(), tex.w, tex.h, false);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name);
stats.successful_textures++;

View File

@ -10,6 +10,7 @@
#include "decompiler/level_extractor/extract_level.h"
#include "decompiler/data/TextureDB.h"
#include "common/util/os.h"
#include "common/util/diff.h"
int main(int argc, char** argv) {
fmt::print("[Mem] Size of linked word: {}\n", sizeof(decompiler::LinkedWord));
@ -25,8 +26,10 @@ int main(int argc, char** argv) {
file_util::init_crc();
init_opcode_info();
if (argc != 4) {
printf("Usage: decompiler <config_file> <in_folder> <out_folder>\n");
if (argc < 4) {
printf(
"Usage: decompiler <config_file> <in_folder> <out_folder> "
"[bool_flag_name=true/false...]\n");
return 1;
}
fmt::print("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024));
@ -34,7 +37,44 @@ int main(int argc, char** argv) {
// collect all files to process
Config config;
try {
config = read_config_file(argv[1]);
// Allow overriding config boolean flags via CLI
// There are very minimum guard-rails here
//
// "<key>=<override>"
//
// This allows us to run scripts that deviate from the defaults
std::map<std::string, bool> overrides;
if (argc > 4) {
for (int i = 4; i < argc; i++) {
std::string val = argv[i];
if (val.find('=') == std::string::npos) {
printf("Aborting - invalid flag override syntax\n");
printf(
"Usage: decompiler <config_file> <in_folder> <out_folder> "
"[bool_flag_name=true/false...]\n");
return 1;
}
auto pair = split_string(argv[i], '=');
if (pair.size() > 2) {
printf("Aborting - invalid flag override syntax, provide pairs!\n");
printf(
"Usage: decompiler <config_file> <in_folder> <out_folder> "
"[bool_flag_name=true/false...]\n");
return 1;
}
if (pair.at(1) != "true" && pair.at(1) != "false") {
printf("Aborting - invalid flag override syntax, true|false only!\n");
printf(
"Usage: decompiler <config_file> <in_folder> <out_folder> "
"[bool_flag_name=true/false...]\n");
return 1;
}
overrides.insert({pair.at(0), pair.at(0) == "true"});
}
}
config = read_config_file(argv[1], overrides);
} catch (const std::exception& e) {
lg::error("Failed to parse config: {}", e.what());
return 1;
@ -43,6 +83,15 @@ int main(int argc, char** argv) {
std::string in_folder = file_util::combine_path(argv[2], config.game_name);
std::string out_folder = file_util::combine_path(argv[3], config.game_name);
// Verify the in_folder is correct
// TODO - refactor to use ghc::filesystem, cleanup file_util
if (!file_util::file_exists(in_folder) ||
(!config.expected_elf_name.empty() &&
!file_util::file_exists(file_util::combine_path(in_folder, config.expected_elf_name)))) {
printf("Aborting - 'in_folder' does not exist or does not contain the expected files\n");
return 1;
}
std::vector<std::string> dgos, objs, strs;
for (const auto& dgo_name : config.dgo_names) {
dgos.push_back(file_util::combine_path(in_folder, dgo_name));

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 7.5 KiB

View File

@ -256,7 +256,8 @@ void OpenGLRenderer::render(DmaFollower dma, const RenderOptions& settings) {
if (settings.save_screenshot) {
finish_screenshot(settings.screenshot_path, settings.window_width_px, settings.window_height_px,
settings.lbox_width_px, settings.lbox_height_px);
settings.lbox_width_px, settings.lbox_height_px,
settings.screenshot_should_compress);
}
m_render_state.loader.update();
@ -430,7 +431,8 @@ void OpenGLRenderer::finish_screenshot(const std::string& output_name,
int width,
int height,
int x,
int y) {
int y,
bool compress) {
std::vector<u32> buffer(width * height);
glPixelStorei(GL_PACK_ALIGNMENT, 1);
glReadBuffer(GL_BACK);
@ -446,5 +448,5 @@ void OpenGLRenderer::finish_screenshot(const std::string& output_name,
for (auto& px : buffer) {
px |= 0xff000000;
}
file_util::write_rgba_png(output_name, buffer.data(), width, height);
file_util::write_rgba_png(output_name, buffer.data(), width, height, compress);
}

View File

@ -18,6 +18,7 @@ struct RenderOptions {
bool playing_from_dump = false;
bool save_screenshot = false;
bool screenshot_should_compress = false;
std::string screenshot_path;
};
@ -34,7 +35,12 @@ class OpenGLRenderer {
void init_bucket_renderers();
void draw_renderer_selection_window();
void finish_screenshot(const std::string& output_name, int px, int py, int x, int y);
void finish_screenshot(const std::string& output_name,
int px,
int py,
int x,
int y,
bool compress);
template <typename T, class... Args>
void init_bucket_renderer(const std::string& name, BucketId id, Args&&... args) {

View File

@ -96,6 +96,7 @@ void OpenGlDebugGui::draw(const DmaStats& dma_stats) {
}
if (ImGui::BeginMenu("Gfx Dump")) {
ImGui::Checkbox("Compress Screenshot", &m_compress_screenshot);
ImGui::MenuItem("Screenshot Next Frame!", nullptr, &m_want_screenshot);
ImGui::InputText("File", m_screenshot_save_name, 30);
ImGui::Separator();
@ -112,8 +113,7 @@ void OpenGlDebugGui::draw(const DmaStats& dma_stats) {
}
if (ImGui::BeginMenu("Frame Rate")) {
ImGui::MenuItem("Enable V-Sync", nullptr, &m_vsync);
ImGui::MenuItem("Disable V-Sync", nullptr, &m_nosync);
ImGui::Checkbox("Enable V-Sync", &m_vsync);
ImGui::Separator();
ImGui::Checkbox("Framelimiter", &framelimiter);
ImGui::InputFloat("Target FPS", &m_target_fps_text);
@ -131,4 +131,4 @@ void OpenGlDebugGui::draw(const DmaStats& dma_stats) {
if (m_draw_frame_time) {
m_frame_timer.draw_window(dma_stats);
}
}
}

View File

@ -61,21 +61,9 @@ class OpenGlDebugGui {
return false;
}
bool get_nosync_flag() {
if (m_nosync) {
m_nosync = false;
return true;
}
return false;
}
bool screenshot_compress_flag() { return m_compress_screenshot; }
bool get_vsync_flag() {
if (m_vsync) {
m_vsync = false;
return true;
}
return false;
}
bool get_vsync_flag() { return m_vsync; }
bool framelimiter = false;
float target_fps = 60.f;
@ -90,10 +78,10 @@ class OpenGlDebugGui {
bool m_want_save = false;
bool m_want_replay = false;
bool m_want_dump_load = false;
bool m_compress_screenshot = false;
bool m_want_screenshot = false;
char m_dump_save_name[256] = "dump.bin";
char m_screenshot_save_name[256] = "screenshot.png";
bool m_vsync = false;
bool m_nosync = false;
bool m_vsync = true;
float m_target_fps_text = 60.0;
};
};

View File

@ -35,6 +35,7 @@ struct GraphicsData {
// vsync
std::mutex sync_mutex;
std::condition_variable sync_cv;
bool vsync_enabled;
// dma chain transfer
std::mutex dma_mutex;
@ -262,6 +263,7 @@ void render_game_frame(int width, int height, int lbox_width, int lbox_height) {
options.draw_profiler_window = g_gfx_data->debug_gui.should_draw_profiler();
options.playing_from_dump = false;
options.save_screenshot = g_gfx_data->debug_gui.get_screenshot_flag();
options.screenshot_should_compress = g_gfx_data->debug_gui.screenshot_compress_flag();
if (options.save_screenshot) {
options.screenshot_path = make_output_file_name(g_gfx_data->debug_gui.screenshot_name());
}
@ -428,10 +430,10 @@ static void gl_render_display(GfxDisplay* display) {
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
// switch vsync modes, if requested
if (g_gfx_data->debug_gui.get_nosync_flag()) {
glfwSwapInterval(0);
} else if (g_gfx_data->debug_gui.get_vsync_flag()) {
glfwSwapInterval(1);
bool req_vsync = g_gfx_data->debug_gui.get_vsync_flag();
if (req_vsync != g_gfx_data->vsync_enabled) {
g_gfx_data->vsync_enabled = req_vsync;
glfwSwapInterval(req_vsync);
}
// actual vsync

View File

@ -249,7 +249,7 @@ std::vector<std::shared_ptr<TextureRecord>> TexturePool::convert_textures(const
fmt::format(
file_util::get_file_path({"debug_out", "textures", tpage_name, "{}-{}-{}.png"}),
tex_idx, tex_name, mip_idx),
texture_record->data.data(), ww, hh);
texture_record->data.data(), ww, hh, false);
}
result.push_back(std::move(texture_record));
}

View File

@ -0,0 +1,4 @@
version: '3'
vars:
RELEASE_PATH: './build'

View File

@ -0,0 +1,4 @@
version: '3'
vars:
RELEASE_PATH: './build'

View File

@ -0,0 +1,4 @@
version: '3'
vars:
RELEASE_PATH: './out/build/Release/bin'

View File

@ -180,7 +180,7 @@ Decompiler setup_decompiler(const std::vector<DecompilerFile>& files,
file_util::init_crc();
decompiler::init_opcode_info();
dc.config = std::make_unique<decompiler::Config>(decompiler::read_config_file(
file_util::get_file_path({"decompiler", "config", "jak1_ntsc_black_label.jsonc"})));
file_util::get_file_path({"decompiler", "config", "jak1_ntsc_black_label.jsonc"}), {}));
// modify the config
std::unordered_set<std::string> object_files;

3118
third-party/fpng/fpng.cpp generated vendored Normal file

File diff suppressed because it is too large Load Diff

109
third-party/fpng/fpng.h generated vendored Normal file
View File

@ -0,0 +1,109 @@
// fpng.h - unlicense (see end of fpng.cpp)
#pragma once
#include <stdlib.h>
#include <stdint.h>
#include <vector>
namespace fpng
{
// ---- Library initialization - call once to identify if the process supports SSE.
// Otherwise you'll only get scalar fallbacks.
void fpng_init();
// ---- Useful Utilities
// Returns true if the CPU supports SSE 4.1, and SSE support wasn't disabled by setting FPNG_NO_SSE=1.
// fpng_init() must have been called first, or it'll assert and return false.
bool fpng_cpu_supports_sse41();
// Fast CRC-32 SSE4.1+pclmul or a scalar fallback (slice by 4)
const uint32_t FPNG_CRC32_INIT = 0;
uint32_t fpng_crc32(const void* pData, size_t size, uint32_t prev_crc32 = FPNG_CRC32_INIT);
// Fast Adler32 SSE4.1 Adler-32 with a scalar fallback.
const uint32_t FPNG_ADLER32_INIT = 1;
uint32_t fpng_adler32(const uint8_t* ptr, size_t buf_len, uint32_t adler = FPNG_ADLER32_INIT);
// ---- Compression
enum
{
// Enables computing custom Huffman tables for each file, instead of using the custom global tables.
// Results in roughly 6% smaller files on average, but compression is around 40% slower.
FPNG_ENCODE_SLOWER = 1,
// Only use raw Deflate blocks (no compression at all). Intended for testing.
FPNG_FORCE_UNCOMPRESSED = 2,
};
// Fast PNG encoding. The resulting file can be decoded either using a standard PNG decoder or by the fpng_decode_memory() function below.
// pImage: pointer to RGB or RGBA image pixels, R first in memory, B/A last.
// w/h - image dimensions. Image's row pitch in bytes must is w*num_chans.
// num_chans must be 3 or 4.
bool fpng_encode_image_to_memory(const void* pImage, uint32_t w, uint32_t h, uint32_t num_chans, std::vector<uint8_t>& out_buf, uint32_t flags = 0);
#ifndef FPNG_NO_STDIO
// Fast PNG encoding to the specified file.
bool fpng_encode_image_to_file(const char* pFilename, const void* pImage, uint32_t w, uint32_t h, uint32_t num_chans, uint32_t flags = 0);
#endif
// ---- Decompression
enum
{
FPNG_DECODE_SUCCESS = 0, // file is a valid PNG file and written by FPNG and the decode succeeded
FPNG_DECODE_NOT_FPNG, // file is a valid PNG file, but it wasn't written by FPNG so you should try decoding it with a general purpose PNG decoder
FPNG_DECODE_INVALID_ARG, // invalid function parameter
FPNG_DECODE_FAILED_NOT_PNG, // file cannot be a PNG file
FPNG_DECODE_FAILED_HEADER_CRC32, // a chunk CRC32 check failed, file is likely corrupted or not PNG
FPNG_DECODE_FAILED_INVALID_DIMENSIONS, // invalid image dimensions in IHDR chunk (0 or too large)
FPNG_DECODE_FAILED_DIMENSIONS_TOO_LARGE, // decoding the file fully into memory would likely require too much memory (only on 32bpp builds)
FPNG_DECODE_FAILED_CHUNK_PARSING, // failed while parsing the chunk headers, or file is corrupted
FPNG_DECODE_FAILED_INVALID_IDAT, // IDAT data length is too small and cannot be valid, file is either corrupted or it's a bug
// fpng_decode_file() specific errors
FPNG_DECODE_FILE_OPEN_FAILED,
FPNG_DECODE_FILE_TOO_LARGE,
FPNG_DECODE_FILE_READ_FAILED,
FPNG_DECODE_FILE_SEEK_FAILED
};
// Fast PNG decoding of files ONLY created by fpng_encode_image_to_memory() or fpng_encode_image_to_file().
// If fpng_get_info() or fpng_decode_memory() returns FPNG_DECODE_NOT_FPNG, you should decode the PNG by falling back to a general purpose decoder.
//
// fpng_get_info() parses the PNG header and iterates through all chunks to determine if it's a file written by FPNG, but does not decompress the actual image data so it's relatively fast.
//
// pImage, image_size: Pointer to PNG image data and its size
// width, height: output image's dimensions
// channels_in_file: will be 3 or 4
//
// Returns FPNG_DECODE_SUCCESS on success, otherwise one of the failure codes above.
// If FPNG_DECODE_NOT_FPNG is returned, you must decompress the file with a general purpose PNG decoder.
// If another error occurs, the file is likely corrupted or invalid, but you can still try to decompress the file with another decoder (which will likely fail).
int fpng_get_info(const void* pImage, uint32_t image_size, uint32_t& width, uint32_t& height, uint32_t& channels_in_file);
// fpng_decode_memory() decompresses 24/32bpp PNG files ONLY encoded by this module.
// If the image was written by FPNG, it will decompress the image data, otherwise it will return FPNG_DECODE_NOT_FPNG in which case you should fall back to a general purpose PNG decoder (lodepng, stb_image, libpng, etc.)
//
// pImage, image_size: Pointer to PNG image data and its size
// out: Output 24/32bpp image buffer
// width, height: output image's dimensions
// channels_in_file: will be 3 or 4
// desired_channels: must be 3 or 4
//
// If the image is 24bpp and 32bpp is requested, the alpha values will be set to 0xFF.
// If the image is 32bpp and 24bpp is requested, the alpha values will be discarded.
//
// Returns FPNG_DECODE_SUCCESS on success, otherwise one of the failure codes above.
// If FPNG_DECODE_NOT_FPNG is returned, you must decompress the file with a general purpose PNG decoder.
// If another error occurs, the file is likely corrupted or invalid, but you can still try to decompress the file with another decoder (which will likely fail).
int fpng_decode_memory(const void* pImage, uint32_t image_size, std::vector<uint8_t>& out, uint32_t& width, uint32_t& height, uint32_t& channels_in_file, uint32_t desired_channels);
#ifndef FPNG_NO_STDIO
int fpng_decode_file(const char* pFilename, std::vector<uint8_t>& out, uint32_t& width, uint32_t& height, uint32_t& channels_in_file, uint32_t desired_channels);
#endif
} // namespace fpng

103
third-party/svpng.h generated vendored
View File

@ -1,103 +0,0 @@
#pragma once
#include <cstdio>
/*
Copyright (C) 2017 Milo Yip. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of pngout nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
inline void svpng(FILE* fp, unsigned w, unsigned h, const unsigned char* img, int alpha) {
static const unsigned t[] = {0, 0x1db71064, 0x3b6e20c8, 0x26d930ac,
0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c,
0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c};
unsigned a = 1, b = 0, c, p = w * (alpha ? 4 : 3) + 1, x, y, i; /* ADLER-a, ADLER-b, CRC, pitch */
#define SVPNG_PUT(u) fputc(u, fp)
#define SVPNG_U8A(ua, l) \
for (i = 0; i < l; i++) \
SVPNG_PUT((ua)[i]);
#define SVPNG_U32(u) \
do { \
SVPNG_PUT((u) >> 24); \
SVPNG_PUT(((u) >> 16) & 255); \
SVPNG_PUT(((u) >> 8) & 255); \
SVPNG_PUT((u)&255); \
} while (0)
#define SVPNG_U8C(u) \
do { \
SVPNG_PUT(u); \
c ^= (u); \
c = (c >> 4) ^ t[c & 15]; \
c = (c >> 4) ^ t[c & 15]; \
} while (0)
#define SVPNG_U8AC(ua, l) \
for (i = 0; i < l; i++) \
SVPNG_U8C((ua)[i])
#define SVPNG_U16LC(u) \
do { \
SVPNG_U8C((u)&255); \
SVPNG_U8C(((u) >> 8) & 255); \
} while (0)
#define SVPNG_U32C(u) \
do { \
SVPNG_U8C((u) >> 24); \
SVPNG_U8C(((u) >> 16) & 255); \
SVPNG_U8C(((u) >> 8) & 255); \
SVPNG_U8C((u)&255); \
} while (0)
#define SVPNG_U8ADLER(u) \
do { \
SVPNG_U8C(u); \
a = (a + (u)) % 65521; \
b = (b + a) % 65521; \
} while (0)
#define SVPNG_BEGIN(s, l) \
do { \
SVPNG_U32(l); \
c = ~0U; \
SVPNG_U8AC(s, 4); \
} while (0)
#define SVPNG_END() SVPNG_U32(~c)
SVPNG_U8A("\x89PNG\r\n\32\n", 8); /* Magic */
SVPNG_BEGIN("IHDR", 13); /* IHDR chunk { */
SVPNG_U32C(w);
SVPNG_U32C(h); /* Width & Height (8 bytes) */
SVPNG_U8C(8);
SVPNG_U8C(alpha ? 6 : 2); /* Depth=8, Color=True color with/without alpha (2 bytes) */
SVPNG_U8AC("\0\0\0", 3); /* Compression=Deflate, Filter=No, Interlace=No (3 bytes) */
SVPNG_END(); /* } */
SVPNG_BEGIN("IDAT", 2 + h * (5 + p) + 4); /* IDAT chunk { */
SVPNG_U8AC("\x78\1", 2); /* Deflate block begin (2 bytes) */
for (y = 0; y < h; y++) { /* Each horizontal line makes a block for simplicity */
SVPNG_U8C(y == h - 1); /* 1 for the last block, 0 for others (1 byte) */
SVPNG_U16LC(p);
SVPNG_U16LC(~p); /* Size of block in little endian and its 1's complement (4 bytes) */
SVPNG_U8ADLER(0); /* No filter prefix (1 byte) */
for (x = 0; x < p - 1; x++, img++)
SVPNG_U8ADLER(*img); /* Image pixel data */
}
SVPNG_U32C((b << 16) | a); /* Deflate block end with adler (4 bytes) */
SVPNG_END(); /* } */
SVPNG_BEGIN("IEND", 0);
SVPNG_END(); /* IEND chunk {} */
}

View File

@ -1,3 +0,0 @@
third-party/glfw: df8d7bc892937a8b0f7c604c92a9f64f383cf48c
third-party/googletest: 14aa11db02d9851d957f93ef9fddb110c1aafdc6
third-party/zydis: 562a7c1e5f2c5017ea882ad0aa3360c465a3eb8f

10
vendor.yaml Normal file
View File

@ -0,0 +1,10 @@
third-party/glfw:
sha: df8d7bc892937a8b0f7c604c92a9f64f383cf48c
third-party/googletest:
sha: 14aa11db02d9851d957f93ef9fddb110c1aafdc6
third-party/zydis:
sha: 562a7c1e5f2c5017ea882ad0aa3360c465a3eb8f
third-party/discord-rpc:
sha: 963aa9f3e5ce81a4682c6ca3d136cddda614db33
third-party/fng:
sha: bfe5f9c69e93b99b31268c10db8e645c9125a07f