compiler speed improvement (#1547)

This commit is contained in:
water111 2022-06-24 18:21:24 -04:00 committed by GitHub
parent 80e84f2225
commit 628ce47b2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 254 additions and 268 deletions

View File

@ -19,7 +19,7 @@ template <typename T>
void for_each_in_list(const goos::Object& list, T f) {
const goos::Object* iter = &list;
while (iter->is_pair()) {
auto lap = iter->as_pair();
const auto& lap = iter->as_pair();
f(lap->car);
iter = &lap->cdr;
}

View File

@ -9,10 +9,8 @@
namespace versions {
// language version (OpenGOAL)
constexpr s32 GOAL_VERSION_MAJOR = 0;
constexpr s32 GOAL_VERSION_MINOR = 9;
constexpr int DECOMPILER_VERSION = 4;
constexpr s32 GOAL_VERSION_MAJOR = 1;
constexpr s32 GOAL_VERSION_MINOR = 0;
namespace jak1 {
// these versions are from the game

View File

@ -29,7 +29,6 @@ int main(int argc, char** argv) {
lg::set_stdout_level(lg::level::info);
lg::set_flush_level(lg::level::info);
lg::initialize();
lg::info("GOAL Decompiler version {}\n", versions::DECOMPILER_VERSION);
init_opcode_info();

View File

@ -220,4 +220,7 @@
The compiler is now much more aggressive in where and how it expands macros and handles expressions at compiler time.
- Several places where macros could be incorrectly executed more than once (possibly causing unwanted side effects) have been fixed.
- Fixed bug in size calculation of non-inline stack arrays. Previous behavior was a compiler assert.
- Correctly handle `mod` for unsigned numbers. Previous behavior was to treat all inputs as 32-bit signed integers.
- Correctly handle `mod` for unsigned numbers. Previous behavior was to treat all inputs as 32-bit signed integers.
## V1.0 Revised constant propagation, speed improvements
Improved error messages around macros

View File

@ -8,6 +8,7 @@
#include "common/goos/PrettyPrinter.h"
#include "common/link_types.h"
#include "common/util/FileUtil.h"
#include "goalc/make/Tools.h"
#include "goalc/regalloc/Allocator.h"
@ -78,37 +79,6 @@ Compiler::~Compiler() {
}
}
void Compiler::save_repl_history() {
m_repl->save_history();
}
void Compiler::print_to_repl(const std::string_view& str) {
m_repl->print_to_repl(str);
}
std::string Compiler::get_prompt() {
std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > ");
if (m_listener.is_connected()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> ");
}
if (m_debugger.is_halted()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> ");
} else if (m_debugger.is_attached()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> ");
}
return "\033[0m" + prompt;
}
std::string Compiler::get_repl_input() {
auto str = m_repl->readline(get_prompt());
if (str) {
m_repl->add_to_history(str);
return str;
} else {
return "";
}
}
ReplStatus Compiler::handle_repl_string(const std::string& input) {
if (input.empty()) {
return ReplStatus::OK;
@ -329,74 +299,6 @@ bool Compiler::codegen_and_disassemble_object_file(FileEnv* env,
return ok;
}
void Compiler::compile_and_send_from_string(const std::string& source_code) {
if (!connect_to_target()) {
throw std::runtime_error(
"Compiler failed to connect to target for compile_and_send_from_string.");
}
auto code = m_goos.reader.read_from_string(source_code);
auto compiled = compile_object_file("test-code", code, true);
ASSERT(!compiled->is_empty());
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
}
std::vector<std::string> Compiler::run_test_from_file(const std::string& source_code) {
try {
if (!connect_to_target()) {
throw std::runtime_error("Compiler::run_test_from_file couldn't connect!");
}
auto code = m_goos.reader.read_from_file({source_code});
auto compiled = compile_object_file("test-code", code, true);
if (compiled->is_empty()) {
return {};
}
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.record_messages(ListenerMessageKind::MSG_PRINT);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
return m_listener.stop_recording_messages();
} catch (std::exception& e) {
fmt::print("[Compiler] Failed to compile test program {}: {}\n", source_code, e.what());
throw e;
}
}
std::vector<std::string> Compiler::run_test_from_string(const std::string& src,
const std::string& obj_name) {
try {
if (!connect_to_target()) {
throw std::runtime_error("Compiler::run_test_from_file couldn't connect!");
}
auto code = m_goos.reader.read_from_string({src});
auto compiled = compile_object_file(obj_name, code, true);
if (compiled->is_empty()) {
return {};
}
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.record_messages(ListenerMessageKind::MSG_PRINT);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
return m_listener.stop_recording_messages();
} catch (std::exception& e) {
fmt::print("[Compiler] Failed to compile test program from string {}: {}\n", src, e.what());
throw e;
}
}
bool Compiler::connect_to_target() {
if (!m_listener.is_connected()) {
for (int i = 0; i < 1000; i++) {
@ -413,52 +315,6 @@ bool Compiler::connect_to_target() {
return true;
}
/*!
* Just run the front end on a string. Will not do register allocation or code generation.
* Useful for typechecking, defining types, or running strings that invoke the compiler again.
*/
void Compiler::run_front_end_on_string(const std::string& src) {
auto code = m_goos.reader.read_from_string({src});
compile_object_file("run-on-string", code, true);
}
/*!
* Just run the front end on a file. Will not do register allocation or code generation.
* Useful for typechecking, defining types, or running strings that invoke the compiler again.
*/
void Compiler::run_front_end_on_file(const std::vector<std::string>& path) {
auto code = m_goos.reader.read_from_file(path);
compile_object_file("run-on-file", code, true);
}
/*!
* Run the entire compilation process on the input source code. Will generate an object file, but
* won't save it anywhere.
*/
void Compiler::run_full_compiler_on_string_no_save(const std::string& src,
const std::optional<std::string>& string_name) {
auto code = m_goos.reader.read_from_string(src, true, string_name);
auto compiled = compile_object_file("run-on-string", code, true);
color_object_file(compiled);
codegen_object_file(compiled);
}
std::vector<std::string> Compiler::run_test_no_load(const std::string& source_code) {
auto code = m_goos.reader.read_from_file({source_code});
compile_object_file("test-code", code, true);
return {};
}
void Compiler::shutdown_target() {
if (m_debugger.is_attached()) {
m_debugger.detach();
}
if (m_listener.is_connected()) {
m_listener.send_reset(true);
}
}
void Compiler::typecheck(const goos::Object& form,
const TypeSpec& expected,
const TypeSpec& actual,
@ -487,10 +343,6 @@ void Compiler::typecheck_reg_type_allow_false(const goos::Object& form,
typecheck(form, expected, coerce_to_reg_type(actual->type()), error_message);
}
bool Compiler::knows_object_file(const std::string& name) {
return m_debugger.knows_object(name);
}
void Compiler::setup_goos_forms() {
m_goos.register_form("get-enum-vals", [&](const goos::Object& form, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
@ -522,3 +374,68 @@ void Compiler::setup_goos_forms() {
return goos::build_list(enum_vals);
});
}
void Compiler::asm_file(const CompilationOptions& options) {
auto code = m_goos.reader.read_from_file({options.filename});
std::string obj_file_name = options.filename;
// Extract object name from file name.
for (int idx = int(options.filename.size()) - 1; idx-- > 0;) {
if (options.filename.at(idx) == '\\' || options.filename.at(idx) == '/') {
obj_file_name = options.filename.substr(idx + 1);
break;
}
}
obj_file_name = obj_file_name.substr(0, obj_file_name.find_last_of('.'));
// COMPILE
auto obj_file = compile_object_file(obj_file_name, code, !options.no_code);
if (options.color) {
// register allocation
color_object_file(obj_file);
// code/object file generation
std::vector<u8> data;
std::string disasm;
if (options.disassemble) {
codegen_and_disassemble_object_file(obj_file, &data, &disasm);
if (options.disassembly_output_file.empty()) {
printf("%s\n", disasm.c_str());
} else {
file_util::write_text_file(options.disassembly_output_file, disasm);
}
} else {
data = codegen_object_file(obj_file);
}
// send to target
if (options.load) {
if (m_listener.is_connected()) {
m_listener.send_code(data, obj_file_name);
} else {
printf("WARNING - couldn't load because listener isn't connected\n"); // todo log warn
}
}
// save file
if (options.write) {
auto path = file_util::get_file_path({"out", "obj", obj_file_name + ".o"});
file_util::create_dir_if_needed_for_file(path);
file_util::write_binary_file(path, (void*)data.data(), data.size());
}
} else {
if (options.load) {
printf("WARNING - couldn't load because coloring is not enabled\n");
}
if (options.write) {
printf("WARNING - couldn't write because coloring is not enabled\n");
}
if (options.disassemble) {
printf("WARNING - couldn't disassemble because coloring is not enabled\n");
}
}
}

View File

@ -26,10 +26,23 @@ enum MathMode { MATH_INT, MATH_BINT, MATH_FLOAT, MATH_INVALID };
enum class ReplStatus { OK, WANT_EXIT, WANT_RELOAD };
struct CompilationOptions {
std::string filename; // input file
std::string disassembly_output_file; // file to write, containing x86 assembly output
bool load = false; // send to target
bool color = false; // do register allocation/code generation passes
bool write = false; // write object file to out/obj
bool no_code = false; // file shouldn't generate code, throw error if it does
bool disassemble = false; // either print disassembly to stdout or output_file
bool print_time = false; // print timing statistics
};
class Compiler {
public:
Compiler(const std::string& user_profile = "#f", std::unique_ptr<ReplWrapper> repl = nullptr);
~Compiler();
void asm_file(const CompilationOptions& options);
void save_repl_history();
void print_to_repl(const std::string_view& str);
std::string get_prompt();

View File

@ -4,6 +4,155 @@
#include "goalc/compiler/Compiler.h"
#include "goalc/compiler/IR.h"
void Compiler::save_repl_history() {
m_repl->save_history();
}
void Compiler::print_to_repl(const std::string_view& str) {
m_repl->print_to_repl(str);
}
std::string Compiler::get_prompt() {
std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > ");
if (m_listener.is_connected()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> ");
}
if (m_debugger.is_halted()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> ");
} else if (m_debugger.is_attached()) {
prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> ");
}
return "\033[0m" + prompt;
}
std::string Compiler::get_repl_input() {
auto str = m_repl->readline(get_prompt());
if (str) {
m_repl->add_to_history(str);
return str;
} else {
return "";
}
}
void Compiler::compile_and_send_from_string(const std::string& source_code) {
if (!connect_to_target()) {
throw std::runtime_error(
"Compiler failed to connect to target for compile_and_send_from_string.");
}
auto code = m_goos.reader.read_from_string(source_code);
auto compiled = compile_object_file("test-code", code, true);
ASSERT(!compiled->is_empty());
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
}
std::vector<std::string> Compiler::run_test_from_file(const std::string& source_code) {
try {
if (!connect_to_target()) {
throw std::runtime_error("Compiler::run_test_from_file couldn't connect!");
}
auto code = m_goos.reader.read_from_file({source_code});
auto compiled = compile_object_file("test-code", code, true);
if (compiled->is_empty()) {
return {};
}
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.record_messages(ListenerMessageKind::MSG_PRINT);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
return m_listener.stop_recording_messages();
} catch (std::exception& e) {
fmt::print("[Compiler] Failed to compile test program {}: {}\n", source_code, e.what());
throw e;
}
}
std::vector<std::string> Compiler::run_test_from_string(const std::string& src,
const std::string& obj_name) {
try {
if (!connect_to_target()) {
throw std::runtime_error("Compiler::run_test_from_file couldn't connect!");
}
auto code = m_goos.reader.read_from_string({src});
auto compiled = compile_object_file(obj_name, code, true);
if (compiled->is_empty()) {
return {};
}
color_object_file(compiled);
auto data = codegen_object_file(compiled);
m_listener.record_messages(ListenerMessageKind::MSG_PRINT);
m_listener.send_code(data);
if (!m_listener.most_recent_send_was_acked()) {
print_compiler_warning("Runtime is not responding after sending test code. Did it crash?\n");
}
return m_listener.stop_recording_messages();
} catch (std::exception& e) {
fmt::print("[Compiler] Failed to compile test program from string {}: {}\n", src, e.what());
throw e;
}
}
/*!
* Just run the front end on a string. Will not do register allocation or code generation.
* Useful for typechecking, defining types, or running strings that invoke the compiler again.
*/
void Compiler::run_front_end_on_string(const std::string& src) {
auto code = m_goos.reader.read_from_string({src});
compile_object_file("run-on-string", code, true);
}
/*!
* Just run the front end on a file. Will not do register allocation or code generation.
* Useful for typechecking, defining types, or running strings that invoke the compiler again.
*/
void Compiler::run_front_end_on_file(const std::vector<std::string>& path) {
auto code = m_goos.reader.read_from_file(path);
compile_object_file("run-on-file", code, true);
}
/*!
* Run the entire compilation process on the input source code. Will generate an object file, but
* won't save it anywhere.
*/
void Compiler::run_full_compiler_on_string_no_save(const std::string& src,
const std::optional<std::string>& string_name) {
auto code = m_goos.reader.read_from_string(src, true, string_name);
auto compiled = compile_object_file("run-on-string", code, true);
color_object_file(compiled);
codegen_object_file(compiled);
}
std::vector<std::string> Compiler::run_test_no_load(const std::string& source_code) {
auto code = m_goos.reader.read_from_file({source_code});
compile_object_file("test-code", code, true);
return {};
}
void Compiler::shutdown_target() {
if (m_debugger.is_attached()) {
m_debugger.detach();
}
if (m_listener.is_connected()) {
m_listener.send_reset(true);
}
}
bool Compiler::knows_object_file(const std::string& name) {
return m_debugger.knows_object(name);
}
/*!
* Parse arguments into a goos::Arguments format.
*/

View File

@ -286,8 +286,8 @@ Val* Compiler::compile_no_const_prop(const goos::Object& code, Env* env) {
* Highest level compile function
*/
Val* Compiler::compile(const goos::Object& code, Env* env) {
auto propagated = try_constant_propagation(code, env);
return compile_no_const_prop(propagated.value, env);
// auto propagated = try_constant_propagation(code, env);
return compile_no_const_prop(code, env);
}
/*!

View File

@ -123,49 +123,37 @@ Val* Compiler::compile_asm_text_file(const goos::Object& form, const goos::Objec
Val* Compiler::compile_asm_file(const goos::Object& form, const goos::Object& rest, Env* env) {
(void)env;
int i = 0;
std::string filename;
std::string disasm_filename = "";
bool load = false;
bool color = false;
bool write = false;
bool no_code = false;
bool disassemble = false;
bool no_time_prints = false;
CompilationOptions options;
bool no_throw = false;
std::vector<std::pair<std::string, double>> timing;
Timer total_timer;
// parse arguments
bool last_was_disasm = false;
for_each_in_list(rest, [&](const goos::Object& o) {
if (last_was_disasm) {
last_was_disasm = false;
if (o.type == goos::ObjectType::STRING) {
disasm_filename = as_string(o);
options.disassembly_output_file = as_string(o);
i++;
return;
}
}
if (i == 0) {
filename = as_string(o);
options.filename = as_string(o);
} else {
auto setting = symbol_string(o);
if (setting == ":load") {
load = true;
options.load = true;
} else if (setting == ":color") {
color = true;
options.color = true;
} else if (setting == ":write") {
write = true;
options.write = true;
} else if (setting == ":no-code") {
no_code = true;
options.no_code = true;
} else if (setting == ":no-throw") {
no_throw = true;
} else if (setting == ":disassemble") {
disassemble = true;
options.disassemble = true;
last_was_disasm = true;
} else if (setting == ":no-time-prints") {
no_time_prints = true;
} else {
throw_compiler_error(form, "The option {} was not recognized for asm-file.", setting);
}
@ -173,92 +161,8 @@ Val* Compiler::compile_asm_file(const goos::Object& form, const goos::Object& re
i++;
});
// READ
Timer reader_timer;
try {
auto code = m_goos.reader.read_from_file({filename});
timing.emplace_back("read", reader_timer.getMs());
Timer compile_timer;
std::string obj_file_name = filename;
// Extract object name from file name.
for (int idx = int(filename.size()) - 1; idx-- > 0;) {
if (filename.at(idx) == '\\' || filename.at(idx) == '/') {
obj_file_name = filename.substr(idx + 1);
break;
}
}
obj_file_name = obj_file_name.substr(0, obj_file_name.find_last_of('.'));
// COMPILE
auto obj_file = compile_object_file(obj_file_name, code, !no_code);
timing.emplace_back("compile", compile_timer.getMs());
if (color) {
// register allocation
Timer color_timer;
color_object_file(obj_file);
timing.emplace_back("color", color_timer.getMs());
// code/object file generation
Timer codegen_timer;
std::vector<u8> data;
std::string disasm;
if (disassemble) {
codegen_and_disassemble_object_file(obj_file, &data, &disasm);
if (disasm_filename == "") {
printf("%s\n", disasm.c_str());
} else {
file_util::write_text_file(disasm_filename, disasm);
}
} else {
data = codegen_object_file(obj_file);
}
timing.emplace_back("codegen", codegen_timer.getMs());
// send to target
if (load) {
if (m_listener.is_connected()) {
m_listener.send_code(data, obj_file_name);
} else {
printf("WARNING - couldn't load because listener isn't connected\n"); // todo log warn
}
}
// save file
if (write) {
auto path = file_util::get_file_path({"out", "obj", obj_file_name + ".o"});
file_util::create_dir_if_needed_for_file(path);
file_util::write_binary_file(path, (void*)data.data(), data.size());
}
} else {
if (load) {
printf("WARNING - couldn't load because coloring is not enabled\n");
}
if (write) {
printf("WARNING - couldn't write because coloring is not enabled\n");
}
if (disassemble) {
printf("WARNING - couldn't disassemble because coloring is not enabled\n");
}
}
if (m_settings.print_timing) {
printf("F: %36s ", obj_file_name.c_str());
timing.emplace_back("total", total_timer.getMs());
for (auto& e : timing) {
printf(" %12s %4.0f", e.first.c_str(), e.second);
}
printf("\n");
} else {
auto total_time = total_timer.getMs();
if (total_time > 10.0 && color && !no_time_prints) {
fmt::print("[ASM-FILE] {} took {:.2f} ms\n", obj_file_name, total_time);
}
}
asm_file(options);
} catch (std::runtime_error& e) {
if (!no_throw) {
throw_compiler_error(form, "Error while compiling file: {}", e.what());

View File

@ -267,7 +267,7 @@ bool Compiler::expand_macro_once(const goos::Object& src, goos::Object* out, Env
auto goos_result = m_goos.eval_list_return_last(macro->body, macro->body, mac_env);
// make the macro expanded form point to the source where the macro was used for error messages.
m_goos.reader.db.inherit_info(src, goos_result);
// m_goos.reader.db.inherit_info(src, goos_result);
*out = goos_result;
return true;

View File

@ -30,8 +30,11 @@ bool CompilerTool::needs_run(const ToolInput& task) {
bool CompilerTool::run(const ToolInput& task) {
// todo check inputs
try {
m_compiler->run_front_end_on_string(
fmt::format("(asm-file \"{}\" :no-time-prints :color :write)", task.input.at(0)));
CompilationOptions options;
options.filename = task.input.at(0);
options.color = true;
options.write = true;
m_compiler->asm_file(options);
} catch (std::exception& e) {
fmt::print("Compilation failed: {}\n", e.what());
return false;

View File

@ -222,7 +222,7 @@ class VarAssignment {
}
}
const std::vector<bool> live_vector() const { return m_live; }
const std::vector<bool>& live_vector() const { return m_live; }
private:
// common info

View File

@ -20,7 +20,7 @@
class IRegSet {
public:
IRegSet() { m_data.reserve(4); }
IRegSet() { resize(64 * 4); }
/*!
* Add the given ireg to the set.