From d1ad6c38179c0e33654d5403b349fa91350f5539 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sat, 23 Jul 2022 10:30:23 -0400 Subject: [PATCH] utf8: fix locale mis-handling on linux (#1698) * utf8: fix locale mis-handling on linux * lint: formatting --- .github/workflows/linting.yaml | 2 +- Taskfile.yml | 2 +- common/util/unicode_util.h | 26 ++++++++++++++++++++++++++ decompiler/extractor/main.cpp | 8 ++------ decompiler/main.cpp | 5 +---- game/main.cpp | 5 +---- lsp/handlers/lsp_router.cpp | 3 +-- lsp/main.cpp | 6 ++---- lsp/protocol/hover.h | 2 +- lsp/state/workspace.cpp | 1 - test/offline/offline_test_main.cpp | 6 +----- test/test_main.cpp | 6 +----- tools/dgo_packer.cpp | 13 ++++++------- tools/dgo_unpacker.cpp | 10 ++++------ tools/level_tools/level_dump/main.cpp | 17 +++++++---------- tools/memory_dump_tool/main.cpp | 5 +---- 16 files changed, 56 insertions(+), 61 deletions(-) diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml index d12c5708b..ccb2d38bd 100644 --- a/.github/workflows/linting.yaml +++ b/.github/workflows/linting.yaml @@ -25,4 +25,4 @@ jobs: - name: Check Clang-Formatting run: | chmod +x ./third-party/run-clang-format/run-clang-format.py - ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test --color always + ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp --color always diff --git a/Taskfile.yml b/Taskfile.yml index b42d88e95..00d58d228 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -61,7 +61,7 @@ tasks: format: desc: "Format code" cmds: - - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test -i + - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i # npm install -g prettier - cmd: npx prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc ignore_error: true diff --git a/common/util/unicode_util.h b/common/util/unicode_util.h index 165ce4279..92f34f420 100644 --- a/common/util/unicode_util.h +++ b/common/util/unicode_util.h @@ -1,8 +1,11 @@ #pragma once #include +#include #include +#include "common/util/FileUtil.h" + #ifdef _WIN32 std::wstring utf8_string_to_wide_string(const std::string_view& str); bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str); @@ -11,3 +14,26 @@ bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str) #endif std::string get_env(const std::string& name); + +/// @brief Windows's command line args are not UTF-8 so they need to be converted +/// ghc::filesystem comes with an u8guard class to help with this, but on linux it calls +/// setlocale +/// +/// This is problematic for several annoying reasons, so we are going off the (hopefully sane) +/// assumption that linux is using utf-8 by default and that the args are already proper +/// +/// This may need to be updated on linux if that assumption proves wrong. +class ArgumentGuard { + public: +#ifdef _WIN32 + ArgumentGuard(int& argc, char**& argv) : u8_guard(argc, argv) { + if (!u8_guard.valid()) { + exit(EXIT_FAILURE); + } + } + fs::u8arguments u8_guard; +#else + // if linux + ArgumentGuard(int&, char**&) {} +#endif +}; diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index a881521a2..3f7e1b34b 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -223,6 +223,8 @@ void launch_game() { } int main(int argc, char** argv) { + ArgumentGuard u8_guard(argc, argv); + fs::path input_file_path; fs::path project_path_override; bool flag_runall = false; @@ -234,12 +236,6 @@ int main(int argc, char** argv) { bool flag_folder = false; std::string game_name = "jak1"; - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - lg::error("Bad encoding, needs UTF-8"); - exit(EXIT_FAILURE); - } - lg::initialize(); CLI::App app{"OpenGOAL Level Extraction Tool"}; diff --git a/decompiler/main.cpp b/decompiler/main.cpp index cda17f71d..eb4a80dc7 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -18,10 +18,7 @@ #include "decompiler/level_extractor/extract_level.h" int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); Timer decomp_timer; diff --git a/game/main.cpp b/game/main.cpp index 7fcc081b9..40df7eb7c 100644 --- a/game/main.cpp +++ b/game/main.cpp @@ -42,10 +42,7 @@ void setup_logging(bool verbose) { * Entry point for the game. */ int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); // TODO - replace with CLI11 and just propagate args through // - https://github.com/CLIUtils/CLI11/issues/744 diff --git a/lsp/handlers/lsp_router.cpp b/lsp/handlers/lsp_router.cpp index ed645790e..42fc7749c 100644 --- a/lsp/handlers/lsp_router.cpp +++ b/lsp/handlers/lsp_router.cpp @@ -1,9 +1,8 @@ #include "lsp_router.h" -#include "lsp/handlers/initialize.h" - #include "common/log/log.h" +#include "lsp/handlers/initialize.h" #include "lsp/protocol/error_codes.h" #include "text_document/document_symbol.h" #include "text_document/document_synchronization.h" diff --git a/lsp/main.cpp b/lsp/main.cpp index 13ad889e0..7e4cce0e7 100644 --- a/lsp/main.cpp +++ b/lsp/main.cpp @@ -13,6 +13,7 @@ #include #include "common/log/log.h" +#include "common/util/unicode_util.h" #include "lsp/handlers/lsp_router.h" #include "lsp/state/workspace.h" @@ -48,10 +49,7 @@ void setup_logging(bool verbose, std::string log_file) { } int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); CLI::App app{"OpenGOAL Language Server"}; diff --git a/lsp/protocol/hover.h b/lsp/protocol/hover.h index c10ac45d6..ad35b5e33 100644 --- a/lsp/protocol/hover.h +++ b/lsp/protocol/hover.h @@ -30,7 +30,7 @@ namespace LSPSpec { * decide to remove HTML from the markdown to avoid script execution. */ struct MarkupContent { - std::string m_kind; // Actually a MarkupKind which is either 'plaintext' or 'markdown' + std::string m_kind; // Actually a MarkupKind which is either 'plaintext' or 'markdown' std::string m_value; }; diff --git a/lsp/state/workspace.cpp b/lsp/state/workspace.cpp index 2f3cb3f9b..26be5d9ea 100644 --- a/lsp/state/workspace.cpp +++ b/lsp/state/workspace.cpp @@ -61,7 +61,6 @@ void Workspace::start_tracking_file(const LSPSpec::DocumentUri& file_uri, file.m_all_types_uri, file.m_game_version, file.m_all_types_file_path); m_tracked_all_types_files[file.m_all_types_uri].parse_type_system(); } - } } // TODO - only supporting IR files currently! diff --git a/test/offline/offline_test_main.cpp b/test/offline/offline_test_main.cpp index e55729990..5a4e1bcf0 100644 --- a/test/offline/offline_test_main.cpp +++ b/test/offline/offline_test_main.cpp @@ -389,11 +389,7 @@ std::optional parse_config(const std::string_view& game_name) } int main(int argc, char* argv[]) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - std::cerr << "Bad encoding, needs UTF-8." << std::endl; - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); lg::initialize(); diff --git a/test/test_main.cpp b/test/test_main.cpp index 2be6d8b2f..5d124e4d0 100644 --- a/test/test_main.cpp +++ b/test/test_main.cpp @@ -19,11 +19,7 @@ // to make it easier to test a subset of tests int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - std::cerr << "Bad encoding, needs UTF-8." << std::endl; - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); // hopefully get a debug print on github actions setup_cpu_info(); diff --git a/tools/dgo_packer.cpp b/tools/dgo_packer.cpp index 059670a66..e75953a7b 100644 --- a/tools/dgo_packer.cpp +++ b/tools/dgo_packer.cpp @@ -1,15 +1,14 @@ #include -#include "common/versions.h" -#include "common/util/FileUtil.h" + #include "common/util/BinaryWriter.h" -#include "third-party/json.hpp" +#include "common/util/FileUtil.h" +#include "common/versions.h" #include +#include "third-party/json.hpp" + int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); printf("DGO Packing Tool\n"); diff --git a/tools/dgo_unpacker.cpp b/tools/dgo_unpacker.cpp index 2b188bdca..89fcb229a 100644 --- a/tools/dgo_unpacker.cpp +++ b/tools/dgo_unpacker.cpp @@ -1,8 +1,9 @@ #include #include -#include "common/versions.h" -#include "common/util/FileUtil.h" + #include "common/util/DgoReader.h" +#include "common/util/FileUtil.h" +#include "common/versions.h" #include namespace { @@ -49,10 +50,7 @@ int run(int argc, char** argv) { } // namespace int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); try { return run(argc, argv); diff --git a/tools/level_tools/level_dump/main.cpp b/tools/level_tools/level_dump/main.cpp index fdc61c831..d253c612c 100644 --- a/tools/level_tools/level_dump/main.cpp +++ b/tools/level_tools/level_dump/main.cpp @@ -1,14 +1,14 @@ -#include "third-party/fmt/core.h" +#include "common/util/Assert.h" +#include "common/util/DgoReader.h" #include "common/util/FileUtil.h" +#include + #include "decompiler/ObjectFile/LinkedObjectFile.h" #include "decompiler/ObjectFile/LinkedObjectFileCreation.h" -#include "common/util/DgoReader.h" - -#include "decompiler/util/goal_data_reader.h" #include "decompiler/level_extractor/BspHeader.h" +#include "decompiler/util/goal_data_reader.h" -#include "common/util/Assert.h" -#include +#include "third-party/fmt/core.h" constexpr GameVersion kGameVersion = GameVersion::Jak1; @@ -56,10 +56,7 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) { } int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); try { fmt::print("Level Dump Tool\n"); diff --git a/tools/memory_dump_tool/main.cpp b/tools/memory_dump_tool/main.cpp index 9ca51d9f4..309e6455c 100644 --- a/tools/memory_dump_tool/main.cpp +++ b/tools/memory_dump_tool/main.cpp @@ -612,10 +612,7 @@ void inspect_symbols(const Ram& ram, } int main(int argc, char** argv) { - fs::u8arguments u8guard(argc, argv); - if (!u8guard.valid()) { - exit(EXIT_FAILURE); - } + ArgumentGuard u8_guard(argc, argv); fs::path dump_path; fs::path output_path;