utf8: fix locale mis-handling on linux (#1698)

* utf8: fix locale mis-handling on linux

* lint: formatting

.github/workflows/linting.yaml

@@ -25,4 +25,4 @@ jobs:
       - name: Check Clang-Formatting
         run: |
           chmod +x ./third-party/run-clang-format/run-clang-format.py
-          ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test --color always
+          ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp --color always

Taskfile.yml

@@ -61,7 +61,7 @@ tasks:
   format:
     desc: "Format code"
     cmds:
-      - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test -i
+      - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
       # npm install -g prettier
       - cmd: npx prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
         ignore_error: true

common/util/unicode_util.h

@@ -1,8 +1,11 @@
 #pragma once
 
 #include <string>
+#include <string_view>
 #include <vector>
 
+#include "common/util/FileUtil.h"
+
 #ifdef _WIN32
 std::wstring utf8_string_to_wide_string(const std::string_view& str);
 bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str);
@@ -11,3 +14,26 @@ bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str)
 #endif
 
 std::string get_env(const std::string& name);
+
+/// @brief Windows's command line args are not UTF-8 so they need to be converted
+/// ghc::filesystem comes with an u8guard class to help with this, but on linux it calls
+/// setlocale
+///
+/// This is problematic for several annoying reasons, so we are going off the (hopefully sane)
+/// assumption that linux is using utf-8 by default and that the args are already proper
+///
+/// This may need to be updated on linux if that assumption proves wrong.
+class ArgumentGuard {
+ public:
+#ifdef _WIN32
+  ArgumentGuard(int& argc, char**& argv) : u8_guard(argc, argv) {
+    if (!u8_guard.valid()) {
+      exit(EXIT_FAILURE);
+    }
+  }
+  fs::u8arguments u8_guard;
+#else
+  // if linux
+  ArgumentGuard(int&, char**&) {}
+#endif
+};

decompiler/extractor/main.cpp

@@ -223,6 +223,8 @@ void launch_game() {
 }
 
 int main(int argc, char** argv) {
+  ArgumentGuard u8_guard(argc, argv);
+
   fs::path input_file_path;
   fs::path project_path_override;
   bool flag_runall = false;
@@ -234,12 +236,6 @@ int main(int argc, char** argv) {
   bool flag_folder = false;
   std::string game_name = "jak1";
 
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    lg::error("Bad encoding, needs UTF-8");
-    exit(EXIT_FAILURE);
-  }
-
   lg::initialize();
 
   CLI::App app{"OpenGOAL Level Extraction Tool"};

decompiler/main.cpp

@@ -18,10 +18,7 @@
 #include "decompiler/level_extractor/extract_level.h"
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   Timer decomp_timer;
 

game/main.cpp

@@ -42,10 +42,7 @@ void setup_logging(bool verbose) {
  * Entry point for the game.
  */
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   // TODO - replace with CLI11 and just propagate args through
   // - https://github.com/CLIUtils/CLI11/issues/744

lsp/handlers/lsp_router.cpp

@@ -1,9 +1,8 @@
 #include "lsp_router.h"
 
-#include "lsp/handlers/initialize.h"
-
 #include "common/log/log.h"
 
+#include "lsp/handlers/initialize.h"
 #include "lsp/protocol/error_codes.h"
 #include "text_document/document_symbol.h"
 #include "text_document/document_synchronization.h"

lsp/main.cpp

@@ -13,6 +13,7 @@
 #include <regex>
 
 #include "common/log/log.h"
+#include "common/util/unicode_util.h"
 
 #include "lsp/handlers/lsp_router.h"
 #include "lsp/state/workspace.h"
@@ -48,10 +49,7 @@ void setup_logging(bool verbose, std::string log_file) {
 }
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   CLI::App app{"OpenGOAL Language Server"};
 

lsp/protocol/hover.h

@@ -30,7 +30,7 @@ namespace LSPSpec {
  * decide to remove HTML from the markdown to avoid script execution.
  */
 struct MarkupContent {
-  std::string m_kind; // Actually a MarkupKind which is either 'plaintext' or 'markdown'
+  std::string m_kind;  // Actually a MarkupKind which is either 'plaintext' or 'markdown'
   std::string m_value;
 };
 

lsp/state/workspace.cpp

@@ -61,7 +61,6 @@ void Workspace::start_tracking_file(const LSPSpec::DocumentUri& file_uri,
             file.m_all_types_uri, file.m_game_version, file.m_all_types_file_path);
         m_tracked_all_types_files[file.m_all_types_uri].parse_type_system();
       }
-      
     }
   }
   // TODO - only supporting IR files currently!

test/offline/offline_test_main.cpp

@@ -389,11 +389,7 @@ std::optional<OfflineTestConfig> parse_config(const std::string_view& game_name)
 }
 
 int main(int argc, char* argv[]) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    std::cerr << "Bad encoding, needs UTF-8." << std::endl;
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   lg::initialize();
 

test/test_main.cpp

@@ -19,11 +19,7 @@
 // to make it easier to test a subset of tests
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    std::cerr << "Bad encoding, needs UTF-8." << std::endl;
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   // hopefully get a debug print on github actions
   setup_cpu_info();

tools/dgo_packer.cpp

@@ -1,15 +1,14 @@
 #include <cstdio>
-#include "common/versions.h"
-#include "common/util/FileUtil.h"
+
 #include "common/util/BinaryWriter.h"
-#include "third-party/json.hpp"
+#include "common/util/FileUtil.h"
+#include "common/versions.h"
 #include <common/util/unicode_util.h>
 
+#include "third-party/json.hpp"
+
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
   printf("DGO Packing Tool\n");

tools/dgo_unpacker.cpp

@@ -1,8 +1,9 @@
 #include <cstdio>
 #include <stdexcept>
-#include "common/versions.h"
-#include "common/util/FileUtil.h"
+
 #include "common/util/DgoReader.h"
+#include "common/util/FileUtil.h"
+#include "common/versions.h"
 #include <common/util/unicode_util.h>
 
 namespace {
@@ -49,10 +50,7 @@ int run(int argc, char** argv) {
 }  // namespace
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   try {
     return run(argc, argv);

tools/level_tools/level_dump/main.cpp

@@ -1,14 +1,14 @@
-#include "third-party/fmt/core.h"
+#include "common/util/Assert.h"
+#include "common/util/DgoReader.h"
 #include "common/util/FileUtil.h"
+#include <common/util/unicode_util.h>
+
 #include "decompiler/ObjectFile/LinkedObjectFile.h"
 #include "decompiler/ObjectFile/LinkedObjectFileCreation.h"
-#include "common/util/DgoReader.h"
-
-#include "decompiler/util/goal_data_reader.h"
 #include "decompiler/level_extractor/BspHeader.h"
+#include "decompiler/util/goal_data_reader.h"
 
-#include "common/util/Assert.h"
-#include <common/util/unicode_util.h>
+#include "third-party/fmt/core.h"
 
 constexpr GameVersion kGameVersion = GameVersion::Jak1;
 
@@ -56,10 +56,7 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) {
 }
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   try {
     fmt::print("Level Dump Tool\n");

tools/memory_dump_tool/main.cpp

@@ -612,10 +612,7 @@ void inspect_symbols(const Ram& ram,
 }
 
 int main(int argc, char** argv) {
-  fs::u8arguments u8guard(argc, argv);
-  if (!u8guard.valid()) {
-    exit(EXIT_FAILURE);
-  }
+  ArgumentGuard u8_guard(argc, argv);
 
   fs::path dump_path;
   fs::path output_path;