repl: Add a few new quality of life improvements (#2030)

- You can define a `startup.gc` in your user folder, each line will be
executed on startup (deprecates the usefulness of some cli flags)
- You can define a `repl-config.json` file to override REPL settings.
Long-term this is a better approach than a bunch of CLI flags as well
- Via this, you can override the amount of time the repl will attempt to
listen for the target
- At the same time, I think i may have found why on Windows it can
sometimes take forever to timeout when the game dies, will dig into this
later
- Added some keybinds for common operations, shown here


https://user-images.githubusercontent.com/13153231/202890278-1ff2bb06-dddf-4bde-9178-aa0883799167.mp4
> builds the game, connects to it, attaches a debugger and continues,
launches it, gets the backtrace, stops the target -- all with only
keybinds.

If you want these keybinds to work inside VSCode's integrated terminal,
you need to add the following to your settings file
```json
"terminal.integrated.commandsToSkipShell": [
    "-workbench.action.quickOpen",
    "-workbench.action.quickOpenView"
  ]
```

.vscode/settings.json

@@ -8,5 +8,5 @@
     "editor.wordBasedSuggestions": true,
     "editor.snippetSuggestions": "top"
   },
-  "python.formatting.provider": "black",
+  "python.formatting.provider": "black"
 }

Taskfile.yml

@@ -67,10 +67,10 @@ tasks:
       - sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
         msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
     cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --game {{.GAME}}"
+      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --user-auto --game {{.GAME}}"
   repl-lt:
     cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt --game {{.GAME}}"
+      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt --user-auto --game {{.GAME}}"
   format:
     desc: "Format code"
     cmds:

common/cross_sockets/XSocket.cpp

@@ -142,6 +142,8 @@ int set_socket_timeout(int socket, long microSeconds) {
   return ret;
 #elif _WIN32
   DWORD timeout = microSeconds / 1000;  // milliseconds
+  // TODO - NOTE this might be bad / unreliable if the socket ends up being in a bad state
+  // select() instead should be used, will do so if ends up being an issue in practice
   return set_socket_option(socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
 #endif
 }

common/goos/ReplUtils.cpp

@@ -130,6 +130,31 @@ void ReplWrapper::print_help_message() {
   fmt::print(" - Enter a GOOS REPL\n");
 }
 
-void ReplWrapper::init_default_settings() {
-  repl.set_word_break_characters(" \t");
+replxx::Replxx::key_press_handler_t ReplWrapper::commit_text_action(std::string text_to_commit) {
+  return [this, text_to_commit](char32_t code) {
+    repl.set_state(
+        replxx::Replxx::State(text_to_commit.c_str(), static_cast<int>(text_to_commit.size())));
+    return repl.invoke(Replxx::ACTION::COMMIT_LINE, code);
+  };
+}
+
+void ReplWrapper::init_default_settings() {
+  // NOTE - a nice popular project that uses replxx
+  // - https://github.com/ClickHouse/ClickHouse/blob/master/base/base/ReplxxLineReader.cpp#L366
+  repl.set_word_break_characters(" \t");
+  // Setup default keybinds
+  // (test-play) : Ctrl-P
+  repl.bind_key(Replxx::KEY::control('T'), commit_text_action("(test-play)"));
+  // (e) : Ctrl-Q
+  repl.bind_key(Replxx::KEY::control('Q'), commit_text_action("(e)"));
+  // (lt) : Ctrl-G
+  repl.bind_key(Replxx::KEY::control('L'), commit_text_action("(lt)"));
+  /// (:stop) : Ctrl-H
+  repl.bind_key(Replxx::KEY::control('W'), commit_text_action("(:stop)"));
+  // (dbgc) : Ctrl-F
+  repl.bind_key(Replxx::KEY::control('G'), commit_text_action("(dbgc)"));
+  // (:di) : Ctrl-B
+  repl.bind_key(Replxx::KEY::control('B'), commit_text_action("(:di)"));
+  // (mi) : Ctrl-M
+  repl.bind_key(Replxx::KEY::control('N'), commit_text_action("(mi)"));
 }

common/goos/ReplUtils.h

@@ -31,4 +31,7 @@ class ReplWrapper {
   std::vector<std::string> examples{};
   using cl = Replxx::Color;
   std::vector<std::pair<std::string, cl>> regex_colors{};
+
+ private:
+  replxx::Replxx::key_press_handler_t commit_text_action(std::string text_to_commit);
 };

decompiler/config/jak2/hacks.jsonc

@@ -395,56 +395,56 @@
     "find-offending-process-focusable": [16, 19],
 
     "target-standard-event-handler": [
-        5,  // fwd L31/7
-        6,  // to 152
-        7,  // fwd L38
-        20, // to 152
-        21, // fwd L39
-        22, // to 152
-        23, // to L40
-        24, // to 152
-        25, // to L45
-        43, // to l152
-        44, // to 46
-        45, // to l152
+      5, // fwd L31/7
+      6, // to 152
+      7, // fwd L38
+      20, // to 152
+      21, // fwd L39
+      22, // to 152
+      23, // to L40
+      24, // to 152
+      25, // to L45
+      43, // to l152
+      44, // to 46
+      45, // to l152
 
-        46, // to l114
+      46, // to l114
 
-        190, // to l152
-        191, // to 121
-        210,
-        211,
-        212,
-        213,
-        214,
+      190, // to l152
+      191, // to 121
+      210,
+      211,
+      212,
+      213,
+      214,
 
-        224,
-        225,
-        226,
-        227,
-        232,
-        233,
-        234,
-        235,
-        236,
-        237,
-        238,
-        239,
-        240,
-        241,
-        242,
+      224,
+      225,
+      226,
+      227,
+      232,
+      233,
+      234,
+      235,
+      236,
+      237,
+      238,
+      239,
+      240,
+      241,
+      242,
 
-        255,
-        256,
+      255,
+      256,
 
-        269,
-        270,
-        285,
-        286,
-        287,
-        288,
-        289,
-        290
+      269,
+      270,
+      285,
+      286,
+      287,
+      288,
+      289,
+      290
     ]
   },
 

decompiler/config/jak2/stack_structures.jsonc

@@ -1191,9 +1191,13 @@
   "birth-func-vector-orient": [[32, "vector"]],
   "(enter cam-launcher-longfall)": [[16, "vector"]],
   "explosion-init-by-other": [[16, "traffic-danger-info"]],
-  "process-drawable-shock-skel-effect": [[176, "matrix"], [256, "vector"], [272, "vector"],
-   [288, "vector"], [304, "quaternion"], [384, "vector"]],
-  "(method 20 collide-cache)": [
-    [16, "matrix"]
-  ]
+  "process-drawable-shock-skel-effect": [
+    [176, "matrix"],
+    [256, "vector"],
+    [272, "vector"],
+    [288, "vector"],
+    [304, "quaternion"],
+    [384, "vector"]
+  ],
+  "(method 20 collide-cache)": [[16, "matrix"]]
 }

decompiler/config/jak2/type_casts.jsonc

@@ -3138,7 +3138,10 @@
   "memcard-unlocked-secrets?": [[50, "s5", "int"]],
   "(method 10 progress)": [[45, "t9", "(function progress none)"]],
   "load-game-text-info": [[4, "v1", "game-text-info"]],
-  "(method 16 camera-master)": [[[11, 15], "a2", "target"], [[16, 18], "v1", "int"]],
+  "(method 16 camera-master)": [
+    [[11, 15], "a2", "target"],
+    [[16, 18], "v1", "int"]
+  ],
   "master-choose-entity": [
     ["_stack_", 96, "res-tag"],
     [[87, 247], "s3", "(pointer camera-slave)"]
@@ -4935,9 +4938,7 @@
   "(method 21 collide-cache)": [
     [[62, 86], "a3", "(inline-array collide-cache-tri)"]
   ],
-  "(method 13 collide-cache)": [
-    [303, "v1", "process-drawable"]
-  ],
+  "(method 13 collide-cache)": [[303, "v1", "process-drawable"]],
   "(method 19 collide-cache)": [
     [26, "a0", "connection"],
     [27, "a0", "collide-shape"],

decompiler/config/jak2/var_names.jsonc

@@ -2187,11 +2187,11 @@
       "s1-1": "vel-dir"
     }
   },
-    "target-collision-low-coverage": {
+  "target-collision-low-coverage": {
     "vars": {
-      "sv-16":"contact-normal",
-      "sv-56":"overhang-nrm",
-      "sv-52":"tangent"
+      "sv-16": "contact-normal",
+      "sv-56": "overhang-nrm",
+      "sv-52": "tangent"
     }
   }
 }

goal_src/user/.gitignore

@@ -1,3 +1,3 @@
 *
 !.gitignore
-!readme.md
+!readme.md

goal_src/user/readme.md

@@ -12,3 +12,8 @@ which contains just the username you want to log in as. That way you don't have
 modify multiple scripts when you want to change users.
 
 If you want to make your profile public, edit the .gitignore in this directory.
+
+Additionally, you can provide a `repl-config.json` to set various REPL settings:
+- `numConnectToTargetAttempts` - the number of times the REPL will attempt to connect to the target on an `(lt)`
+
+And a `startup.gc` where each line will be executed upon startup

goalc/compiler/Compiler.h

@@ -93,6 +93,7 @@ class Compiler {
                      std::vector<std::pair<std::string, Replxx::Color>> const& user_data);
   bool knows_object_file(const std::string& name);
   MakeSystem& make_system() { return m_make; }
+  void update_via_config_file(const std::string& json);
 
  private:
   GameVersion m_version;
@@ -113,6 +114,7 @@ class Compiler {
   SymbolInfoMap m_symbol_info;
   std::unique_ptr<ReplWrapper> m_repl;
   MakeSystem m_make;
+  int m_target_connect_attempts = 30;
 
   struct DebugStats {
     int num_spills = 0;
@@ -628,6 +630,7 @@ class Compiler {
 
   // Debug
   Val* compile_dbg(const goos::Object& form, const goos::Object& rest, Env* env);
+  Val* compile_dbg_and_continue(const goos::Object& form, const goos::Object& rest, Env* env);
   Val* compile_dbs(const goos::Object& form, const goos::Object& rest, Env* env);
   Val* compile_break(const goos::Object& form, const goos::Object& rest, Env* env);
   Val* compile_cont(const goos::Object& form, const goos::Object& rest, Env* env);

goalc/compiler/Util.cpp

@@ -1,5 +1,6 @@
 #include "common/goos/ParseHelpers.h"
 #include "common/type_system/deftype.h"
+#include "common/util/json_util.h"
 
 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
@@ -153,6 +154,13 @@ bool Compiler::knows_object_file(const std::string& name) {
   return m_debugger.knows_object(name);
 }
 
+void Compiler::update_via_config_file(const std::string& json) {
+  auto cfg = parse_commented_json(json, "repl-config.json");
+  if (cfg.contains("numConnectToTargetAttempts")) {
+    m_target_connect_attempts = cfg.at("numConnectToTargetAttempts").get<int>();
+  }
+}
+
 /*!
  * Parse arguments into a goos::Arguments format.
  */

goalc/compiler/compilation/Atoms.cpp

@@ -177,6 +177,7 @@ const std::unordered_map<
         // DEBUGGING
         {"dbs", &Compiler::compile_dbs},
         {"dbg", &Compiler::compile_dbg},
+        {"dbgc", &Compiler::compile_dbg_and_continue},
         {":cont", &Compiler::compile_cont},
         {":stop", &Compiler::compile_stop},
         {":break", &Compiler::compile_break},

goalc/compiler/compilation/CompilerControl.cpp

@@ -207,7 +207,7 @@ Val* Compiler::compile_listen_to_target(const goos::Object& form,
     }
   });
 
-  m_listener.connect_to_target(30, ip, port);
+  m_listener.connect_to_target(m_target_connect_attempts, ip, port);
   return get_none();
 }
 

goalc/compiler/compilation/Debug.cpp

@@ -76,6 +76,14 @@ Val* Compiler::compile_dbg(const goos::Object& form, const goos::Object& rest, E
   return get_none();
 }
 
+Val* Compiler::compile_dbg_and_continue(const goos::Object& form,
+                                        const goos::Object& rest,
+                                        Env* env) {
+  compile_dbg(form, rest, env);
+  compile_cont(form, rest, env);
+  return get_none();
+}
+
 Val* Compiler::compile_dbs(const goos::Object& form, const goos::Object& rest, Env* env) {
   // todo - do something with args.
   (void)form;

goalc/main.cpp

@@ -5,9 +5,9 @@
 #include "common/log/log.h"
 #include "common/nrepl/ReplServer.h"
 #include "common/util/FileUtil.h"
+#include "common/util/diff.h"
 #include "common/versions.h"
 
-#include "SQLiteCpp/SQLiteCpp.h"
 #include "goalc/compiler/Compiler.h"
 
 #include "third-party/CLI11.hpp"
@@ -33,6 +33,8 @@ int main(int argc, char** argv) {
   int nrepl_port = 8181;
   fs::path project_path_override;
 
+  // TODO - a lot of these flags could be deprecated and moved into `repl-config.json`
+  // TODO - auto-find the user if there is only one folder within `user/`
   CLI::App app{"OpenGOAL Compiler / REPL"};
   app.add_option("-c,--cmd", cmd, "Specify a command to run");
   app.add_option("--startup-cmd", startup_cmd,
@@ -67,6 +69,9 @@ int main(int argc, char** argv) {
     return 1;
   }
 
+  std::vector<std::string> user_startup_commands = {};
+  std::optional<std::string> repl_config = {};
+
   if (auto_find_user) {
     username = "#f";
     std::regex allowed_chars("[0-9a-zA-Z\\-\\.\\!\\?<>]");
@@ -85,6 +90,22 @@ int main(int argc, char** argv) {
       }
       if (!found_username.empty()) {
         username = found_username;
+        // Check for a `startup.gc` file, each line will be executed on the REPL on startup
+        auto startup_file_path =
+            file_util::get_file_path({"goal_src", "user", username, "startup.gc"});
+        if (file_util::file_exists(startup_file_path)) {
+          auto data = file_util::read_text_file(startup_file_path);
+          auto startup_cmds = split_string(data);
+          for (const auto& cmd : startup_cmds) {
+            user_startup_commands.push_back(cmd);
+          }
+        }
+        // Check for a `repl-config.json` file, so things can be configured without tons of flags
+        auto repl_config_path =
+            file_util::get_file_path({"goal_src", "user", username, "repl-config.json"});
+        if (file_util::file_exists(repl_config_path)) {
+          repl_config = file_util::read_text_file(repl_config_path);
+        }
       }
     } catch (std::exception& e) {
       printf("error opening user desc file: %s\n", e.what());
@@ -104,7 +125,6 @@ int main(int argc, char** argv) {
   // the compiler may throw an exception if it fails to load its standard library.
   try {
     std::unique_ptr<Compiler> compiler;
-    // TODO - allow passing in an iso_data override
     std::mutex compiler_mutex;
     // if a command is provided on the command line, no REPL just run the compiler on it
     if (!cmd.empty()) {
@@ -112,6 +132,10 @@ int main(int argc, char** argv) {
       compiler->run_front_end_on_string(cmd);
       return 0;
     }
+    compiler = std::make_unique<Compiler>(game_version, username, std::make_unique<ReplWrapper>());
+    if (repl_config) {
+      compiler->update_via_config_file(repl_config.value());
+    }
     // Start nREPL Server
     if (repl_server_ok) {
       nrepl_thread = std::thread([&]() {
@@ -128,14 +152,23 @@ int main(int argc, char** argv) {
       });
     }
     // Run automatic forms if applicable
-    if (auto_debug || auto_listen) {
-      std::lock_guard<std::mutex> lock(compiler_mutex);
-      status = compiler->handle_repl_string("(lt)");
+    // - this should probably be deprecated in favor of the `startup.gc` file
+    if (user_startup_commands.empty() && (auto_debug || auto_listen)) {
+      user_startup_commands.push_back("(lt)");
     }
-    if (auto_debug) {
+    if (user_startup_commands.empty() && auto_debug) {
       std::lock_guard<std::mutex> lock(compiler_mutex);
-      status = compiler->handle_repl_string("(dbg) (:cont)");
+      status = compiler->handle_repl_string("(dbgc)");
+      user_startup_commands.push_back("(dbgc)");
     }
+
+    if (!user_startup_commands.empty()) {
+      std::lock_guard<std::mutex> lock(compiler_mutex);
+      for (const auto& cmd : user_startup_commands) {
+        status = compiler->handle_repl_string(cmd);
+      }
+    }
+
     // Poll Terminal
     while (status != ReplStatus::WANT_EXIT) {
       if (status == ReplStatus::WANT_RELOAD) {
@@ -146,6 +179,9 @@ int main(int argc, char** argv) {
         }
         compiler =
             std::make_unique<Compiler>(game_version, username, std::make_unique<ReplWrapper>());
+        if (repl_config) {
+          compiler->update_via_config_file(repl_config.value());
+        }
         if (!startup_cmd.empty()) {
           compiler->handle_repl_string(startup_cmd);
           // reset to prevent re-executing on manual reload