LSP: A bunch of new OpenGOAL language features (#3437)

- Integrate the AST into the LSP, this makes parsing and tokenizing the
files much easier
- Consolidate most of the symbol info tracking in `goalc` to a single
map. Fixed some issues where the old map would never evict symbols when
re-compiling files. There is still some more to cleanup, but this now
can be used as an incrementally updated source-of-truth for the LSP
- re-compile files when they are saved. Ideally this would be done
everytime they are changed but that:
  - may be too aggressive
- goalc doesn't compile incrementally yet so it likely would be a worse
UX

Features added, see
https://github.com/open-goal/opengoal-vscode/issues/256
- Hover

![image](https://github.com/open-goal/jak-project/assets/13153231/58dadb5d-582c-4c1f-9ffe-eaa4c85a0255)

![image](https://github.com/open-goal/jak-project/assets/13153231/b383adde-57fc-462c-a256-b2de5c30ca9a)
- LSP Status fixed
- Type Hierarchy

![image](https://github.com/open-goal/jak-project/assets/13153231/8e681377-1d4e-4336-ad70-1695a4607340)
- Document Color

![image](https://github.com/open-goal/jak-project/assets/13153231/4e48ccd8-0ed1-4459-a133-5277561e4201)
- Document Symbols
![Screenshot 2024-03-27
004105](https://github.com/open-goal/jak-project/assets/13153231/8e655034-43c4-4261-b6e0-85de00cbfc7f)
- Completions
![Screenshot 2024-03-30
004504](https://github.com/open-goal/jak-project/assets/13153231/d123a187-af90-466b-9eb7-561b2ee97cd1)

---------

Co-authored-by: Hat Kid <6624576+Hat-Kid@users.noreply.github.com>

.github/workflows/windows-build-clang.yaml

@@ -61,7 +61,6 @@ jobs:
         run: cmake --build build --parallel %NUMBER_OF_PROCESSORS%
 
       - name: Run Tests
-        timeout-minutes: 10
         env:
           GTEST_OUTPUT: "xml:opengoal-test-report.xml"
         run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"

.vs/launch.vs.json

@@ -143,13 +143,6 @@
       "name": "Game - Jak 2 - Runtime (release)",
       "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso"]
     },
-    {
-      "type": "default",
-      "project": "CMakeLists.txt",
-      "projectTarget": "goalc.exe (bin\\goalc.exe)",
-      "name": "REPL",
-      "args": ["--user-auto"]
-    },
     {
       "type": "default",
       "project": "CMakeLists.txt",

.vscode/settings.json

@@ -7,5 +7,6 @@
     },
     "editor.wordBasedSuggestions": "matchingDocuments",
     "editor.snippetSuggestions": "top"
-  }
+  },
+  "cmake.configureOnOpen": false
 }

common/CMakeLists.txt

@@ -66,6 +66,7 @@ add_library(common
         type_system/TypeSpec.cpp
         type_system/TypeSystem.cpp
         util/Assert.cpp
+        util/ast_util.cpp
         util/BitUtils.cpp
         util/compress.cpp
         util/crc32.cpp
@@ -87,7 +88,7 @@ add_library(common
         util/Timer.cpp
         util/unicode_util.cpp
         versions/versions.cpp
-        )
+         "util/trie_map.h")
 
 target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs)
 

common/formatter/formatter.cpp

@@ -2,8 +2,11 @@
 
 #include "formatter_tree.h"
 
+#include "common/formatter/rules/formatting_rules.h"
+#include "common/formatter/rules/rule_config.h"
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
+#include "common/util/ast_util.h"
 #include "common/util/string_util.h"
 
 #include "tree_sitter/api.h"
@@ -400,8 +403,6 @@ std::string join_formatted_lines(const std::vector<std::string>& lines,
 std::optional<std::string> formatter::format_code(const std::string& source) {
   // Create a parser.
   std::shared_ptr<TSParser> parser(ts_parser_new(), TreeSitterParserDeleter());
-
-  // Set the parser's language (JSON in this case).
   ts_parser_set_language(parser.get(), tree_sitter_opengoal());
 
   // Build a syntax tree based on source code stored in a string.

common/formatter/formatter.h

@@ -3,13 +3,8 @@
 #include <optional>
 #include <string>
 
-#include "common/formatter/rules/formatting_rules.h"
-#include "common/formatter/rules/rule_config.h"
-
 #include "tree_sitter/api.h"
 
-// TODO:
-// - Considering _eventually_ adding line-length heuristics
 namespace formatter {
 
 struct TreeSitterParserDeleter {

common/type_system/TypeSystem.cpp

@@ -1442,6 +1442,21 @@ std::vector<std::string> TypeSystem::search_types_by_parent_type(
   return results;
 }
 
+std::vector<std::string> TypeSystem::search_types_by_parent_type_strict(
+    const std::string& parent_type) {
+  std::vector<std::string> results = {};
+  for (const auto& [type_name, type_info] : m_types) {
+    // Only NullType's have no parent
+    if (!type_info->has_parent()) {
+      continue;
+    }
+    if (type_info->get_parent() == parent_type) {
+      results.push_back(type_name);
+    }
+  }
+  return results;
+}
+
 std::vector<std::string> TypeSystem::search_types_by_minimum_method_id(
     const int minimum_method_id,
     const std::optional<std::vector<std::string>>& existing_matches) {

common/type_system/TypeSystem.h

@@ -278,6 +278,7 @@ class TypeSystem {
   std::vector<std::string> search_types_by_parent_type(
       const std::string& parent_type,
       const std::optional<std::vector<std::string>>& existing_matches = {});
+  std::vector<std::string> search_types_by_parent_type_strict(const std::string& parent_type);
 
   std::vector<std::string> search_types_by_minimum_method_id(
       const int minimum_method_id,

common/util/FileUtil.cpp

@@ -736,4 +736,25 @@ std::string get_majority_file_line_endings(const std::string& file_contents) {
   return "\n";
 }
 
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents) {
+  size_t lf_count = 0;
+  size_t crlf_count = 0;
+
+  for (size_t i = 0; i < file_contents.size(); ++i) {
+    if (file_contents[i] == '\n') {
+      if (i > 0 && file_contents[i - 1] == '\r') {
+        crlf_count++;
+      } else {
+        lf_count++;
+      }
+    }
+  }
+
+  if (crlf_count > lf_count) {
+    return {lf_count + crlf_count, "\r\n"};
+  }
+  return {lf_count + crlf_count, "\n"};
+}
+
 }  // namespace file_util

common/util/FileUtil.h

@@ -72,4 +72,6 @@ std::vector<fs::path> sort_filepaths(const std::vector<fs::path>& paths, const b
 void copy_file(const fs::path& src, const fs::path& dst);
 std::string make_screenshot_filepath(const GameVersion game_version, const std::string& name = "");
 std::string get_majority_file_line_endings(const std::string& file_contents);
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents);
 }  // namespace file_util

common/util/Range.h

@@ -67,4 +67,4 @@ class Range {
  private:
   T m_start = {};
   T m_end = {};
-};
+};

common/util/Trie.h

@@ -12,7 +12,7 @@
  * It owns the memory for the objects it stores.
  * Doing an insert will create a copy of your object.
  *
- * Other that deleting the whole thing, there is no support for removing a node.
+ * Other than deleting the whole thing, there is no support for removing a node.
  */
 template <typename T>
 class Trie {

common/util/ast_util.cpp

@@ -0,0 +1,31 @@
+#include "ast_util.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node) {
+  uint32_t start = ts_node_start_byte(node);
+  uint32_t end = ts_node_end_byte(node);
+  return source.substr(start, end - start);
+}
+
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results) {
+  if (ts_node_child_count(curr_node) == 0) {
+    return;
+  }
+  std::vector<std::string> node_elements;
+  bool added = false;
+  for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
+    const auto child_node = ts_node_child(curr_node, i);
+    const auto contents = get_source_code(source, child_node);
+    node_elements.push_back(contents);
+    // Check for a match
+    if (node_elements == prefix && !added) {
+      results.push_back(curr_node);
+      added = true;
+    }
+    search_for_forms_that_begin_with(source, child_node, prefix, results);
+  }
+}
+}  // namespace ast_util

common/util/ast_util.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "tree_sitter/api.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node);
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results);
+
+}  // namespace ast_util

common/util/trie_map.h

@@ -0,0 +1,160 @@
+#pragma once
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// TrieMap class
+template <typename T>
+class TrieMap {
+ private:
+  // TrieNode structure
+  struct TrieNode {
+    std::unordered_map<char, std::shared_ptr<TrieNode>> children;
+    std::vector<std::shared_ptr<T>> elements;
+  };
+
+  std::shared_ptr<TrieNode> root;
+
+ public:
+  TrieMap() : root(std::make_shared<TrieNode>()) {}
+
+  // Insert an element with a key into the TrieMap and return the inserted element
+  std::shared_ptr<T> insert(const std::string& key, const T& element) {
+    std::shared_ptr<T> shared_element = std::make_shared<T>(element);
+    std::shared_ptr<TrieNode> node = root;
+    for (char c : key) {
+      if (node->children.find(c) == node->children.end()) {
+        node->children[c] = std::make_shared<TrieNode>();
+      }
+      node = node->children[c];
+    }
+    // Store element at the leaf node
+    node->elements.push_back(shared_element);
+    return shared_element;
+  }
+
+  // Retrieve elements with a given prefix
+  std::vector<std::shared_ptr<T>> retrieve_with_prefix(const std::string& prefix) const {
+    std::vector<std::shared_ptr<T>> result;
+    std::shared_ptr<TrieNode> node = root;
+    // Traverse to the node representing the prefix
+    for (char c : prefix) {
+      if (node->children.find(c) == node->children.end()) {
+        return result;  // No elements with the given prefix
+      }
+      node = node->children[c];
+    }
+    // Gather all elements stored at or below this node
+    retrieve_elements(node, result);
+    return result;
+  }
+
+  // Retrieve elements with an exact key match
+  std::vector<std::shared_ptr<T>> retrieve_with_exact(const std::string& key) const {
+    std::vector<std::shared_ptr<T>> result;
+    std::shared_ptr<TrieNode> node = root;
+    // Traverse to the node representing the key
+    for (char c : key) {
+      if (node->children.find(c) == node->children.end()) {
+        return result;  // No elements with the given key
+      }
+      node = node->children[c];
+    }
+    // Return elements stored at this node
+    return node->elements;
+  }
+
+  // Remove the specified element from the TrieMap
+  void remove(const std::shared_ptr<T>& element) { remove_element(root, element); }
+
+  // Return the total number of elements stored in the TrieMap
+  int size() const {
+    int count = 0;
+    count_elements(root, count);
+    return count;
+  }
+
+  // Return a vector containing shared pointers to all elements stored in the TrieMap
+  std::vector<std::shared_ptr<T>> get_all_elements() const {
+    std::vector<std::shared_ptr<T>> result;
+    get_all_elements_helper(root, result);
+    return result;
+  }
+
+ private:
+  // Recursive function to retrieve elements stored at or below the given node
+  void retrieve_elements(std::shared_ptr<TrieNode> node,
+                         std::vector<std::shared_ptr<T>>& result) const {
+    // Add elements stored at this node to the result
+    for (const auto& element : node->elements) {
+      result.push_back(element);
+    }
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      retrieve_elements(child.second, result);
+    }
+  }
+
+  // Recursive function to remove the specified element from the TrieMap
+  bool remove_element(std::shared_ptr<TrieNode> node, const std::shared_ptr<T>& element) {
+    // Remove the element if it exists at this node
+    auto& elements = node->elements;
+    auto it = std::find(elements.begin(), elements.end(), element);
+    if (it != elements.end()) {
+      elements.erase(it);
+      return true;
+    }
+    // Recursively search children
+    for (auto& child : node->children) {
+      if (remove_element(child.second, element)) {
+        // Remove child node if it's empty after removal
+        if (child.second->elements.empty() && child.second->children.empty()) {
+          node->children.erase(child.first);
+        }
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // Recursive function to count elements stored at or below the given node
+  void count_elements(std::shared_ptr<TrieNode> node, int& count) const {
+    // Increment count by the number of elements stored at this node
+    count += node->elements.size();
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      count_elements(child.second, count);
+    }
+  }
+
+  // Recursive helper function to collect all elements stored in the TrieMap
+  void get_all_elements_helper(std::shared_ptr<TrieNode> node,
+                               std::vector<std::shared_ptr<T>>& result) const {
+    // Add elements stored at this node to the result
+    for (const auto& element : node->elements) {
+      result.push_back(element);
+    }
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      get_all_elements_helper(child.second, result);
+    }
+  }
+};
+
+// TrieMap<std::string> trie_map;
+//
+//// Insert elements
+// std::shared_ptr<std::string> inserted_element_1 = trie_map.insert("apple", "A fruit");
+// std::shared_ptr<std::string> inserted_element_2 = trie_map.insert("app", "An application");
+// std::shared_ptr<std::string> inserted_element_3 = trie_map.insert("banana", "Another fruit");
+// std::shared_ptr<std::string> inserted_element_4 = trie_map.insert("apple", "Another apple");
+//
+//// Remove an element
+// trie_map.remove(inserted_element_1);
+//
+//// Retrieve elements with a prefix
+// std::vector<std::shared_ptr<std::string>> prefix_results = trie_map.retrieve_with_prefix("app");

goalc/CMakeLists.txt

@@ -32,6 +32,7 @@ add_library(compiler
         compiler/CompilerSettings.cpp
         compiler/CodeGenerator.cpp
         compiler/StaticObject.cpp
+        compiler/symbol_info.cpp
         compiler/compilation/Asm.cpp
         compiler/compilation/Atoms.cpp
         compiler/compilation/CompilerControl.cpp

goalc/compiler/Compiler.cpp

@@ -25,8 +25,9 @@ Compiler::Compiler(GameVersion version,
     : m_version(version),
       m_goos(user_profile),
       m_debugger(&m_listener, &m_goos.reader, version),
+      m_make(repl_config, user_profile),
       m_repl(std::move(repl)),
-      m_make(repl_config, user_profile) {
+      m_symbol_info(&m_goos.reader.db) {
   m_listener.add_debugger(&m_debugger);
   m_listener.set_default_port(version);
   m_ts.add_builtin_types(m_version);
@@ -57,9 +58,7 @@ Compiler::Compiler(GameVersion version,
 
   // add built-in forms to symbol info
   for (const auto& [builtin_name, builtin_info] : g_goal_forms) {
-    SymbolInfo::Metadata sym_meta;
-    sym_meta.docstring = builtin_info.first;
-    m_symbol_info.add_builtin(builtin_name, sym_meta);
+    m_symbol_info.add_builtin(builtin_name, builtin_info.first);
   }
 
   // load auto-complete history, only if we are running in the interactive mode.
@@ -463,6 +462,10 @@ void Compiler::asm_file(const CompilationOptions& options) {
     file_path = candidate_paths.at(0).string();
   }
 
+  // Evict any symbols we have indexed for this file, this is what
+  // helps to ensure we have an up to date and accurate symbol index
+  m_symbol_info.evict_symbols_using_file_index(file_path);
+
   auto code = m_goos.reader.read_from_file({file_path});
 
   std::string obj_file_name = file_path;
@@ -489,7 +492,7 @@ void Compiler::asm_file(const CompilationOptions& options) {
     if (options.disassemble) {
       codegen_and_disassemble_object_file(obj_file, &data, &disasm, options.disasm_code_only);
       if (options.disassembly_output_file.empty()) {
-        printf("%s\n", disasm.c_str());
+        lg::print("{}\n", disasm);
       } else {
         file_util::write_text_file(options.disassembly_output_file, disasm);
       }
@@ -502,7 +505,7 @@ void Compiler::asm_file(const CompilationOptions& options) {
       if (m_listener.is_connected()) {
         m_listener.send_code(data, obj_file_name);
       } else {
-        printf("WARNING - couldn't load because listener isn't connected\n");  // todo log warn
+        lg::print("WARNING - couldn't load because listener isn't connected\n");  // todo log warn
       }
     }
 
@@ -515,15 +518,15 @@ void Compiler::asm_file(const CompilationOptions& options) {
     }
   } else {
     if (options.load) {
-      printf("WARNING - couldn't load because coloring is not enabled\n");
+      lg::print("WARNING - couldn't load because coloring is not enabled\n");
     }
 
     if (options.write) {
-      printf("WARNING - couldn't write because coloring is not enabled\n");
+      lg::print("WARNING - couldn't write because coloring is not enabled\n");
     }
 
     if (options.disassemble) {
-      printf("WARNING - couldn't disassemble because coloring is not enabled\n");
+      lg::print("WARNING - couldn't disassemble because coloring is not enabled\n");
     }
   }
 }

goalc/compiler/Compiler.h

@@ -12,7 +12,8 @@
 #include "goalc/compiler/CompilerSettings.h"
 #include "goalc/compiler/Env.h"
 #include "goalc/compiler/IR.h"
-#include "goalc/compiler/SymbolInfo.h"
+#include "goalc/compiler/docs/DocTypes.h"
+#include "goalc/compiler/symbol_info.h"
 #include "goalc/data_compiler/game_text_common.h"
 #include "goalc/debugger/Debugger.h"
 #include "goalc/emitter/Register.h"
@@ -96,9 +97,20 @@ class Compiler {
                      std::vector<std::pair<std::string, replxx::Replxx::Color>> const& user_data);
   bool knows_object_file(const std::string& name);
   MakeSystem& make_system() { return m_make; }
-  std::set<std::string> lookup_symbol_infos_starting_with(const std::string& prefix) const;
-  std::vector<SymbolInfo>* lookup_exact_name_info(const std::string& name) const;
+  std::vector<std::shared_ptr<symbol_info::SymbolInfo>> lookup_symbol_info_by_file(
+      const std::string& file_path) const;
+  std::vector<std::shared_ptr<symbol_info::SymbolInfo>> lookup_symbol_info_by_prefix(
+      const std::string& prefix) const;
+  std::set<std::string> lookup_symbol_names_starting_with(const std::string& prefix) const;
+  std::vector<std::shared_ptr<symbol_info::SymbolInfo>> lookup_exact_name_info(
+      const std::string& name) const;
   std::optional<TypeSpec> lookup_typespec(const std::string& symbol_name);
+  TypeSystem& type_system() { return m_ts; };
+  // TODO - rename these types / namespaces -- consolidate with SymbolInfo and whatever else tries
+  // to also do this work
+  std::tuple<std::unordered_map<std::string, Docs::SymbolDocumentation>,
+             std::unordered_map<std::string, Docs::FileDocumentation>>
+  generate_per_file_symbol_info();
 
  private:
   GameVersion m_version;
@@ -110,7 +122,9 @@ class Compiler {
   listener::Listener m_listener;
   goos::Interpreter m_goos;
   Debugger m_debugger;
+  // TODO - this should be able to be removed, these are stored in `m_symbol_info`
   std::unordered_map<std::string, goos::ArgumentSpec> m_macro_specs;
+  // TODO - this should be able to be removed, these are stored in `m_symbol_info`
   std::unordered_map<goos::InternedSymbolPtr, TypeSpec, goos::InternedSymbolPtr::hash>
       m_symbol_types;
   std::unordered_map<goos::InternedSymbolPtr, goos::Object, goos::InternedSymbolPtr::hash>
@@ -120,9 +134,9 @@ class Compiler {
   CompilerSettings m_settings;
   bool m_throw_on_define_extern_redefinition = false;
   std::unordered_set<std::string> m_allow_inconsistent_definition_symbols;
-  SymbolInfoMap m_symbol_info;
-  std::unique_ptr<REPL::Wrapper> m_repl;
   MakeSystem m_make;
+  std::unique_ptr<REPL::Wrapper> m_repl;
+  symbol_info::SymbolInfoMap m_symbol_info;
 
   struct DebugStats {
     int num_spills = 0;
@@ -307,7 +321,7 @@ class Compiler {
                                 int offset,
                                 Env* env);
 
-  std::string make_symbol_info_description(const SymbolInfo& info);
+  std::string make_symbol_info_description(const std::shared_ptr<symbol_info::SymbolInfo> info);
 
   MathMode get_math_mode(const TypeSpec& ts);
   bool is_number(const TypeSpec& ts);

goalc/compiler/SymbolInfo.h

@@ -1,249 +0,0 @@
-#pragma once
-
-#include <set>
-#include <string>
-#include <vector>
-
-#include "common/goos/Object.h"
-#include "common/util/Assert.h"
-#include "common/util/Trie.h"
-
-#include "goalc/compiler/Val.h"
-
-/*!
- * Info about a single symbol, representing one of:
- *  - Global variable
- *  - Global function
- *  - Type
- *  - Constant
- *  - Macro
- *  - Builtin keyword of the OpenGOAL language
- */
-class SymbolInfo {
- public:
-  struct Metadata {
-    std::string docstring = "";
-  };
-
-  // TODO - states
-  // TODO - enums
-  enum class Kind {
-    GLOBAL_VAR,
-    FWD_DECLARED_SYM,
-    FUNCTION,
-    TYPE,
-    CONSTANT,
-    MACRO,
-    LANGUAGE_BUILTIN,
-    METHOD,
-    INVALID
-  };
-
-  static SymbolInfo make_global(const std::string& name,
-                                const goos::Object& defining_form,
-                                const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::GLOBAL_VAR;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    return info;
-  }
-
-  static SymbolInfo make_fwd_declared_sym(const std::string& name,
-                                          const goos::Object& defining_form) {
-    SymbolInfo info;
-    info.m_kind = Kind::FWD_DECLARED_SYM;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    return info;
-  }
-
-  static SymbolInfo make_function(const std::string& name,
-                                  const std::vector<GoalArg> args,
-                                  const goos::Object& defining_form,
-                                  const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::FUNCTION;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    info.m_args = args;
-    return info;
-  }
-
-  static SymbolInfo make_type(const std::string& name,
-                              const goos::Object& defining_form,
-                              const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::TYPE;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    return info;
-  }
-
-  static SymbolInfo make_constant(const std::string& name,
-                                  const goos::Object& defining_form,
-                                  const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::CONSTANT;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    return info;
-  }
-
-  static SymbolInfo make_macro(const std::string& name,
-                               const goos::Object& defining_form,
-                               const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::MACRO;
-    info.m_name = name;
-    info.m_def_form = defining_form;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    return info;
-  }
-
-  static SymbolInfo make_builtin(const std::string& name, const std::optional<Metadata> meta = {}) {
-    SymbolInfo info;
-    info.m_kind = Kind::LANGUAGE_BUILTIN;
-    info.m_name = name;
-    if (meta) {
-      info.m_meta = *meta;
-    }
-    return info;
-  }
-
-  static SymbolInfo make_method(const std::string& method_name,
-                                const std::vector<GoalArg> args,
-                                const MethodInfo& method_info,
-                                const goos::Object& defining_form) {
-    SymbolInfo info;
-    info.m_kind = Kind::METHOD;
-    info.m_name = method_name;
-    info.m_method_info = method_info;
-    info.m_def_form = defining_form;
-    info.m_meta.docstring =
-        info.m_method_info.docstring.has_value() ? info.m_method_info.docstring.value() : "";
-    info.m_args = args;
-    return info;
-  }
-
-  const std::string& name() const { return m_name; }
-  const MethodInfo& method_info() const { return m_method_info; }
-  Kind kind() const { return m_kind; }
-  const goos::Object& src_form() const { return m_def_form; }
-  const Metadata& meta() const { return m_meta; }
-  const std::vector<GoalArg>& args() const { return m_args; }
-
- private:
-  Kind m_kind = Kind::INVALID;
-  goos::Object m_def_form;
-  std::string m_name;
-  MethodInfo m_method_info;
-  Metadata m_meta;
-  std::vector<GoalArg> m_args;
-
-  std::string m_return_type;
-};
-
-/*!
- * A map of symbol info. It internally stores the info in a prefix tree so you can quickly get
- * a list of all symbols starting with a given prefix.
- */
-class SymbolInfoMap {
- public:
-  SymbolInfoMap() = default;
-  void add_global(const std::string& name,
-                  const goos::Object& defining_form,
-                  const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_global(name, defining_form, meta));
-  }
-
-  void add_fwd_dec(const std::string& name, const goos::Object& defining_form) {
-    m_map[name]->push_back(SymbolInfo::make_fwd_declared_sym(name, defining_form));
-  }
-
-  // The m_symbol_types container stores TypeSpecs -- this does have argument information but not
-  // the names, which is why they have to be explicitly provided
-  void add_function(const std::string& name,
-                    const std::vector<GoalArg> args,
-                    const goos::Object& defining_form,
-                    const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_function(name, args, defining_form, meta));
-  }
-
-  void add_type(const std::string& name,
-                const goos::Object& defining_form,
-                const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_type(name, defining_form, meta));
-  }
-
-  void add_constant(const std::string& name,
-                    const goos::Object& defining_form,
-                    const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_constant(name, defining_form, meta));
-  }
-
-  void add_macro(const std::string& name,
-                 const goos::Object& defining_form,
-                 const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_macro(name, defining_form, meta));
-  }
-
-  void add_builtin(const std::string& name, const std::optional<SymbolInfo::Metadata> meta = {}) {
-    m_map[name]->push_back(SymbolInfo::make_builtin(name, meta));
-  }
-
-  // The m_symbol_types container stores TypeSpecs -- this does have argument information but not
-  // the names, which is why they have to be explicitly provided
-  void add_method(const std::string& method_name,
-                  const std::vector<GoalArg> args,
-                  const MethodInfo& method_info,
-                  const goos::Object& defining_form) {
-    m_map[method_name]->push_back(
-        SymbolInfo::make_method(method_name, args, method_info, defining_form));
-  }
-
-  std::vector<SymbolInfo>* lookup_exact_name(const std::string& name) const {
-    return m_map.lookup(name);
-  }
-
-  std::set<std::string> lookup_symbols_starting_with(const std::string& prefix) const {
-    std::set<std::string> result;
-    auto lookup = m_map.lookup_prefix(prefix);
-    for (auto& x : lookup) {
-      for (auto& y : *x) {
-        result.insert(y.name());
-      }
-    }
-    return result;
-  }
-
-  int symbol_count() const { return m_map.size(); }
-
-  std::vector<SymbolInfo> get_all_symbols() const {
-    std::vector<SymbolInfo> info;
-    auto lookup = m_map.get_all_nodes();
-    for (auto& x : lookup) {
-      for (auto& y : *x) {
-        info.push_back(y);
-      }
-    }
-    return info;
-  }
-
- private:
-  Trie<std::vector<SymbolInfo>> m_map;
-};

goalc/compiler/compilation/CompilerControl.cpp

@@ -14,8 +14,8 @@
 
 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
-#include "goalc/compiler/SymbolInfo.h"
 #include "goalc/compiler/docs/DocTypes.h"
+#include "goalc/compiler/symbol_info.h"
 #include "goalc/data_compiler/dir_tpages.h"
 #include "goalc/data_compiler/game_count.h"
 #include "goalc/data_compiler/game_text_common.h"
@@ -358,35 +358,36 @@ Val* Compiler::compile_reload(const goos::Object& form, const goos::Object& rest
   return get_none();
 }
 
-std::string Compiler::make_symbol_info_description(const SymbolInfo& info) {
-  switch (info.kind()) {
-    case SymbolInfo::Kind::GLOBAL_VAR:
+std::string Compiler::make_symbol_info_description(
+    const std::shared_ptr<symbol_info::SymbolInfo> info) {
+  switch (info->m_kind) {
+    case symbol_info::Kind::GLOBAL_VAR:
       return fmt::format("[Global Variable] Type: {} Defined: {}",
-                         m_symbol_types.at(m_goos.intern_ptr(info.name())).print(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::LANGUAGE_BUILTIN:
-      return fmt::format("[Built-in Form] {}\n", info.name());
-    case SymbolInfo::Kind::METHOD:
+                         m_symbol_types.at(m_goos.intern_ptr(info->m_name)).print(),
+                         m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::LANGUAGE_BUILTIN:
+      return fmt::format("[Built-in Form] {}\n", info->m_name);
+    case symbol_info::Kind::METHOD:
       return fmt::format("[Method] Type: {} Method Name: {} Defined: {}",
-                         info.method_info().defined_in_type, info.name(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::TYPE:
-      return fmt::format("[Type] Name: {} Defined: {}", info.name(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::MACRO:
-      return fmt::format("[Macro] Name: {} Defined: {}", info.name(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::CONSTANT:
+                         info->m_method_info.defined_in_type, info->m_name,
+                         m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::TYPE:
+      return fmt::format("[Type] Name: {} Defined: {}", info->m_name,
+                         m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::MACRO:
+      return fmt::format("[Macro] Name: {} Defined: {}", info->m_name,
+                         m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::CONSTANT:
       return fmt::format(
-          "[Constant] Name: {} Value: {} Defined: {}", info.name(),
-          m_global_constants.at(m_goos.reader.symbolTable.intern(info.name().c_str())).print(),
-          m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::FUNCTION:
-      return fmt::format("[Function] Name: {} Defined: {}", info.name(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
-    case SymbolInfo::Kind::FWD_DECLARED_SYM:
-      return fmt::format("[Forward-Declared] Name: {} Defined: {}", info.name(),
-                         m_goos.reader.db.get_info_for(info.src_form()));
+          "[Constant] Name: {} Value: {} Defined: {}", info->m_name,
+          m_global_constants.at(m_goos.reader.symbolTable.intern(info->m_name.c_str())).print(),
+          m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::FUNCTION:
+      return fmt::format("[Function] Name: {} Defined: {}", info->m_name,
+                         m_goos.reader.db.get_info_for(info->m_def_form));
+    case symbol_info::Kind::FWD_DECLARED_SYM:
+      return fmt::format("[Forward-Declared] Name: {} Defined: {}", info->m_name,
+                         m_goos.reader.db.get_info_for(info->m_def_form));
     default:
       ASSERT(false);
       return {};
@@ -398,11 +399,11 @@ Val* Compiler::compile_get_info(const goos::Object& form, const goos::Object& re
   auto args = get_va(form, rest);
   va_check(form, args, {goos::ObjectType::SYMBOL}, {});
 
-  auto result = m_symbol_info.lookup_exact_name(args.unnamed.at(0).as_symbol().name_ptr);
-  if (!result) {
+  const auto result = m_symbol_info.lookup_exact_name(args.unnamed.at(0).as_symbol().name_ptr);
+  if (result.empty()) {
     lg::print("No results found.\n");
   } else {
-    for (auto& info : *result) {
+    for (const auto& info : result) {
       lg::print("{}", make_symbol_info_description(info));
     }
   }
@@ -437,9 +438,12 @@ replxx::Replxx::completions_t Compiler::find_symbols_or_object_file_by_prefix(
       completions.push_back(fmt::format("\"{}\")", match));
     }
   } else {
+    // TODO - GOAL's method calling syntax sucks for method name auto-completion
+    // maybe something that could be improved? Though it would be a radical departure from
+    // the syntax
     const auto [token, stripped_leading_paren] = m_repl->get_current_repl_token(context);
     // Otherwise, look for symbols
-    auto possible_forms = lookup_symbol_infos_starting_with(token);
+    auto possible_forms = lookup_symbol_names_starting_with(token);
 
     for (auto& x : possible_forms) {
       completions.push_back(stripped_leading_paren ? "(" + x : x);
@@ -456,7 +460,7 @@ replxx::Replxx::hints_t Compiler::find_hints_by_prefix(std::string const& contex
   (void)contextLen;
   (void)user_data;
   auto token = m_repl->get_current_repl_token(context);
-  auto possible_forms = lookup_symbol_infos_starting_with(token.first);
+  auto possible_forms = lookup_symbol_names_starting_with(token.first);
 
   replxx::Replxx::hints_t hints;
 
@@ -497,9 +501,8 @@ void Compiler::repl_coloring(
         curr_symbol.second.erase(0, 1);
         curr_symbol.first++;
       }
-      std::vector<SymbolInfo>* sym_match = lookup_exact_name_info(curr_symbol.second);
-      if (sym_match != nullptr && sym_match->size() == 1) {
-        SymbolInfo sym_info = sym_match->at(0);
+      const auto matching_symbols = lookup_exact_name_info(curr_symbol.second);
+      if (matching_symbols.size() == 1) {
         for (int pos = curr_symbol.first; pos <= int(i); pos++) {
           // TODO - currently just coloring all types brown/gold
           // - would be nice to have a different color for globals, functions, etc
@@ -541,7 +544,7 @@ void Compiler::repl_coloring(
     }
   }
 
-  // TODO - general syntax highlighting with regexes (quotes, symbols, etc)
+  // TODO - general syntax highlighting with AST
 }
 
 Val* Compiler::compile_autocomplete(const goos::Object& form, const goos::Object& rest, Env* env) {
@@ -550,7 +553,7 @@ Val* Compiler::compile_autocomplete(const goos::Object& form, const goos::Object
   va_check(form, args, {goos::ObjectType::SYMBOL}, {});
 
   Timer timer;
-  auto result = m_symbol_info.lookup_symbols_starting_with(args.unnamed.at(0).as_symbol().name_ptr);
+  auto result = m_symbol_info.lookup_names_starting_with(args.unnamed.at(0).as_symbol().name_ptr);
   auto time = timer.getMs();
 
   for (auto& x : result) {
@@ -581,25 +584,33 @@ Val* Compiler::compile_update_macro_metadata(const goos::Object& form,
 
   auto arg_spec = m_goos.parse_arg_spec(form, args.unnamed.at(2));
   m_macro_specs[name] = arg_spec;
-
-  SymbolInfo::Metadata sym_meta;
-  sym_meta.docstring = args.unnamed.at(1).as_string()->data;
-  m_symbol_info.add_macro(name, form, sym_meta);
+  m_symbol_info.add_macro(name, arg_spec, form, args.unnamed.at(1).as_string()->data);
   return get_none();
 }
 
-std::set<std::string> Compiler::lookup_symbol_infos_starting_with(const std::string& prefix) const {
+std::vector<std::shared_ptr<symbol_info::SymbolInfo>> Compiler::lookup_symbol_info_by_file(
+    const std::string& file_path) const {
+  return m_symbol_info.lookup_symbols_by_file(file_path);
+}
+
+std::vector<std::shared_ptr<symbol_info::SymbolInfo>> Compiler::lookup_symbol_info_by_prefix(
+    const std::string& prefix) const {
+  return m_symbol_info.lookup_symbols_starting_with(prefix);
+}
+
+std::set<std::string> Compiler::lookup_symbol_names_starting_with(const std::string& prefix) const {
   if (m_goos.reader.check_string_is_valid(prefix)) {
-    return m_symbol_info.lookup_symbols_starting_with(prefix);
+    return m_symbol_info.lookup_names_starting_with(prefix);
   }
   return {};
 }
 
-std::vector<SymbolInfo>* Compiler::lookup_exact_name_info(const std::string& name) const {
+std::vector<std::shared_ptr<symbol_info::SymbolInfo>> Compiler::lookup_exact_name_info(
+    const std::string& name) const {
   if (m_goos.reader.check_string_is_valid(name)) {
     return m_symbol_info.lookup_exact_name(name);
   } else {
-    return nullptr;
+    return {};
   }
 }
 
@@ -611,6 +622,176 @@ std::optional<TypeSpec> Compiler::lookup_typespec(const std::string& symbol_name
   return {};
 }
 
+std::tuple<std::unordered_map<std::string, Docs::SymbolDocumentation>,
+           std::unordered_map<std::string, Docs::FileDocumentation>>
+Compiler::generate_per_file_symbol_info() {
+  // TODO - remove this function, all required information has been consolidated into `SymbolInfo`
+  // it just has to be serialized in the same way, I will do it later
+  const auto symbols = m_symbol_info.get_all_symbols();
+
+  std::unordered_map<std::string, Docs::SymbolDocumentation> all_symbols;
+  std::unordered_map<std::string, Docs::FileDocumentation> file_docs;
+
+  lg::info("Processing {} symbols...", symbols.size());
+  int count = 0;
+  for (const auto& sym_info : symbols) {
+    count++;
+    if (count % 100 == 0 || count == (int)symbols.size()) {
+      lg::info("Processing [{}/{}] symbols...", count, symbols.size());
+    }
+    std::optional<Docs::DefinitionLocation> def_loc;
+    const auto& goos_info = m_goos.reader.db.get_short_info_for(sym_info->m_def_form);
+    if (goos_info) {
+      Docs::DefinitionLocation new_def_loc;
+      new_def_loc.filename = file_util::convert_to_unix_path_separators(file_util::split_path_at(
+          goos_info->filename, {"goal_src", version_to_game_name(m_version)}));
+      new_def_loc.line_idx = goos_info->line_idx_to_display;
+      new_def_loc.char_idx = goos_info->pos_in_line;
+      def_loc = new_def_loc;
+    }
+
+    Docs::SymbolDocumentation sym_doc;
+    sym_doc.name = sym_info->m_name;
+    sym_doc.description = sym_info->m_docstring;
+    sym_doc.kind = sym_info->m_kind;
+    sym_doc.def_location = def_loc;
+
+    if (all_symbols.count(sym_info->m_name) > 1) {
+      lg::error("A symbol was defined twice, how did this happen? {}", sym_info->m_name);
+    } else {
+      all_symbols.emplace(sym_info->m_name, sym_doc);
+    }
+
+    Docs::FileDocumentation file_doc;
+    std::string file_doc_key;
+    if (!goos_info) {
+      file_doc_key = "unknown";
+    } else {
+      file_doc_key = file_util::convert_to_unix_path_separators(
+          file_util::split_path_at(goos_info->filename, {"goal_src"}));
+    }
+
+    if (file_docs.count(file_doc_key) != 0) {
+      file_doc = file_docs.at(file_doc_key);
+    } else {
+      file_doc = Docs::FileDocumentation();
+    }
+
+    // TODO - states / enums / built-ins
+    if (sym_info->m_kind == symbol_info::Kind::GLOBAL_VAR ||
+        sym_info->m_kind == symbol_info::Kind::CONSTANT) {
+      Docs::VariableDocumentation var;
+      var.name = sym_info->m_name;
+      var.description = sym_info->m_docstring;
+      if (sym_info->m_kind == symbol_info::Kind::CONSTANT) {
+        var.type = "unknown";  // Unfortunately, constants are not properly typed
+      } else {
+        var.type = m_symbol_types.at(m_goos.intern_ptr(var.name)).base_type();
+      }
+      var.def_location = def_loc;
+      if (sym_info->m_kind == symbol_info::Kind::GLOBAL_VAR) {
+        file_doc.global_vars.push_back(var);
+      } else {
+        file_doc.constants.push_back(var);
+      }
+    } else if (sym_info->m_kind == symbol_info::Kind::FUNCTION) {
+      Docs::FunctionDocumentation func;
+      func.name = sym_info->m_name;
+      func.description = sym_info->m_docstring;
+      func.def_location = def_loc;
+      func.args = Docs::get_args_from_docstring(sym_info->m_args, func.description);
+      // The last arg in the typespec is the return type
+      const auto& func_type = m_symbol_types.at(m_goos.intern_ptr(func.name));
+      func.return_type = func_type.last_arg().base_type();
+      file_doc.functions.push_back(func);
+    } else if (sym_info->m_kind == symbol_info::Kind::TYPE) {
+      Docs::TypeDocumentation type;
+      type.name = sym_info->m_name;
+      type.description = sym_info->m_docstring;
+      type.def_location = def_loc;
+      const auto& type_info = m_ts.lookup_type(type.name);
+      type.parent_type = type_info->get_parent();
+      type.size = type_info->get_size_in_memory();
+      type.method_count = type_info->get_methods_defined_for_type().size();
+      if (m_ts.typecheck_and_throw(m_ts.make_typespec("structure"), m_ts.make_typespec(type.name),
+                                   "", false, false, false)) {
+        auto struct_info = dynamic_cast<StructureType*>(type_info);
+        for (const auto& field : struct_info->fields()) {
+          Docs::FieldDocumentation field_doc;
+          field_doc.name = field.name();
+          field_doc.description = "";
+          field_doc.type = field.type().base_type();
+          field_doc.is_array = field.is_array();
+          field_doc.is_inline = field.is_inline();
+          field_doc.is_dynamic = field.is_dynamic();
+          type.fields.push_back(field_doc);
+        }
+      }
+      for (const auto& method : type_info->get_methods_defined_for_type()) {
+        // Check to see if it's a state
+        if (m_ts.typecheck_and_throw(m_ts.make_typespec("state"), method.type, "", false, false,
+                                     false)) {
+          Docs::TypeStateDocumentation state_doc;
+          state_doc.id = method.id;
+          state_doc.is_virtual = true;
+          state_doc.name = method.name;
+          type.states.push_back(state_doc);
+        } else {
+          Docs::TypeMethodDocumentation method_doc;
+          method_doc.id = method.id;
+          method_doc.name = method.name;
+          method_doc.is_override = method.overrides_parent;
+          type.methods.push_back(method_doc);
+        }
+      }
+      for (const auto& [state_name, state_info] : type_info->get_states_declared_for_type()) {
+        Docs::TypeStateDocumentation state_doc;
+        state_doc.name = state_name;
+        state_doc.is_virtual = false;
+        type.states.push_back(state_doc);
+      }
+      file_doc.types.push_back(type);
+    } else if (sym_info->m_kind == symbol_info::Kind::MACRO) {
+      Docs::MacroDocumentation macro_doc;
+      macro_doc.name = sym_info->m_name;
+      macro_doc.description = sym_info->m_docstring;
+      macro_doc.def_location = def_loc;
+      const auto& arg_spec = m_macro_specs[macro_doc.name];
+      for (const auto& arg : arg_spec.unnamed) {
+        macro_doc.args.push_back(arg);
+      }
+      for (const auto& arg : arg_spec.named) {
+        std::optional<std::string> def_value;
+        if (arg.second.has_default) {
+          def_value = arg.second.default_value.print();
+        }
+        macro_doc.kwargs.push_back({arg.first, def_value});
+      }
+      if (!arg_spec.rest.empty()) {
+        macro_doc.variadic_arg = arg_spec.rest;
+      }
+      file_doc.macros.push_back(macro_doc);
+    } else if (sym_info->m_kind == symbol_info::Kind::METHOD) {
+      Docs::MethodDocumentation method_doc;
+      method_doc.name = sym_info->m_name;
+      method_doc.description = sym_info->m_docstring;
+      method_doc.def_location = def_loc;
+      const auto& method_info = sym_info->m_method_info;
+      method_doc.id = method_info.id;
+      method_doc.type = sym_info->m_method_info.defined_in_type;
+      method_doc.is_override = method_info.overrides_parent;
+      method_doc.args = Docs::get_args_from_docstring(sym_info->m_args, method_doc.description);
+      // The last arg in the typespec is the return type
+      const auto& method_type = method_info.type;
+      method_doc.return_type = method_type.last_arg().base_type();
+      method_doc.is_builtin = method_doc.id <= 9;
+      file_doc.methods.push_back(method_doc);
+    }
+    file_docs[file_doc_key] = file_doc;
+  }
+  return {all_symbols, file_docs};
+}
+
 Val* Compiler::compile_load_project(const goos::Object& form, const goos::Object& rest, Env*) {
   auto args = get_va(form, rest);
   va_check(form, args, {goos::ObjectType::STRING}, {});
@@ -660,168 +841,7 @@ Val* Compiler::compile_gen_docs(const goos::Object& form, const goos::Object& re
   const auto& doc_path = fs::path(args.unnamed.at(0).as_string()->data);
   lg::info("Saving docs to: {}", doc_path.string());
 
-  const auto symbols = m_symbol_info.get_all_symbols();
-
-  std::unordered_map<std::string, Docs::SymbolDocumentation> all_symbols;
-  std::unordered_map<std::string, Docs::FileDocumentation> file_docs;
-
-  lg::info("Processing {} symbols...", symbols.size());
-  int count = 0;
-  for (const auto& sym_info : symbols) {
-    count++;
-    if (count % 100 == 0 || count == (int)symbols.size()) {
-      lg::info("Processing [{}/{}] symbols...", count, symbols.size());
-    }
-    std::optional<Docs::DefinitionLocation> def_loc;
-    const auto& goos_info = m_goos.reader.db.get_short_info_for(sym_info.src_form());
-    if (goos_info) {
-      Docs::DefinitionLocation new_def_loc;
-      new_def_loc.filename = file_util::convert_to_unix_path_separators(file_util::split_path_at(
-          goos_info->filename, {"goal_src", version_to_game_name(m_version)}));
-      new_def_loc.line_idx = goos_info->line_idx_to_display;
-      new_def_loc.char_idx = goos_info->pos_in_line;
-      def_loc = new_def_loc;
-    }
-
-    Docs::SymbolDocumentation sym_doc;
-    sym_doc.name = sym_info.name();
-    sym_doc.description = sym_info.meta().docstring;
-    sym_doc.kind = sym_info.kind();
-    sym_doc.def_location = def_loc;
-
-    if (all_symbols.count(sym_info.name()) > 1) {
-      lg::error("A symbol was defined twice, how did this happen? {}", sym_info.name());
-    } else {
-      all_symbols.emplace(sym_info.name(), sym_doc);
-    }
-
-    Docs::FileDocumentation file_doc;
-    std::string file_doc_key;
-    if (!goos_info) {
-      file_doc_key = "unknown";
-    } else {
-      file_doc_key = file_util::convert_to_unix_path_separators(
-          file_util::split_path_at(goos_info->filename, {"goal_src"}));
-    }
-
-    if (file_docs.count(file_doc_key) != 0) {
-      file_doc = file_docs.at(file_doc_key);
-    } else {
-      file_doc = Docs::FileDocumentation();
-    }
-
-    // TODO - states / enums / built-ins
-    if (sym_info.kind() == SymbolInfo::Kind::GLOBAL_VAR ||
-        sym_info.kind() == SymbolInfo::Kind::CONSTANT) {
-      Docs::VariableDocumentation var;
-      var.name = sym_info.name();
-      var.description = sym_info.meta().docstring;
-      if (sym_info.kind() == SymbolInfo::Kind::CONSTANT) {
-        var.type = "unknown";  // Unfortunately, constants are not properly typed
-      } else {
-        var.type = m_symbol_types.at(m_goos.intern_ptr(var.name)).base_type();
-      }
-      var.def_location = def_loc;
-      if (sym_info.kind() == SymbolInfo::Kind::GLOBAL_VAR) {
-        file_doc.global_vars.push_back(var);
-      } else {
-        file_doc.constants.push_back(var);
-      }
-    } else if (sym_info.kind() == SymbolInfo::Kind::FUNCTION) {
-      Docs::FunctionDocumentation func;
-      func.name = sym_info.name();
-      func.description = sym_info.meta().docstring;
-      func.def_location = def_loc;
-      func.args = Docs::get_args_from_docstring(sym_info.args(), func.description);
-      // The last arg in the typespec is the return type
-      const auto& func_type = m_symbol_types.at(m_goos.intern_ptr(func.name));
-      func.return_type = func_type.last_arg().base_type();
-      file_doc.functions.push_back(func);
-    } else if (sym_info.kind() == SymbolInfo::Kind::TYPE) {
-      Docs::TypeDocumentation type;
-      type.name = sym_info.name();
-      type.description = sym_info.meta().docstring;
-      type.def_location = def_loc;
-      const auto& type_info = m_ts.lookup_type(type.name);
-      type.parent_type = type_info->get_parent();
-      type.size = type_info->get_size_in_memory();
-      type.method_count = type_info->get_methods_defined_for_type().size();
-      if (m_ts.typecheck_and_throw(m_ts.make_typespec("structure"), m_ts.make_typespec(type.name),
-                                   "", false, false, false)) {
-        auto struct_info = dynamic_cast<StructureType*>(type_info);
-        for (const auto& field : struct_info->fields()) {
-          Docs::FieldDocumentation field_doc;
-          field_doc.name = field.name();
-          field_doc.description = "";
-          field_doc.type = field.type().base_type();
-          field_doc.is_array = field.is_array();
-          field_doc.is_inline = field.is_inline();
-          field_doc.is_dynamic = field.is_dynamic();
-          type.fields.push_back(field_doc);
-        }
-      }
-      for (const auto& method : type_info->get_methods_defined_for_type()) {
-        // Check to see if it's a state
-        if (m_ts.typecheck_and_throw(m_ts.make_typespec("state"), method.type, "", false, false,
-                                     false)) {
-          Docs::TypeStateDocumentation state_doc;
-          state_doc.id = method.id;
-          state_doc.is_virtual = true;
-          state_doc.name = method.name;
-          type.states.push_back(state_doc);
-        } else {
-          Docs::TypeMethodDocumentation method_doc;
-          method_doc.id = method.id;
-          method_doc.name = method.name;
-          method_doc.is_override = method.overrides_parent;
-          type.methods.push_back(method_doc);
-        }
-      }
-      for (const auto& [state_name, state_info] : type_info->get_states_declared_for_type()) {
-        Docs::TypeStateDocumentation state_doc;
-        state_doc.name = state_name;
-        state_doc.is_virtual = false;
-        type.states.push_back(state_doc);
-      }
-      file_doc.types.push_back(type);
-    } else if (sym_info.kind() == SymbolInfo::Kind::MACRO) {
-      Docs::MacroDocumentation macro_doc;
-      macro_doc.name = sym_info.name();
-      macro_doc.description = sym_info.meta().docstring;
-      macro_doc.def_location = def_loc;
-      const auto& arg_spec = m_macro_specs[macro_doc.name];
-      for (const auto& arg : arg_spec.unnamed) {
-        macro_doc.args.push_back(arg);
-      }
-      for (const auto& arg : arg_spec.named) {
-        std::optional<std::string> def_value;
-        if (arg.second.has_default) {
-          def_value = arg.second.default_value.print();
-        }
-        macro_doc.kwargs.push_back({arg.first, def_value});
-      }
-      if (!arg_spec.rest.empty()) {
-        macro_doc.variadic_arg = arg_spec.rest;
-      }
-      file_doc.macros.push_back(macro_doc);
-    } else if (sym_info.kind() == SymbolInfo::Kind::METHOD) {
-      Docs::MethodDocumentation method_doc;
-      method_doc.name = sym_info.name();
-      method_doc.description = sym_info.meta().docstring;
-      method_doc.def_location = def_loc;
-      const auto& method_info = sym_info.method_info();
-      method_doc.id = method_info.id;
-      method_doc.type = sym_info.method_info().defined_in_type;
-      method_doc.is_override = method_info.overrides_parent;
-      method_doc.args = Docs::get_args_from_docstring(sym_info.args(), method_doc.description);
-      // The last arg in the typespec is the return type
-      const auto& method_type = method_info.type;
-      method_doc.return_type = method_type.last_arg().base_type();
-      method_doc.is_builtin = method_doc.id <= 9;
-      file_doc.methods.push_back(method_doc);
-    }
-    file_docs[file_doc_key] = file_doc;
-  }
+  const auto [all_symbols, file_docs] = generate_per_file_symbol_info();
 
   json symbol_map_data(all_symbols);
   file_util::write_text_file(

goalc/compiler/compilation/Define.cpp

@@ -11,10 +11,10 @@
  */
 Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest, Env* env) {
   auto args = get_va(form, rest);
-  SymbolInfo::Metadata sym_meta;
+  std::string docstring;
   // Grab the docstring (if it's there) and then rip it out so we can do the normal validation
   if (args.unnamed.size() == 3 && args.unnamed.at(1).is_string()) {
-    sym_meta.docstring = args.unnamed.at(1).as_string()->data;
+    docstring = args.unnamed.at(1).as_string()->data;
     args.unnamed.erase(args.unnamed.begin() + 1);
   }
 
@@ -35,6 +35,7 @@ Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest
   auto sym_val = fe->alloc_val<SymbolVal>(symbol_string(sym), m_ts.make_typespec("symbol"));
   auto compiled_val = compile_error_guard(val, env);
   auto as_lambda = dynamic_cast<LambdaVal*>(compiled_val);
+  auto in_gpr = compiled_val->to_gpr(form, fe);
   if (as_lambda) {
     // there are two cases in which we save a function body that is passed to a define:
     // 1. It generated code [so went through the compiler] and the allow_inline flag is set.
@@ -50,11 +51,14 @@ Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest
     }
     // Most defines come via macro invokations, we want the TRUE defining form location
     // if we can get it
+    // TODO - test the return value changes
     if (env->macro_expand_env()) {
-      m_symbol_info.add_function(symbol_string(sym), as_lambda->lambda.params,
-                                 env->macro_expand_env()->root_form(), sym_meta);
+      m_symbol_info.add_function(symbol_string(sym), in_gpr->type().last_arg().base_type(),
+                                 as_lambda->lambda.params, env->macro_expand_env()->root_form(),
+                                 docstring);
     } else {
-      m_symbol_info.add_function(symbol_string(sym), as_lambda->lambda.params, form, sym_meta);
+      m_symbol_info.add_function(symbol_string(sym), in_gpr->type().last_arg().base_type(),
+                                 as_lambda->lambda.params, form, docstring);
     }
   }
 
@@ -62,7 +66,6 @@ Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest
     throw_compiler_error(form, "Cannot define {} because it cannot be set.", sym_val->print());
   }
 
-  auto in_gpr = compiled_val->to_gpr(form, fe);
   auto existing_type = m_symbol_types.find(sym.as_symbol());
   if (existing_type == m_symbol_types.end()) {
     m_symbol_types[sym.as_symbol()] = in_gpr->type();
@@ -79,7 +82,7 @@ Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest
 
   if (!as_lambda) {
     // Don't double-add functions as globals
-    m_symbol_info.add_global(symbol_string(sym), form, sym_meta);
+    m_symbol_info.add_global(symbol_string(sym), in_gpr->type().base_type(), form, docstring);
   }
 
   env->emit(form, std::make_unique<IR_SetSymbolValue>(sym_val, in_gpr));

goalc/compiler/compilation/Macro.cpp

@@ -45,6 +45,7 @@ Val* Compiler::compile_goos_macro(const goos::Object& o,
       env->function_env()->alloc_env<MacroExpandEnv>(env, name.as_symbol(), macro->body, o);
   try {
     const auto& compile_result = compile(goos_result, compile_env_for_macro);
+    // TODO - is this critical (do the args and such change?)?
     m_macro_specs.emplace(macro->name, macro->args);
     return compile_result;
   } catch (CompilerException& ce) {
@@ -180,10 +181,9 @@ Val* Compiler::compile_define_constant(const goos::Object& form,
   rest = &pair_cdr(*rest);
 
   // check for potential docstring
-  SymbolInfo::Metadata sym_meta;
+  std::string docstring = "";
   if (rest->is_pair() && pair_car(*rest).is_string() && !pair_cdr(*rest).is_empty_list()) {
-    std::string docstring = pair_car(*rest).as_string()->data;
-    sym_meta.docstring = docstring;
+    docstring = pair_car(*rest).as_string()->data;
     rest = &pair_cdr(*rest);
   }
 
@@ -218,7 +218,7 @@ Val* Compiler::compile_define_constant(const goos::Object& form,
 
   // TODO - eventually, it'd be nice if global constants were properly typed
   // and this information was propagated
-  m_symbol_info.add_constant(sym.name_ptr, form, sym_meta);
+  m_symbol_info.add_constant(sym.name_ptr, form, docstring);
 
   return get_none();
 }

goalc/compiler/compilation/Type.cpp

@@ -441,7 +441,7 @@ Val* Compiler::compile_deftype(const goos::Object& form, const goos::Object& res
     }
   }
 
-  m_symbol_info.add_type(result.type.base_type(), form);
+  m_symbol_info.add_type(result.type.base_type(), result.type_info, form);
 
   // return none, making the value of (deftype..) unusable
   return get_none();

goalc/compiler/docs/DocTypes.cpp

@@ -123,14 +123,15 @@ void to_json(json& j, const MacroDocumentation& obj) {
   }
 }
 
-std::vector<ArgumentDocumentation> get_args_from_docstring(std::vector<GoalArg> args,
-                                                           std::string docstring) {
+std::vector<ArgumentDocumentation> get_args_from_docstring(
+    std::vector<symbol_info::ArgumentInfo> args,
+    std::string docstring) {
   std::vector<ArgumentDocumentation> arg_docs;
   for (const auto& arg : args) {
     ArgumentDocumentation arg_doc;
     arg_doc.name = arg.name;
     // TODO - is this type reliable?
-    arg_doc.type = arg.type.base_type();
+    arg_doc.type = arg.type;
     arg_docs.push_back(arg_doc);
   }
   if (docstring.empty()) {

goalc/compiler/docs/DocTypes.h

@@ -3,12 +3,15 @@
 #include <optional>
 #include <string>
 
-#include "goalc/compiler/SymbolInfo.h"
+#include "goalc/compiler/symbol_info.h"
 
 #include "third-party/json.hpp"
 
 using json = nlohmann::json;
 
+// TODO - deprecate this file in factor of the now consolidated `SymbolInfo`
+// which now contains comprehensive info on all forms of symbols
+
 namespace Docs {
 
 struct DefinitionLocation {
@@ -69,6 +72,7 @@ struct FieldDocumentation {
 void to_json(json& j, const FieldDocumentation& obj);
 
 struct TypeMethodDocumentation {
+  // TODO - relevant?
   int id;
   std::string name;
   bool is_override = false;
@@ -89,6 +93,7 @@ struct TypeDocumentation {
   std::optional<DefinitionLocation> def_location;
   int size;
   std::vector<FieldDocumentation> fields = {};
+  // TODO - who cares, remove this probably
   int method_count;
   std::vector<TypeMethodDocumentation> methods = {};
   std::vector<TypeStateDocumentation> states = {};
@@ -96,10 +101,14 @@ struct TypeDocumentation {
 void to_json(json& j, const TypeDocumentation& obj);
 
 struct MethodDocumentation {
+  // TODO - relevant?
   int id;
   bool is_builtin;
   std::string name;
   std::string description = "";
+  // TODO - this is `object` sometimes, for example `(defmethod print ((this light))`
+  // i believe this is because we always grab the first symbol, but of course, overridden methods
+  // dont work like that so things are likely working as intended
   std::string type;
   std::optional<DefinitionLocation> def_location;
   // TODO - need to track function calls to determine this, obviously cant be determined from just
@@ -139,13 +148,14 @@ struct SymbolDocumentation {
   // TODO - forward declared symbols
   std::string name;
   std::string description = "";
-  SymbolInfo::Kind kind;
+  symbol_info::Kind kind;
   std::optional<DefinitionLocation> def_location = {};
   std::vector<DefinitionLocation> forward_declared_in = {};
 };
 void to_json(json& j, const SymbolDocumentation& obj);
 
-std::vector<ArgumentDocumentation> get_args_from_docstring(std::vector<GoalArg> args,
-                                                           std::string docstring);
+std::vector<ArgumentDocumentation> get_args_from_docstring(
+    std::vector<symbol_info::ArgumentInfo> args,
+    std::string docstring);
 
 }  // namespace Docs

goalc/compiler/symbol_info.cpp

@@ -0,0 +1,318 @@
+#include "symbol_info.h"
+
+#include "common/log/log.h"
+#include "common/util/FileUtil.h"
+#include "common/util/string_util.h"
+
+namespace symbol_info {
+void SymbolInfo::update_args_from_docstring() {
+  if (m_docstring.empty()) {
+    return;
+  }
+  auto lines = str_util::split(m_docstring);
+  for (const auto& line : lines) {
+    const auto trimmed_line = str_util::ltrim(line);
+    if (str_util::starts_with(trimmed_line, "@param")) {
+      // Get the info from the @param line
+      const auto& tokens =
+          str_util::regex_get_capture_groups(trimmed_line, "(@param.)\\s?([^\\s]*)\\s(.*)");
+      if (tokens.size() != 3) {
+        lg::warn("invalid docstring line - {}, skipping", trimmed_line);
+        continue;
+      }
+      const auto& param_type = str_util::trim(tokens[0]);
+      const auto& param_name = str_util::trim(tokens[1]);
+      const auto& param_description = str_util::trim(tokens[2]);
+      // Locate the appropriate arg based on the name
+      for (auto& arg : m_args) {
+        if (arg.name == param_name) {
+          arg.description = param_description;
+          if (param_type == "@param") {
+            // a normal arg, nothing fancy
+          } else if (param_type == "@param_") {
+            // it's unused
+            arg.is_unused = true;
+          } else if (param_type == "@param!") {
+            // the params value is mutated within the function body
+            arg.is_mutated = true;
+          } else if (param_type == "@param?") {
+            // the param is optional -- there are checks to see if it was provided or not so its
+            // safe to pass "nothing"
+            arg.is_optional = true;
+          }
+        }
+      }
+    }
+  }
+}
+
+void SymbolInfo::set_definition_location(const goos::TextDb* textdb) {
+  const auto& goos_info = textdb->get_short_info_for(m_def_form);
+  if (goos_info) {
+    DefinitionLocation def_loc;
+    def_loc.line_idx = goos_info->line_idx_to_display;
+    def_loc.char_idx = goos_info->pos_in_line;
+    def_loc.file_path = file_util::convert_to_unix_path_separators(goos_info->filename);
+    m_def_location = def_loc;
+  }
+}
+
+void SymbolInfoMap::add_symbol_to_file_index(const std::string& file_path,
+                                             std::shared_ptr<SymbolInfo> symbol) {
+  if (m_file_symbol_index.find(file_path) == m_file_symbol_index.end()) {
+    m_file_symbol_index[file_path] = {};
+  }
+  m_file_symbol_index[file_path].push_back(symbol);
+}
+
+void SymbolInfoMap::add_global(const std::string& name,
+                               const std::string& type,
+                               const goos::Object& defining_form,
+                               const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::GLOBAL_VAR,
+      .m_name = name,
+      .m_def_form = defining_form,
+      .m_docstring = docstring,
+      .m_type = type,
+  };
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_fwd_dec(const std::string& name, const goos::Object& defining_form) {
+  SymbolInfo info = {.m_kind = Kind::FWD_DECLARED_SYM, .m_name = name, .m_def_form = defining_form};
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_function(const std::string& name,
+                                 const std::string& return_type,
+                                 const std::vector<GoalArg>& args,
+                                 const goos::Object& defining_form,
+                                 const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::FUNCTION,
+      .m_name = name,
+      .m_def_form = defining_form,
+      .m_docstring = docstring,
+      .m_return_type = return_type,
+  };
+  for (const auto& goal_arg : args) {
+    ArgumentInfo arg_info;
+    arg_info.name = goal_arg.name;
+    arg_info.type_spec = goal_arg.type;
+    // TODO - is this reliable?
+    arg_info.type = goal_arg.type.base_type();
+    info.m_args.push_back(arg_info);
+  }
+  info.update_args_from_docstring();
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_type(const std::string& name,
+                             Type* type_info,
+                             const goos::Object& defining_form,
+                             const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::TYPE,
+      .m_name = name,
+      .m_def_form = defining_form,
+      .m_docstring = docstring,
+      .m_parent_type = type_info->get_parent(),
+      .m_type_size = type_info->get_size_in_memory(),
+  };
+  // Only structure types have fields
+  auto as_structure_type = dynamic_cast<StructureType*>(type_info);
+  if (as_structure_type) {  // generate the inspect method
+    for (const auto& field : as_structure_type->fields()) {
+      // TODO - field docstrings arent a thing, yet!
+      FieldInfo field_info = {
+          .name = field.name(),
+          .description = "",
+          .type = field.type().base_type(),
+          .is_array = field.is_array(),
+          .is_dynamic = field.is_dynamic(),
+          .is_inline = field.is_inline(),
+      };
+      info.m_type_fields.push_back(field_info);
+    }
+  }
+  for (const auto& method : type_info->get_methods_defined_for_type()) {
+    if (method.type.base_type() == "state") {
+      TypeStateInfo state_info = {
+          .name = method.name,
+          .is_virtual = true,
+          .id = method.id,
+      };
+      info.m_type_states.push_back(state_info);
+    } else {
+      TypeMethodInfo method_info = {
+          .id = method.id,
+          .name = method.name,
+          .is_override = method.overrides_parent,
+      };
+      info.m_type_methods.push_back(method_info);
+    }
+  }
+  for (const auto& [state_name, state_info] : type_info->get_states_declared_for_type()) {
+    TypeStateInfo type_state_info = {
+        .name = state_name,
+        .is_virtual = false,
+    };
+    info.m_type_states.push_back(type_state_info);
+  }
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_constant(const std::string& name,
+                                 const goos::Object& defining_form,
+                                 const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::CONSTANT,
+      .m_name = name,
+      .m_def_form = defining_form,
+      .m_docstring = docstring,
+      // TODO - unfortunately, constants are not properly typed
+      .m_type = "unknown",
+  };
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_macro(const std::string& name,
+                              const goos::ArgumentSpec arg_spec,
+                              const goos::Object& defining_form,
+                              const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::MACRO,
+      .m_name = name,
+      .m_def_form = defining_form,
+      .m_docstring = docstring,
+  };
+  for (const auto& arg : arg_spec.unnamed) {
+    info.m_macro_args.push_back(arg);
+  }
+  for (const auto& arg : arg_spec.named) {
+    std::optional<std::string> def_value;
+    if (arg.second.has_default) {
+      def_value = arg.second.default_value.print();
+    }
+    info.m_macro_kwargs.push_back({arg.first, def_value});
+  }
+  if (!arg_spec.rest.empty()) {
+    info.m_variadic_arg = arg_spec.rest;
+  }
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+void SymbolInfoMap::add_builtin(const std::string& name, const std::string& docstring) {
+  SymbolInfo info = {
+      .m_kind = Kind::LANGUAGE_BUILTIN,
+      .m_name = name,
+      .m_docstring = docstring,
+  };
+  info.set_definition_location(m_textdb);
+  m_symbol_map.insert(name, info);
+}
+
+void SymbolInfoMap::add_method(const std::string& method_name,
+                               const std::vector<GoalArg>& args,
+                               const MethodInfo& method_info,
+                               const goos::Object& defining_form) {
+  SymbolInfo info = {
+      .m_kind = Kind::METHOD,
+      .m_name = method_name,
+      .m_method_info = method_info,
+      .m_method_builtin = method_info.id <= 9,
+  };
+  if (method_info.docstring) {
+    info.m_docstring = method_info.docstring.value();
+  }
+  for (const auto& goal_arg : args) {
+    ArgumentInfo arg_info;
+    arg_info.name = goal_arg.name;
+    arg_info.type_spec = goal_arg.type;
+    // TODO - is this reliable?
+    arg_info.type = goal_arg.type.base_type();
+    info.m_args.push_back(arg_info);
+  }
+  info.update_args_from_docstring();
+  info.set_definition_location(m_textdb);
+  const auto inserted_symbol = m_symbol_map.insert(method_name, info);
+  if (info.m_def_location) {
+    add_symbol_to_file_index(info.m_def_location->file_path, inserted_symbol);
+  }
+}
+
+std::vector<std::shared_ptr<SymbolInfo>> SymbolInfoMap::lookup_symbols_by_file(
+    const std::string& file_path) const {
+  if (m_file_symbol_index.find(file_path) != m_file_symbol_index.end()) {
+    return m_file_symbol_index.at(file_path);
+  }
+  return {};
+}
+
+std::vector<std::shared_ptr<SymbolInfo>> SymbolInfoMap::lookup_exact_name(
+    const std::string& name) const {
+  return m_symbol_map.retrieve_with_exact(name);
+}
+
+std::vector<std::shared_ptr<SymbolInfo>> SymbolInfoMap::lookup_symbols_starting_with(
+    const std::string& prefix) const {
+  std::vector<std::shared_ptr<SymbolInfo>> symbols;
+  const auto lookup = m_symbol_map.retrieve_with_prefix(prefix);
+  for (const auto& result : lookup) {
+    symbols.push_back(result);
+  }
+  return symbols;
+}
+
+std::set<std::string> SymbolInfoMap::lookup_names_starting_with(const std::string& prefix) const {
+  std::set<std::string> names;
+  const auto lookup = m_symbol_map.retrieve_with_prefix(prefix);
+  for (const auto& result : lookup) {
+    names.insert(result->m_name);
+  }
+  return names;
+}
+
+int SymbolInfoMap::symbol_count() const {
+  return m_symbol_map.size();
+}
+
+std::vector<std::shared_ptr<SymbolInfo>> SymbolInfoMap::get_all_symbols() const {
+  return m_symbol_map.get_all_elements();
+}
+
+void SymbolInfoMap::evict_symbols_using_file_index(const std::string& file_path) {
+  const auto standardized_path = file_util::convert_to_unix_path_separators(file_path);
+  if (m_file_symbol_index.find(standardized_path) != m_file_symbol_index.end()) {
+    for (const auto& symbol : m_file_symbol_index.at(standardized_path)) {
+      m_symbol_map.remove(symbol);
+    }
+    m_file_symbol_index.erase(standardized_path);
+  }
+}
+}  // namespace symbol_info

goalc/compiler/symbol_info.h

@@ -0,0 +1,173 @@
+#pragma once
+
+#include <set>
+#include <string>
+#include <vector>
+
+#include "common/goos/Object.h"
+#include "common/util/Assert.h"
+#include "common/util/trie_map.h"
+
+#include "goalc/compiler/Val.h"
+
+namespace symbol_info {
+
+// TODO - states
+// TODO - enums
+enum class Kind {
+  GLOBAL_VAR,
+  FWD_DECLARED_SYM,
+  FUNCTION,
+  TYPE,
+  CONSTANT,
+  MACRO,
+  LANGUAGE_BUILTIN,
+  METHOD,
+  INVALID
+};
+
+struct DefinitionLocation {
+  std::string file_path;
+  uint32_t line_idx;
+  uint32_t char_idx;
+  // TODO - store the extent of the symbol definition as well
+};
+
+struct ArgumentInfo {
+  std::string name;
+  // TODO - anything use this?
+  TypeSpec type_spec;
+  std::string type;
+  std::string description = "";
+  // !var
+  bool is_mutated = false;
+  // ?var
+  bool is_optional = false;
+  // _var
+  bool is_unused = false;
+};
+
+struct FieldInfo {
+  std::string name;
+  // TODO - DefinitionLocation def_location;
+  std::string description = "";
+  std::string type;
+  // ?? TODO
+  bool is_array = false;
+  // :dynamic
+  bool is_dynamic = false;
+  // :inline
+  bool is_inline = false;
+};
+
+struct TypeMethodInfo {
+  int id;  // TODO - is this even relevant anymore?
+  std::string name;
+  // TODO - DefinitionLocation def_location;
+  bool is_override = false;
+};
+
+struct TypeStateInfo {
+  std::string name;
+  // TODO - DefinitionLocation def_location;
+  bool is_virtual = false;
+  std::optional<int> id;  // TODO - is this even relevant anymore?
+};
+
+/*!
+ * Info about a single symbol, representing one of:
+ *  - Global variable
+ *  - Global function
+ *  - Type
+ *  - Constant
+ *  - Macro
+ *  - Builtin keyword of the OpenGOAL language
+ */
+struct SymbolInfo {
+  Kind m_kind = Kind::INVALID;
+  std::string m_name;
+  goos::Object m_def_form;
+  std::optional<DefinitionLocation> m_def_location;
+  std::string m_docstring = "";
+  std::string m_type = "";
+  // Method or Function Related
+  std::vector<ArgumentInfo> m_args = {};
+  std::string m_return_type = "";
+  // Method Related
+  MethodInfo m_method_info;
+  bool m_method_builtin = false;
+  // Type Related
+  std::string m_parent_type = "";
+  int m_type_size = -1;
+  // NOTE - removed method count...seems unnecessary?
+  std::vector<FieldInfo> m_type_fields = {};
+  std::vector<TypeMethodInfo> m_type_methods = {};
+  std::vector<TypeStateInfo> m_type_states = {};
+  // Macro Related
+  std::vector<std::string> m_macro_args = {};
+  std::vector<std::pair<std::string, std::optional<std::string>>> m_macro_kwargs = {};
+  std::optional<std::string> m_variadic_arg = {};
+  // TODO: need to track references for this, this is a TODO for LSP work
+  // bool is_unused = false;
+
+  void update_args_from_docstring();
+  void set_definition_location(const goos::TextDb* textdb);
+};
+
+/*!
+ * A map of symbol info. It internally stores the info in a prefix tree so you can quickly get
+ * a list of all symbols starting with a given prefix.
+ */
+class SymbolInfoMap {
+  goos::TextDb* m_textdb;
+  TrieMap<SymbolInfo> m_symbol_map;
+  // Indexes references to symbols by the file they are defined within
+  // This allows us to not only efficiently retrieve symbols by file, but also allows us to
+  // cleanup symbols when files are re-compiled.
+  std::unordered_map<std::string, std::vector<std::shared_ptr<SymbolInfo>>> m_file_symbol_index;
+
+  void add_symbol_to_file_index(const std::string& file_path, std::shared_ptr<SymbolInfo> symbol);
+
+ public:
+  SymbolInfoMap(goos::TextDb* textdb) : m_textdb(textdb) {}
+  void add_global(const std::string& name,
+                  const std::string& type,
+                  const goos::Object& defining_form,
+                  const std::string& docstring = "");
+  void add_fwd_dec(const std::string& name, const goos::Object& defining_form);
+  void add_function(const std::string& name,
+                    const std::string& return_type,
+                    const std::vector<GoalArg>& args,
+                    const goos::Object& defining_form,
+                    const std::string& docstring = "");
+  void add_type(const std::string& name,
+                Type* type_info,
+                const goos::Object& defining_form,
+                const std::string& docstring = "");
+  void add_constant(const std::string& name,
+                    const goos::Object& defining_form,
+                    const std::string& docstring = "");
+  void add_macro(const std::string& name,
+                 const goos::ArgumentSpec arg_spec,
+                 const goos::Object& defining_form,
+                 const std::string& docstring = "");
+  void add_builtin(const std::string& name, const std::string& docstring = "");
+  void add_method(const std::string& method_name,
+                  const std::vector<GoalArg>& args,
+                  const MethodInfo& method_info,
+                  const goos::Object& defining_form);
+  std::vector<std::shared_ptr<SymbolInfo>> lookup_symbols_by_file(
+      const std::string& file_path) const;
+  std::vector<std::shared_ptr<SymbolInfo>> lookup_exact_name(const std::string& name) const;
+  std::vector<std::shared_ptr<SymbolInfo>> lookup_symbols_starting_with(
+      const std::string& prefix) const;
+  std::set<std::string> lookup_names_starting_with(const std::string& prefix) const;
+  int symbol_count() const;
+  std::vector<std::shared_ptr<SymbolInfo>> get_all_symbols() const;
+  // Uses the per-file index to find and evict symbols globally
+  // This should be done before re-compiling a file, symbols will be re-added to the DB if they are
+  // found again
+  void evict_symbols_using_file_index(const std::string& file_path);
+};
+
+}  // namespace symbol_info

goalc/main.cpp

@@ -8,6 +8,7 @@
 #include "common/util/diff.h"
 #include "common/util/string_util.h"
 #include "common/util/term_util.h"
+#include "common/util/trie_map.h"
 #include "common/util/unicode_util.h"
 #include "common/versions/versions.h"
 

lsp/CMakeLists.txt

@@ -1,5 +1,14 @@
 add_executable(lsp
   handlers/lsp_router.cpp
+  handlers/initialize.cpp
+  handlers/text_document/completion.cpp
+  handlers/text_document/document_color.cpp
+  handlers/text_document/document_symbol.cpp
+  handlers/text_document/document_synchronization.cpp
+  handlers/text_document/formatting.cpp
+  handlers/text_document/go_to.cpp
+  handlers/text_document/hover.cpp
+  handlers/text_document/type_hierarchy.cpp
   main.cpp
   protocol/common_types.cpp
   protocol/completion.cpp
@@ -10,10 +19,12 @@ add_executable(lsp
   protocol/formatting.cpp
   protocol/hover.cpp
   protocol/progress_report.cpp
+  protocol/type_hierarchy.cpp
   state/data/mips_instruction.cpp
   state/lsp_requester.cpp
   state/workspace.cpp
-  transport/stdio.cpp)
+  transport/stdio.cpp
+  lsp_util.cpp)
 
 target_compile_definitions(lsp PRIVATE -DJSON_DIAGNOSTICS=1)
 

lsp/handlers/initialize.cpp

@@ -1,19 +1,11 @@
-// TODO - convert this to a proper class
+#include "initialize.h"
 
-#include "third-party/json.hpp"
-
-using json = nlohmann::json;
-
-class InitializeResult {
- public:
-  InitializeResult(){};
-  json to_json() { return result; }
-
- private:
+namespace lsp_handlers {
+std::optional<json> initialize(Workspace& workspace, int id, json params) {
   json text_document_sync{
       {"openClose", true},
       {"change", 1},  // Full sync
-      {"willSave", false},
+      {"willSave", true},
       {"willSaveWaitUntil", false},
       {"save", {{"includeText", false}}},
   };
@@ -55,6 +47,9 @@ class InitializeResult {
                    {"renameProvider", false},
                    {"documentLinkProvider", document_link_provider},
                    {"executeCommandProvider", execute_command_provider},
+                   {"typeHierarchyProvider", true},
                    {"experimental", {}},
                }}};
-};
+  return result;
+}
+}  // namespace lsp_handlers

lsp/handlers/initialize.h

@@ -1,14 +1,10 @@
 #pragma once
 
 #include "common/log/log.h"
+#include "common/util/json_util.h"
 
-#include "lsp/protocol/initialize_result.h"
+#include "lsp/state/workspace.h"
 
-#include "third-party/json.hpp"
-
-using json = nlohmann::json;
-
-std::optional<json> initialize_handler(Workspace& /*workspace*/, int /*id*/, json /*params*/) {
-  InitializeResult result;
-  return result.to_json();
+namespace lsp_handlers {
+std::optional<json> initialize(Workspace& workspace, int id, json params);
 }

lsp/handlers/lsp_router.cpp

@@ -3,6 +3,7 @@
 #include "common/log/log.h"
 
 #include "lsp/handlers/initialize.h"
+#include "lsp/handlers/text_document/type_hierarchy.h"
 #include "lsp/protocol/error_codes.h"
 #include "text_document/completion.h"
 #include "text_document/document_color.h"
@@ -14,6 +15,14 @@
 
 #include "fmt/core.h"
 
+json error_resp(ErrorCodes error_code, const std::string& error_message) {
+  json error{
+      {"code", static_cast<int>(error_code)},
+      {"message", error_message},
+  };
+  return json{{"error", error}};
+}
+
 LSPRoute::LSPRoute() : m_route_type(LSPRouteType::NOOP) {}
 
 LSPRoute::LSPRoute(std::function<void(Workspace&, json)> notification_handler)
@@ -29,41 +38,43 @@ LSPRoute::LSPRoute(std::function<std::optional<json>(Workspace&, int, json)> req
     : m_route_type(LSPRouteType::REQUEST_RESPONSE), m_request_handler(request_handler) {}
 
 void LSPRouter::init_routes() {
+  m_routes["exit"] = LSPRoute([](Workspace& /*workspace*/, nlohmann::json /*params*/) {
+    lg::info("Shutting down LSP due to explicit request");
+    exit(0);
+  });
   m_routes["shutdown"] = LSPRoute(
       [](Workspace& /*workspace*/, int /*id*/, nlohmann::json /*params*/) -> std::optional<json> {
-        lg::info("Shutting down LSP due to explicit request");
-        exit(0);
+        lg::info("Received shutdown request");
+        return error_resp(ErrorCodes::UnknownErrorCode, "Problem occurred while existing");
       });
-  m_routes["initialize"] = LSPRoute(initialize_handler);
+  m_routes["initialize"] = LSPRoute(lsp_handlers::initialize);
   m_routes["initialize"].m_generic_post_action = [](Workspace& workspace) {
     workspace.set_initialized(true);
   };
   m_routes["initialized"] = LSPRoute();
-  m_routes["textDocument/documentSymbol"] = LSPRoute(document_symbols_handler);
-  m_routes["textDocument/didOpen"] = LSPRoute(did_open_handler, did_open_push_diagnostics);
-  m_routes["textDocument/didChange"] = LSPRoute(did_change_handler, did_change_push_diagnostics);
-  m_routes["textDocument/didClose"] = LSPRoute(did_close_handler);
-  m_routes["textDocument/hover"] = LSPRoute(hover_handler);
-  m_routes["textDocument/definition"] = LSPRoute(go_to_definition_handler);
-  m_routes["textDocument/completion"] = LSPRoute(get_completions_handler);
-  m_routes["textDocument/documentColor"] = LSPRoute(document_color_handler);
-  m_routes["textDocument/formatting"] = LSPRoute(formatting_handler);
+  m_routes["textDocument/documentSymbol"] = LSPRoute(lsp_handlers::document_symbols);
+  m_routes["textDocument/didOpen"] =
+      LSPRoute(lsp_handlers::did_open, lsp_handlers::did_open_push_diagnostics);
+  m_routes["textDocument/didChange"] =
+      LSPRoute(lsp_handlers::did_change, lsp_handlers::did_change_push_diagnostics);
+  m_routes["textDocument/didClose"] = LSPRoute(lsp_handlers::did_close);
+  m_routes["textDocument/willSave"] = LSPRoute(lsp_handlers::will_save);
+  m_routes["textDocument/hover"] = LSPRoute(lsp_handlers::hover);
+  m_routes["textDocument/definition"] = LSPRoute(lsp_handlers::go_to_definition);
+  m_routes["textDocument/completion"] = LSPRoute(lsp_handlers::get_completions);
+  m_routes["textDocument/documentColor"] = LSPRoute(lsp_handlers::document_color);
+  m_routes["textDocument/formatting"] = LSPRoute(lsp_handlers::formatting);
+  m_routes["textDocument/prepareTypeHierarchy"] = LSPRoute(lsp_handlers::prepare_type_hierarchy);
+  m_routes["typeHierarchy/supertypes"] = LSPRoute(lsp_handlers::supertypes_type_hierarchy);
+  m_routes["typeHierarchy/subtypes"] = LSPRoute(lsp_handlers::subtypes_type_hierarchy);
   // TODO - m_routes["textDocument/signatureHelp"] = LSPRoute(get_completions_handler);
-  // Not Yet Supported Routes, noops
+  // Not Supported Routes, noops
   m_routes["$/cancelRequest"] = LSPRoute();
   m_routes["textDocument/documentLink"] = LSPRoute();
   m_routes["textDocument/codeLens"] = LSPRoute();
   m_routes["textDocument/colorPresentation"] = LSPRoute();
 }
 
-json error_resp(ErrorCodes error_code, const std::string& error_message) {
-  json error{
-      {"code", static_cast<int>(error_code)},
-      {"message", error_message},
-  };
-  return json{{"error", error}};
-}
-
 std::string LSPRouter::make_response(const json& result) {
   json content = result;
   content["jsonrpc"] = "2.0";

lsp/handlers/text_document/completion.cpp

@@ -0,0 +1,58 @@
+#include "completion.h"
+
+namespace lsp_handlers {
+
+std::unordered_map<symbol_info::Kind, LSPSpec::CompletionItemKind> completion_item_kind_map = {
+    {symbol_info::Kind::CONSTANT, LSPSpec::CompletionItemKind::Constant},
+    {symbol_info::Kind::FUNCTION, LSPSpec::CompletionItemKind::Function},
+    {symbol_info::Kind::FWD_DECLARED_SYM, LSPSpec::CompletionItemKind::Reference},
+    {symbol_info::Kind::GLOBAL_VAR, LSPSpec::CompletionItemKind::Variable},
+    {symbol_info::Kind::INVALID, LSPSpec::CompletionItemKind::Text},
+    {symbol_info::Kind::LANGUAGE_BUILTIN, LSPSpec::CompletionItemKind::Function},
+    {symbol_info::Kind::MACRO, LSPSpec::CompletionItemKind::Operator},
+    {symbol_info::Kind::METHOD, LSPSpec::CompletionItemKind::Method},
+    {symbol_info::Kind::TYPE, LSPSpec::CompletionItemKind::Class},
+};
+
+std::optional<json> get_completions(Workspace& workspace, int /*id*/, json params) {
+  auto converted_params = params.get<LSPSpec::CompletionParams>();
+  const auto file_type = workspace.determine_filetype_from_uri(converted_params.textDocument.m_uri);
+
+  if (file_type != Workspace::FileType::OpenGOAL) {
+    return nullptr;
+  }
+  auto maybe_tracked_file = workspace.get_tracked_og_file(converted_params.textDocument.m_uri);
+  if (!maybe_tracked_file) {
+    return nullptr;
+  }
+  std::vector<LSPSpec::CompletionItem> items;
+  const auto& tracked_file = maybe_tracked_file.value().get();
+  // The cursor position in the context of completions is always 1 character ahead of the text, we
+  // move it back 1 spot so we can actually detect what the user has typed so far
+  LSPSpec::Position new_position = converted_params.position;
+  if (new_position.m_character > 0) {
+    new_position.m_character--;
+  }
+  const auto symbol = tracked_file.get_symbol_at_position(new_position);
+  if (!symbol) {
+    lg::debug("get_completions - no symbol to work from");
+  } else {
+    const auto matching_symbols =
+        workspace.get_symbols_starting_with(tracked_file.m_game_version, symbol.value());
+    lg::debug("get_completions - found {} symbols", matching_symbols.size());
+
+    for (const auto& symbol : matching_symbols) {
+      LSPSpec::CompletionItem item;
+      item.label = symbol->m_name;
+      item.kind = completion_item_kind_map.at(symbol->m_kind);
+      // TODO - flesh out this more fully when auto-complete with non-globals works as well
+      items.push_back(item);
+    }
+  }
+  LSPSpec::CompletionList list_result;
+  list_result.isIncomplete = false;  // we want further typing to re-evaluate the list
+  list_result.items = items;
+  return list_result;
+}
+
+}  // namespace lsp_handlers

lsp/handlers/text_document/completion.h

@@ -2,17 +2,13 @@
 
 #include <optional>
 
+#include "common/util/json_util.h"
+
 #include "lsp/protocol/common_types.h"
 #include "lsp/protocol/completion.h"
 #include "lsp/state/data/mips_instructions.h"
 #include "lsp/state/workspace.h"
 
-std::optional<json> get_completions_handler(Workspace& /*workspace*/, int /*id*/, json params) {
-  auto converted_params = params.get<LSPSpec::CompletionParams>();
-
-  // TODO - these need to be cached,
-
-  // TODO - implement response object
-
-  return json::array();
+namespace lsp_handlers {
+std::optional<json> get_completions(Workspace& workspace, int id, json params);
 }

lsp/handlers/text_document/document_color.cpp

@@ -0,0 +1,157 @@
+#include "lsp/protocol/document_color.h"
+
+#include <optional>
+
+#include "lsp/protocol/common_types.h"
+#include "lsp/state/workspace.h"
+
+int hex_to_dec(const std::string& hex) {
+  std::string cleaned_string = hex;
+  if (cleaned_string.starts_with("#x")) {
+    cleaned_string = cleaned_string.substr(2);
+  }
+  return std::stoi(cleaned_string, nullptr, 16);
+}
+
+std::unordered_map<GameVersion, std::unordered_map<int, std::tuple<float, float, float, float>>>
+    game_font_colors = {{GameVersion::Jak1,
+                         {
+                             {0, {223.0, 239.0, 223.0, 255.0}},  {1, {255.0, 255.0, 255.0, 255.0}},
+                             {2, {255.0, 255.0, 255.0, 127.0}},  {3, {255.0, 191.0, 63.0, 255.0}},
+                             {4, {255.0, 199.0, 0.0, 255.0}},    {5, {255.0, 255.0, 0.0, 255.0}},
+                             {6, {63.0, 255.0, 63.0, 255.0}},    {7, {127.0, 127.0, 255.0, 255.0}},
+                             {8, {-1.0, 255.0, 255.0, 255.0}},   {9, {255.0, 127.0, 255.0, 255.0}},
+                             {10, {191.0, 255.0, 255.0, 255.0}}, {11, {127.0, 191.0, 191.0, 255.0}},
+                             {12, {255.0, 255.0, 255.0, 255.0}}, {13, {159.0, 159.0, 159.0, 255.0}},
+                             {14, {255.0, 167.0, 0.0, 255.0}},   {15, {223.0, 255.0, 95.0, 255.0}},
+                             {16, {143.0, 175.0, 15.0, 255.0}},  {17, {175.0, 191.0, 175.0, 255.0}},
+                             {18, {127.0, 143.0, 127.0, 255.0}}, {19, {95.0, 63.0, 95.0, 255.0}},
+                             {20, {255.0, 241.0, 143.0, 255.0}}, {21, {63.0, 187.0, 239.0, 255.0}},
+                             {22, {57.0, 57.0, 57.0, 255.0}},    {23, {127.0, 127.0, 127.0, 255.0}},
+                             {24, {243.0, 153.0, 201.0, 255.0}}, {25, {243.0, 103.0, 103.0, 255.0}},
+                             {26, {31.0, 201.0, 151.0, 255.0}},  {27, {139.0, 147.0, 239.0, 255.0}},
+                             {28, {173.0, 251.0, 255.0, 255.0}}, {29, {253.0, 245.0, 101.0, 255.0}},
+                             {30, {241.0, 241.0, 3.0, 255.0}},   {31, {141.0, 207.0, 243.0, 255.0}},
+                             {32, {223.0, 239.0, 223.0, 255.0}}, {33, {191.0, -1.0, 0.0, 255.0}},
+                             {34, {255.0, 191.0, 63.0, 255.0}},
+                         }},
+                        {GameVersion::Jak2,
+                         {
+                             {0, {223.0, 239.0, 223.0, 255.0}},  {1, {255.0, 255.0, 255.0, 255.0}},
+                             {2, {255.0, 255.0, 255.0, 127.0}},  {3, {255.0, 63.0, 0.0, 255.0}},
+                             {4, {255.0, 199.0, 0.0, 255.0}},    {5, {255.0, 255.0, 0.0, 255.0}},
+                             {6, {63.0, 255.0, 63.0, 255.0}},    {7, {0.0, 63.0, 255.0, 255.0}},
+                             {8, {0.0, 255.0, 255.0, 255.0}},    {9, {255.0, 127.0, 255.0, 255.0}},
+                             {10, {191.0, 255.0, 255.0, 255.0}}, {11, {127.0, 191.0, 191.0, 255.0}},
+                             {12, {255.0, 255.0, 255.0, 255.0}}, {13, {159.0, 159.0, 159.0, 255.0}},
+                             {14, {255.0, 167.0, 0.0, 255.0}},   {15, {223.0, 255.0, 95.0, 255.0}},
+                             {16, {143.0, 175.0, 31.0, 255.0}},  {17, {175.0, 191.0, 175.0, 255.0}},
+                             {18, {127.0, 143.0, 127.0, 255.0}}, {19, {95.0, 63.0, 95.0, 255.0}},
+                             {20, {255.0, 241.0, 143.0, 255.0}}, {21, {63.0, 187.0, 239.0, 255.0}},
+                             {22, {57.0, 57.0, 57.0, 255.0}},    {23, {127.0, 127.0, 127.0, 255.0}},
+                             {24, {243.0, 153.0, 201.0, 255.0}}, {25, {243.0, 103.0, 103.0, 255.0}},
+                             {26, {31.0, 201.0, 151.0, 255.0}},  {27, {139.0, 147.0, 239.0, 255.0}},
+                             {28, {173.0, 251.0, 255.0, 255.0}}, {29, {253.0, 245.0, 101.0, 255.0}},
+                             {30, {241.0, 241.0, 3.0, 255.0}},   {31, {141.0, 207.0, 243.0, 255.0}},
+                             {32, {127.0, 255.0, 255.0, 255.0}}, {33, {127.0, 255.0, 255.0, 255.0}},
+                             {34, {255.0, 255.0, 255.0, 255.0}}, {35, {63.0, 127.0, 127.0, 191.0}},
+                             {36, {223.0, 239.0, 223.0, 255.0}}, {37, {191.0, 0.0, 0.0, 255.0}},
+                             {38, {255.0, 191.0, 63.0, 255.0}},  {39, {0.0, 0.0, 1.0, 255.0}},
+                         }}};
+
+namespace lsp_handlers {
+
+std::optional<json> document_color(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DocumentColorParams>();
+  auto file_type = workspace.determine_filetype_from_uri(params.textDocument.m_uri);
+  const auto game_version = workspace.determine_game_version_from_uri(params.textDocument.m_uri);
+
+  json colors = json::array();
+
+  if (!game_version || file_type != Workspace::FileType::OpenGOAL) {
+    return colors;
+  }
+
+  auto maybe_tracked_file = workspace.get_tracked_og_file(params.textDocument.m_uri);
+  if (!maybe_tracked_file) {
+    return colors;
+  }
+  const auto& tracked_file = maybe_tracked_file.value().get();
+
+  // Search for `(new 'static 'rgba....` forms as these can be colored
+  // for example - `(new 'static 'rgba :r #x70 :g #x78 :b #x70 :a #x80)`
+  const auto rgba_results =
+      tracked_file.search_for_forms_that_begin_with({"(", "new", "'static", "'rgba"});
+  for (const auto& result : rgba_results) {
+    // Iterate the forms and find the color and alpha info
+    float red = 0.0f;
+    float green = 0.0f;
+    float blue = 0.0f;
+    float alpha = 0.0f;
+    int token_idx = 0;
+    while (token_idx < result.tokens.size()) {
+      const auto& token = result.tokens[token_idx];
+      // in OpenGOAL -- 255 is equal to 128, so we double every value and subtract 1
+      if (token == ":r" && result.tokens.size() > token_idx + 1) {
+        red = static_cast<float>((hex_to_dec(result.tokens[token_idx + 1]) * 2) - 1) / 255.0f;
+      } else if (token == ":g" && result.tokens.size() > token_idx + 1) {
+        green = static_cast<float>((hex_to_dec(result.tokens[token_idx + 1]) * 2) - 1) / 255.0f;
+      } else if (token == ":b" && result.tokens.size() > token_idx + 1) {
+        blue = static_cast<float>((hex_to_dec(result.tokens[token_idx + 1]) * 2) - 1) / 255.0f;
+      } else if (token == ":a" && result.tokens.size() > token_idx + 1) {
+        alpha = static_cast<float>((hex_to_dec(result.tokens[token_idx + 1]) * 2) - 1) / 255.0f;
+      }
+      token_idx++;
+    }
+    LSPSpec::ColorInformation color_info;
+    color_info.range = {{(uint32_t)result.start_point.first, (uint32_t)result.start_point.second},
+                        {(uint32_t)result.end_point.first, (uint32_t)result.end_point.second}};
+    color_info.color = LSPSpec::Color{red, green, blue, alpha};
+    colors.push_back(color_info);
+  }
+  // Also search for the `(static-rgba ...` macro
+  const auto static_rgba_results =
+      tracked_file.search_for_forms_that_begin_with({"(", "static-rgba"});
+  for (const auto& result : static_rgba_results) {
+    float red = static_cast<float>((hex_to_dec(result.tokens[2]) * 2) - 1) / 255.0f;
+    float green = static_cast<float>((hex_to_dec(result.tokens[3]) * 2) - 1) / 255.0f;
+    float blue = static_cast<float>((hex_to_dec(result.tokens[4]) * 2) - 1) / 255.0f;
+    float alpha = static_cast<float>((hex_to_dec(result.tokens[5]) * 2) - 1) / 255.0f;
+    LSPSpec::ColorInformation color_info;
+    color_info.range = {{(uint32_t)result.start_point.first, (uint32_t)result.start_point.second},
+                        {(uint32_t)result.end_point.first, (uint32_t)result.end_point.second}};
+    color_info.color = LSPSpec::Color{red, green, blue, alpha};
+    colors.push_back(color_info);
+  }
+
+  // Search for `(font-color ...` forms
+  const auto font_color_results =
+      tracked_file.search_for_forms_that_begin_with({"(", "font-color"});
+  const auto font_color_enum_entries =
+      workspace.get_enum_entries("font-color", game_version.value());
+  if (!font_color_enum_entries.empty() &&
+      game_font_colors.find(game_version.value()) != game_font_colors.end()) {
+    for (const auto& result : font_color_results) {
+      const auto font_color = result.tokens[2];
+      if (font_color_enum_entries.find(font_color) != font_color_enum_entries.end()) {
+        const auto font_color_val = font_color_enum_entries.at(font_color);
+        if (game_font_colors[game_version.value()].find(font_color_val) !=
+            game_font_colors[game_version.value()].end()) {
+          const auto& [red, green, blue, alpha] =
+              game_font_colors[game_version.value()].at(font_color_val);
+          LSPSpec::ColorInformation color_info;
+          color_info.range = {
+              {(uint32_t)result.start_point.first, (uint32_t)result.start_point.second},
+              {(uint32_t)result.end_point.first, (uint32_t)result.end_point.second}};
+          color_info.color =
+              LSPSpec::Color{red / 255.0f, green / 255.0f, blue / 255.0f, alpha / 255.0f};
+          colors.push_back(color_info);
+        }
+      }
+    }
+  }
+
+  return colors;
+}
+
+}  // namespace lsp_handlers

lsp/handlers/text_document/document_color.h

@@ -2,76 +2,14 @@
 
 #include <optional>
 
-#include "common/util/string_util.h"
+#include "common/util/json_util.h"
 
 #include "lsp/protocol/common_types.h"
 #include "lsp/protocol/document_color.h"
+#include "lsp/state/workspace.h"
 
-float hexToFloat(const std::string& hex) {
-  int value = std::stoi(hex, nullptr, 16);
-  return static_cast<float>(value) / 255.0f;
-}
+namespace lsp_handlers {
 
-std::optional<LSPSpec::Color> color_hexstring_to_lsp_color(const std::string& color_name) {
-  if (!str_util::contains(color_name, "#")) {
-    return {};
-  }
-  const auto color_tokens = str_util::split(color_name, '#');
-  const auto hexstring = color_tokens.at(1);
-  std::string red_hex = hexstring.substr(0, 2);
-  std::string green_hex = hexstring.substr(2, 2);
-  std::string blue_hex = hexstring.substr(4, 2);
+std::optional<json> document_color(Workspace& workspace, int id, json raw_params);
 
-  float red = hexToFloat(red_hex);
-  float green = hexToFloat(green_hex);
-  float blue = hexToFloat(blue_hex);
-
-  return LSPSpec::Color{red, green, blue, 1.0};
-}
-
-std::optional<json> document_color_handler(Workspace& /*workspace*/, int /*id*/, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DocumentColorParams>();
-  json colors = json::array();
-
-  // TODO - hex strings aren't desirable in the `font-color` enum
-  // this could be used for the `new 'static 'rgba` instances but that requires proper
-  // AST support as it cannot (and should not) be assumed that all 4 components will be on the same
-  // line
-  return colors;
-
-  //// Iterate through document, mark text colors ourselves
-  // auto file_type = workspace.determine_filetype_from_uri(params.textDocument.m_uri);
-
-  // if (file_type == Workspace::FileType::OpenGOAL) {
-  //   auto tracked_file = workspace.get_tracked_og_file(params.textDocument.m_uri);
-  //   if (!tracked_file) {
-  //     return {};
-  //   }
-
-  //  // This is something that is ok to be a regex, because it's very niche
-  //  for (int i = 0; i < tracked_file->m_lines.size(); i++) {
-  //    const auto& line = tracked_file->m_lines.at(i);
-  //    std::smatch matches;
-  //    std::regex regex("\\(font-color ([^)]*)\\)");
-
-  //    std::sregex_iterator iter(line.begin(), line.end(), regex);
-  //    std::sregex_iterator end;
-
-  //    for (; iter != end; iter++) {
-  //      std::smatch match = *iter;
-  //      std::string capture_group = match.str(1);
-  //      LSPSpec::ColorInformation color_info;
-  //      color_info.range = {{i, match.position(1)}, {i, match.position(1) + match.size()}};
-  //      const auto color = color_hexstring_to_lsp_color(capture_group);
-  //      if (!color) {
-  //        continue;
-  //      }
-  //      color_info.color = color.value();
-  //      colors.push_back(color_info);
-  //      lg::debug("color - {}", capture_group);
-  //    }
-  //  }
-  //}
-
-  // return colors;
-}
+}  // namespace lsp_handlers

lsp/handlers/text_document/document_symbol.cpp

@@ -0,0 +1,53 @@
+#include "document_symbol.h"
+
+#include "third-party/json.hpp"
+
+using json = nlohmann::json;
+
+std::optional<json> ir_symbols(Workspace& workspace, LSPSpec::DocumentSymbolParams params) {
+  json symbols = json::array();
+  auto maybe_tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
+  if (!maybe_tracked_file) {
+    return symbols;
+  }
+
+  const auto& tracked_file = maybe_tracked_file.value().get();
+  for (const auto& symbol : tracked_file.m_symbols) {
+    symbols.push_back(symbol);
+  }
+
+  return symbols;
+}
+
+std::optional<json> og_symbols(Workspace& workspace, LSPSpec::DocumentSymbolParams params) {
+  json symbols = json::array();
+  auto maybe_tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
+  if (!maybe_tracked_file) {
+    return symbols;
+  }
+
+  const auto& tracked_file = maybe_tracked_file.value().get();
+  for (const auto& symbol : tracked_file.m_symbols) {
+    symbols.push_back(symbol);
+  }
+
+  return symbols;
+}
+
+namespace lsp_handlers {
+
+std::optional<json> document_symbols(Workspace& workspace, int /*id*/, json params) {
+  auto converted_params = params.get<LSPSpec::DocumentSymbolParams>();
+  const auto file_type =
+      workspace.determine_filetype_from_uri(converted_params.m_textDocument.m_uri);
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    return ir_symbols(workspace, converted_params);
+  } else if (file_type == Workspace::FileType::OpenGOAL) {
+    return og_symbols(workspace, converted_params);
+  }
+
+  return json::array();
+}
+
+}  // namespace lsp_handlers

lsp/handlers/text_document/document_symbol.h

@@ -2,26 +2,13 @@
 
 #include <optional>
 
+#include "common/util/json_util.h"
+
 #include "lsp/protocol/common_types.h"
 #include "lsp/state/workspace.h"
 
-#include "third-party/json.hpp"
+namespace lsp_handlers {
 
-using json = nlohmann::json;
+std::optional<json> document_symbols(Workspace& workspace, int id, json params);
 
-std::optional<json> document_symbols_handler(Workspace& workspace, int /*id*/, json params) {
-  auto converted_params = params.get<LSPSpec::DocumentSymbolParams>();
-  auto tracked_file = workspace.get_tracked_ir_file(converted_params.m_textDocument.m_uri);
-
-  if (!tracked_file) {
-    return {};
-  }
-
-  // TODO - convert to type!
-
-  json arr = json::array();
-  for (const auto& symbol : tracked_file.value().m_symbols) {
-    arr.push_back(symbol);
-  }
-  return arr;
-}
+}  // namespace lsp_handlers

lsp/handlers/text_document/document_synchronization.cpp

@@ -0,0 +1,77 @@
+#include "document_synchronization.h"
+
+namespace lsp_handlers {
+
+void did_open(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DidOpenTextDocumentParams>();
+  workspace.start_tracking_file(params.m_textDocument.m_uri, params.m_textDocument.m_languageId,
+                                params.m_textDocument.m_text);
+}
+
+void did_change(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DidChangeTextDocumentParams>();
+  for (const auto& change : params.m_contentChanges) {
+    workspace.update_tracked_file(params.m_textDocument.m_uri, change.m_text);
+  }
+}
+
+void did_close(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DidCloseTextDocumentParams>();
+  workspace.stop_tracking_file(params.m_textDocument.m_uri);
+}
+
+void will_save(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::WillSaveTextDocumentParams>();
+  workspace.tracked_file_will_save(params.textDocument.m_uri);
+}
+
+std::optional<json> did_open_push_diagnostics(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DidOpenTextDocumentParams>();
+  const auto file_type =
+      workspace.determine_filetype_from_languageid(params.m_textDocument.m_languageId);
+
+  LSPSpec::PublishDiagnosticParams publish_params;
+  publish_params.m_uri = params.m_textDocument.m_uri;
+  publish_params.m_version = params.m_textDocument.m_version;
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    auto maybe_tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    publish_params.m_diagnostics = tracked_file.m_diagnostics;
+  }
+
+  json response;
+  response["method"] = "textDocument/publishDiagnostics";
+  response["params"] = publish_params;
+
+  return response;
+}
+
+std::optional<json> did_change_push_diagnostics(Workspace& workspace, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DidChangeTextDocumentParams>();
+  const auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
+
+  LSPSpec::PublishDiagnosticParams publish_params;
+  publish_params.m_uri = params.m_textDocument.m_uri;
+  publish_params.m_version = params.m_textDocument.m_version;
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    auto maybe_tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    publish_params.m_diagnostics = tracked_file.m_diagnostics;
+  }
+
+  json response;
+  response["method"] = "textDocument/publishDiagnostics";
+  response["params"] = publish_params;
+
+  return response;
+}
+
+}  // namespace lsp_handlers

lsp/handlers/text_document/document_synchronization.h

@@ -2,76 +2,20 @@
 
 #include <optional>
 
+#include "common/util/json_util.h"
+
 #include "lsp/protocol/document_diagnostics.h"
 #include "lsp/protocol/document_synchronization.h"
 #include "lsp/state/workspace.h"
 
-#include "third-party/json.hpp"
+namespace lsp_handlers {
 
-using json = nlohmann::json;
+void did_open(Workspace& workspace, json raw_params);
+void did_change(Workspace& workspace, json raw_params);
+void did_close(Workspace& workspace, json raw_params);
+void will_save(Workspace& workspace, json raw_params);
 
-void did_open_handler(Workspace& workspace, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DidOpenTextDocumentParams>();
-  workspace.start_tracking_file(params.m_textDocument.m_uri, params.m_textDocument.m_languageId,
-                                params.m_textDocument.m_text);
-}
+std::optional<json> did_open_push_diagnostics(Workspace& workspace, json raw_params);
+std::optional<json> did_change_push_diagnostics(Workspace& workspace, json raw_params);
 
-void did_change_handler(Workspace& workspace, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DidChangeTextDocumentParams>();
-  for (const auto& change : params.m_contentChanges) {
-    workspace.update_tracked_file(params.m_textDocument.m_uri, change.m_text);
-  }
-}
-
-void did_close_handler(Workspace& workspace, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DidCloseTextDocumentParams>();
-  workspace.stop_tracking_file(params.m_textDocument.m_uri);
-}
-
-std::optional<json> did_open_push_diagnostics(Workspace& workspace, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DidOpenTextDocumentParams>();
-  const auto file_type =
-      workspace.determine_filetype_from_languageid(params.m_textDocument.m_languageId);
-
-  LSPSpec::PublishDiagnosticParams publish_params;
-  publish_params.m_uri = params.m_textDocument.m_uri;
-  publish_params.m_version = params.m_textDocument.m_version;
-
-  if (file_type == Workspace::FileType::OpenGOALIR) {
-    auto tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
-    if (!tracked_file) {
-      return {};
-    }
-    publish_params.m_diagnostics = tracked_file.value().m_diagnostics;
-  }
-
-  json response;
-  response["method"] = "textDocument/publishDiagnostics";
-  response["params"] = publish_params;
-
-  return response;
-}
-
-std::optional<json> did_change_push_diagnostics(Workspace& workspace, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DidChangeTextDocumentParams>();
-  const auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
-
-  LSPSpec::PublishDiagnosticParams publish_params;
-  publish_params.m_uri = params.m_textDocument.m_uri;
-  publish_params.m_version = params.m_textDocument.m_version;
-
-  if (file_type == Workspace::FileType::OpenGOALIR) {
-    auto tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
-
-    if (!tracked_file) {
-      return {};
-    }
-    publish_params.m_diagnostics = tracked_file.value().m_diagnostics;
-  }
-
-  json response;
-  response["method"] = "textDocument/publishDiagnostics";
-  response["params"] = publish_params;
-
-  return response;
-}
+}  // namespace lsp_handlers

lsp/handlers/text_document/formatting.cpp

@@ -0,0 +1,38 @@
+#include "formatting.h"
+
+#include "common/formatter/formatter.h"
+
+#include "lsp/protocol/common_types.h"
+#include "lsp/protocol/formatting.h"
+#include "lsp/state/data/mips_instructions.h"
+#include "lsp/state/workspace.h"
+
+namespace lsp_handlers {
+std::optional<json> formatting(Workspace& workspace, int id, json raw_params) {
+  auto params = raw_params.get<LSPSpec::DocumentFormattingParams>();
+  const auto file_type = workspace.determine_filetype_from_uri(params.textDocument.m_uri);
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    return nullptr;
+  } else if (file_type == Workspace::FileType::OpenGOAL) {
+    auto maybe_tracked_file = workspace.get_tracked_og_file(params.textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    const auto result = formatter::format_code(tracked_file.m_content);
+    if (!result) {
+      return nullptr;
+    }
+    json edits = json::array();
+    auto format_edit = LSPSpec::TextEdit();
+    format_edit.range = {{0, 0}, {(uint32_t)tracked_file.m_line_count, 0}};
+    format_edit.newText = result.value();
+    edits.push_back(format_edit);
+    return edits;
+  }
+
+  return nullptr;
+}
+
+}  // namespace lsp_handlers

lsp/handlers/text_document/formatting.h

@@ -2,36 +2,12 @@
 
 #include <optional>
 
-#include "common/formatter/formatter.h"
+#include "common/util/json_util.h"
 
-#include "lsp/protocol/common_types.h"
-#include "lsp/protocol/formatting.h"
-#include "lsp/state/data/mips_instructions.h"
 #include "lsp/state/workspace.h"
 
-std::optional<json> formatting_handler(Workspace& workspace, int /*id*/, json raw_params) {
-  auto params = raw_params.get<LSPSpec::DocumentFormattingParams>();
-  const auto file_type = workspace.determine_filetype_from_uri(params.textDocument.m_uri);
+namespace lsp_handlers {
 
-  if (file_type == Workspace::FileType::OpenGOALIR) {
-    return nullptr;
-  } else if (file_type == Workspace::FileType::OpenGOAL) {
-    auto tracked_file = workspace.get_tracked_og_file(params.textDocument.m_uri);
-    if (!tracked_file) {
-      return nullptr;
-    }
-    // TODO move away from holding the content directly
-    const auto result = formatter::format_code(tracked_file->m_content);
-    if (!result) {
-      return nullptr;
-    }
-    json edits = json::array();
-    auto format_edit = LSPSpec::TextEdit();
-    format_edit.range = {{0, 0}, {(uint32_t)tracked_file->m_lines.size(), 0}};
-    format_edit.newText = result.value();
-    edits.push_back(format_edit);
-    return edits;
-  }
+std::optional<json> formatting(Workspace& workspace, int id, json raw_params);
 
-  return nullptr;
-}
+}  // namespace lsp_handlers

lsp/handlers/text_document/go_to.cpp

@@ -0,0 +1,61 @@
+#include "go_to.h"
+
+namespace lsp_handlers {
+std::optional<json> go_to_definition(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::TextDocumentPositionParams>();
+  const auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
+
+  json locations = json::array();
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    auto maybe_tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    auto symbol_name = tracked_file.get_symbol_at_position(params.m_position);
+    if (!symbol_name) {
+      return {};
+    }
+    auto symbol_info = workspace.get_definition_info_from_all_types(symbol_name.value(),
+                                                                    tracked_file.m_all_types_uri);
+    if (!symbol_info) {
+      return {};
+    }
+    LSPSpec::Location location;
+    location.m_uri = tracked_file.m_all_types_uri;
+    location.m_range.m_start = {(uint32_t)symbol_info.value().definition_info->line_idx_to_display,
+                                (uint32_t)symbol_info.value().definition_info->pos_in_line};
+    location.m_range.m_end = {(uint32_t)symbol_info.value().definition_info->line_idx_to_display,
+                              (uint32_t)symbol_info.value().definition_info->pos_in_line};
+    locations.push_back(location);
+  } else if (file_type == Workspace::FileType::OpenGOAL) {
+    auto maybe_tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    const auto symbol = tracked_file.get_symbol_at_position(params.m_position);
+    if (!symbol) {
+      return {};
+    }
+    const auto& symbol_info = workspace.get_global_symbol_info(tracked_file, symbol.value());
+    if (!symbol_info) {
+      return {};
+    }
+
+    const auto& def_loc = workspace.get_symbol_def_location(tracked_file, symbol_info.value());
+    if (!def_loc) {
+      return {};
+    }
+
+    LSPSpec::Location location;
+    location.m_uri = def_loc->file_path;
+    location.m_range.m_start = {(uint32_t)def_loc->line_idx, (uint32_t)def_loc->char_idx};
+    location.m_range.m_end = {(uint32_t)def_loc->line_idx, (uint32_t)def_loc->char_idx};
+    locations.push_back(location);
+  }
+
+  return locations;
+}
+}  // namespace lsp_handlers

lsp/handlers/text_document/go_to.h

@@ -2,65 +2,11 @@
 
 #include <optional>
 
+#include "common/util/json_util.h"
+
 #include "lsp/protocol/common_types.h"
-#include "lsp/protocol/hover.h"
-#include "lsp/state/data/mips_instructions.h"
 #include "lsp/state/workspace.h"
 
-std::optional<json> go_to_definition_handler(Workspace& workspace, int /*id*/, json raw_params) {
-  auto params = raw_params.get<LSPSpec::TextDocumentPositionParams>();
-  const auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
-
-  json locations = json::array();
-
-  if (file_type == Workspace::FileType::OpenGOALIR) {
-    auto tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
-    if (!tracked_file) {
-      return {};
-    }
-    auto symbol_name = tracked_file->get_symbol_at_position(params.m_position);
-    if (!symbol_name) {
-      return {};
-    }
-    auto symbol_info = workspace.get_definition_info_from_all_types(symbol_name.value(),
-                                                                    tracked_file->m_all_types_uri);
-    if (!symbol_info) {
-      return {};
-    }
-    LSPSpec::Location location;
-    location.m_uri = tracked_file->m_all_types_uri;
-    location.m_range.m_start = {(uint32_t)symbol_info.value().definition_info->line_idx_to_display,
-                                (uint32_t)symbol_info.value().definition_info->pos_in_line};
-    location.m_range.m_end = {(uint32_t)symbol_info.value().definition_info->line_idx_to_display,
-                              (uint32_t)symbol_info.value().definition_info->pos_in_line};
-    locations.push_back(location);
-  } else if (file_type == Workspace::FileType::OpenGOAL) {
-    auto tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
-    if (!tracked_file) {
-      return {};
-    }
-    const auto symbol = tracked_file->get_symbol_at_position(params.m_position);
-    if (!symbol) {
-      return {};
-    }
-    const auto& symbol_info =
-        workspace.get_global_symbol_info(tracked_file.value(), symbol.value());
-    if (!symbol_info) {
-      return {};
-    }
-
-    const auto& def_loc =
-        workspace.get_symbol_def_location(tracked_file.value(), symbol_info.value());
-    if (!def_loc) {
-      return {};
-    }
-
-    LSPSpec::Location location;
-    location.m_uri = def_loc->filename;
-    location.m_range.m_start = {(uint32_t)def_loc->line_idx, (uint32_t)def_loc->char_idx};
-    location.m_range.m_end = {(uint32_t)def_loc->line_idx, (uint32_t)def_loc->char_idx};
-    locations.push_back(location);
-  }
-
-  return locations;
+namespace lsp_handlers {
+std::optional<json> go_to_definition(Workspace& workspace, int id, json raw_params);
 }

lsp/handlers/text_document/hover.cpp

@@ -0,0 +1,269 @@
+#include "hover.h"
+
+bool is_number(const std::string& s) {
+  return !s.empty() && std::find_if(s.begin(), s.end(),
+                                    [](unsigned char c) { return !std::isdigit(c); }) == s.end();
+}
+
+std::vector<std::string> og_method_names = {"new",      "delete", "print",    "inspect",  "length",
+                                            "asize-of", "copy",   "relocate", "mem-usage"};
+
+std::optional<LSPSpec::Hover> hover_handler_ir(Workspace& workspace,
+                                               const LSPSpec::TextDocumentPositionParams& params,
+                                               const WorkspaceIRFile& tracked_file) {
+  // See if it's an OpenGOAL symbol or a MIPS mnemonic
+  auto symbol_name = tracked_file.get_symbol_at_position(params.m_position);
+  auto token_at_pos = tracked_file.get_mips_instruction_at_position(params.m_position);
+  if (!symbol_name && !token_at_pos) {
+    return {};
+  }
+
+  LSPSpec::MarkupContent markup;
+  markup.m_kind = "markdown";
+
+  // TODO - try specifying the range so it highlights everything, ie. `c.lt.s`
+  // Prefer symbols
+  if (symbol_name) {
+    lg::debug("hover - symbol match - {}", symbol_name.value());
+    auto symbol_info = workspace.get_definition_info_from_all_types(symbol_name.value(),
+                                                                    tracked_file.m_all_types_uri);
+    if (symbol_info && symbol_info.value().docstring.has_value()) {
+      std::string docstring = symbol_info.value().docstring.value();
+      lg::debug("hover - symbol has docstring - {}", docstring);
+      // A docstring exists, print it!
+      // By convention, docstrings are assumed to be markdown, they support code-blocks everything
+      // the only thing extra we do, is replace [[<symbol>]] with links if available
+      std::unordered_map<std::string, std::string> symbol_replacements = {};
+      std::smatch match;
+
+      std::string::const_iterator searchStart(docstring.cbegin());
+      while (
+          std::regex_search(searchStart, docstring.cend(), match, std::regex("\\[{2}(.*)\\]{2}"))) {
+        // Have we already accounted for this symbol?
+        const auto& name = match[1].str();
+        if (symbol_replacements.count(name) != 0) {
+          continue;
+        }
+        // Get this symbols info
+        auto symbol_info =
+            workspace.get_definition_info_from_all_types(name, tracked_file.m_all_types_uri);
+        if (!symbol_info) {
+          symbol_replacements[name] = fmt::format("_{}_", name);
+        } else {
+          // Construct path
+          auto symbol_uri =
+              fmt::format("{}#L{}%2C{}", tracked_file.m_all_types_uri,
+                          symbol_info.value().definition_info->line_idx_to_display + 1,
+                          symbol_info.value().definition_info->pos_in_line);
+          symbol_replacements[name] = fmt::format("[{}]({})", name, symbol_uri);
+        }
+        searchStart = match.suffix().first;
+      }
+      // Replace all symbol occurences
+      for (const auto& [key, val] : symbol_replacements) {
+        docstring = std::regex_replace(docstring, std::regex("\\[{2}" + key + "\\]{2}"), val);
+      }
+
+      markup.m_value = docstring;
+      LSPSpec::Hover hover_resp;
+      hover_resp.m_contents = markup;
+      return hover_resp;
+    } else if (!token_at_pos) {
+      // Check if it's a number, and if so we'll do some numeric conversions
+      if (!is_number(symbol_name.value())) {
+        return {};
+      }
+      lg::debug("hover - numeric match - {}", symbol_name.value());
+      // Construct the body
+      std::string body = "";
+      uint32_t num = std::atoi(symbol_name.value().data());
+      // Assuming it comes in as Decimal
+      body += "| Base    | Value |\n";
+      body += "|---------|-------|\n";
+      body += fmt::format("| Decimal        | `{:d}` |\n", num);
+      body += fmt::format("| Hex            | `{:X}` |\n", num);
+      // TODO - would be nice to format as groups of 4
+      body += fmt::format("| Binary         | `{:b}` |\n", num);
+      if (num >= 16 && (num - 16) % 4 == 0) {
+        uint32_t method_id = (num - 16) / 4;
+        std::string method_name = "not built-in";
+        if (method_id <= 8) {
+          method_name = og_method_names.at(method_id);
+        }
+        body += fmt::format("| Method ID   | `{}` - `{}` |\n", method_id, method_name);
+      }
+      body += fmt::format("| Octal          | `{:o}` |\n", num);
+
+      markup.m_value = body;
+      LSPSpec::Hover hover_resp;
+      hover_resp.m_contents = markup;
+      return hover_resp;
+    }
+  }
+
+  // Otherwise, maybe it's a MIPS instruction
+  if (token_at_pos) {
+    lg::debug("hover - token match - {}", token_at_pos.value());
+    auto& token = token_at_pos.value();
+    std::transform(token.begin(), token.end(), token.begin(),
+                   [](unsigned char c) { return std::tolower(c); });
+    // Find the instruction, there are some edge-cases here where they could be multiple
+    // TODO - havn't addressed `bc` and such instructions!  Those need to be prefixed matched
+    std::vector<std::string> ee_instructions = {};
+    std::vector<std::string> vu_instructions = {};
+    for (const auto& instr : LSPData::MIPS_INSTRUCTION_LIST) {
+      auto mnemonic_lower = instr.mnemonic;
+      std::transform(mnemonic_lower.begin(), mnemonic_lower.end(), mnemonic_lower.begin(),
+                     [](unsigned char c) { return std::tolower(c); });
+      if (mnemonic_lower == token) {
+        if (instr.type == "ee") {
+          ee_instructions.push_back(fmt::format("- _{}_\n\n", instr.description));
+        } else {
+          vu_instructions.push_back(fmt::format("- _{}_\n\n", instr.description));
+        }
+      }
+    }
+
+    // Construct the body
+    std::string body = "";
+    if (!ee_instructions.empty()) {
+      body += "**EE Instructions**\n\n";
+      for (const auto& instr : ee_instructions) {
+        body += instr;
+      }
+      body += "___\n\n";
+    }
+
+    if (!vu_instructions.empty()) {
+      body += "**VU Instructions**\n\n";
+      for (const auto& instr : vu_instructions) {
+        body += instr;
+      }
+      body += "___\n\n";
+    }
+
+    markup.m_value = body;
+    LSPSpec::Hover hover_resp;
+    hover_resp.m_contents = markup;
+    return hover_resp;
+  }
+
+  return {};
+}
+
+std::string truncate_docstring(const std::string& docstring) {
+  std::string truncated = "";
+  const auto lines = str_util::split(docstring);
+  for (const auto& line : lines) {
+    const auto trimmed_line = str_util::ltrim(line);
+    if (str_util::starts_with(trimmed_line, "@")) {
+      break;
+    }
+    truncated += trimmed_line + "\n";
+  }
+  return truncated;
+}
+
+namespace lsp_handlers {
+std::optional<json> hover(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::TextDocumentPositionParams>();
+  auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
+
+  if (file_type == Workspace::FileType::OpenGOALIR) {
+    auto tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
+    if (!tracked_file) {
+      return {};
+    }
+    return hover_handler_ir(workspace, params, tracked_file.value());
+  } else if (file_type == Workspace::FileType::OpenGOAL) {
+    auto maybe_tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
+    if (!maybe_tracked_file) {
+      return {};
+    }
+    const auto& tracked_file = maybe_tracked_file.value().get();
+    const auto symbol = tracked_file.get_symbol_at_position(params.m_position);
+    if (!symbol) {
+      lg::debug("hover - no symbol");
+      return {};
+    }
+    // TODO - there is an issue with docstrings and overridden methods
+    const auto& symbol_info = workspace.get_global_symbol_info(tracked_file, symbol.value());
+    if (!symbol_info) {
+      lg::debug("hover - no symbol info - {}", symbol.value());
+      return {};
+    }
+    LSPSpec::MarkupContent markup;
+    markup.m_kind = "markdown";
+
+    const auto args = Docs::get_args_from_docstring(symbol_info.value()->m_args,
+                                                    symbol_info.value()->m_docstring);
+    std::string signature = "";
+    bool takes_args = true;
+    if (symbol_info.value()->m_kind == symbol_info::Kind::FUNCTION) {
+      signature += "function ";
+    } else if (symbol_info.value()->m_kind == symbol_info::Kind::METHOD) {
+      signature += "method ";
+    } else if (symbol_info.value()->m_kind == symbol_info::Kind::MACRO) {
+      signature += "macro ";
+    } else {
+      takes_args = false;
+    }
+    // TODO - others useful, probably states?
+    auto type_info = workspace.get_symbol_typeinfo(tracked_file, symbol.value());
+    signature += symbol.value();
+    if (takes_args) {
+      signature += "(";
+      for (int i = 0; i < (int)args.size(); i++) {
+        const auto& arg = args.at(i);
+        if (i == (int)args.size() - 1) {
+          signature += fmt::format("{}: {}", arg.name, arg.type);
+        } else {
+          signature += fmt::format("{}: {}, ", arg.name, arg.type);
+        }
+      }
+      signature += ")";
+      if (symbol_info.value()->m_kind == symbol_info::Kind::FUNCTION && type_info) {
+        signature += fmt::format(": {}", type_info->first.last_arg().base_type());
+      } else if (symbol_info.value()->m_kind == symbol_info::Kind::METHOD) {
+        signature +=
+            fmt::format(": {}", symbol_info.value()->m_method_info.type.last_arg().base_type());
+      }
+    } else if (type_info) {
+      signature += fmt::format(": {}", type_info->second->get_parent());
+    }
+
+    std::string body = fmt::format("```opengoal\n{}\n```\n\n", signature);
+    body += "___\n\n";
+    if (!symbol_info.value()->m_docstring.empty()) {
+      body += truncate_docstring(symbol_info.value()->m_docstring) + "\n\n";
+    }
+
+    // TODO - support @see/@returns/[[reference]]
+    for (const auto& arg : args) {
+      std::string param_line = "";
+      if (arg.is_mutated) {
+        param_line += fmt::format("*@param!* `{}: {}`", arg.name, arg.type);
+      } else if (arg.is_optional) {
+        param_line += fmt::format("*@param?* `{}: {}`", arg.name, arg.type);
+      } else if (arg.is_unused) {
+        param_line += fmt::format("*@param_* `{}: {}`", arg.name, arg.type);
+      } else {
+        param_line += fmt::format("*@param* `{}: {}`", arg.name, arg.type);
+      }
+      if (!arg.description.empty()) {
+        param_line += fmt::format(" - {}\n\n", arg.description);
+      } else {
+        param_line += "\n\n";
+      }
+      body += param_line;
+    }
+
+    markup.m_value = body;
+    LSPSpec::Hover hover_resp;
+    hover_resp.m_contents = markup;
+    return hover_resp;
+  }
+
+  return {};
+}
+}  // namespace lsp_handlers

lsp/handlers/text_document/hover.h

@@ -3,278 +3,16 @@
 #include <optional>
 #include <regex>
 
+#include "common/util/json_util.h"
+#include "common/util/string_util.h"
+
 #include "goalc/compiler/docs/DocTypes.h"
 #include "lsp/protocol/common_types.h"
 #include "lsp/protocol/hover.h"
 #include "lsp/state/data/mips_instructions.h"
 #include "lsp/state/workspace.h"
 
-bool is_number(const std::string& s) {
-  return !s.empty() && std::find_if(s.begin(), s.end(),
-                                    [](unsigned char c) { return !std::isdigit(c); }) == s.end();
-}
+namespace lsp_handlers {
+std::optional<json> hover(Workspace& workspace, int id, json raw_params);
 
-std::vector<std::string> og_method_names = {"new",      "delete", "print",    "inspect",  "length",
-                                            "asize-of", "copy",   "relocate", "mem-usage"};
-
-std::optional<LSPSpec::Hover> hover_handler_ir(Workspace& workspace,
-                                               const LSPSpec::TextDocumentPositionParams& params,
-                                               const WorkspaceIRFile& tracked_file) {
-  // See if it's an OpenGOAL symbol or a MIPS mnemonic
-  auto symbol_name = tracked_file.get_symbol_at_position(params.m_position);
-  auto token_at_pos = tracked_file.get_mips_instruction_at_position(params.m_position);
-  if (!symbol_name && !token_at_pos) {
-    return {};
-  }
-
-  LSPSpec::MarkupContent markup;
-  markup.m_kind = "markdown";
-
-  // TODO - try specifying the range so it highlights everything, ie. `c.lt.s`
-  // Prefer symbols
-  if (symbol_name) {
-    lg::debug("hover - symbol match - {}", symbol_name.value());
-    auto symbol_info = workspace.get_definition_info_from_all_types(symbol_name.value(),
-                                                                    tracked_file.m_all_types_uri);
-    if (symbol_info && symbol_info.value().docstring.has_value()) {
-      std::string docstring = symbol_info.value().docstring.value();
-      lg::debug("hover - symbol has docstring - {}", docstring);
-      // A docstring exists, print it!
-      // By convention, docstrings are assumed to be markdown, they support code-blocks everything
-      // the only thing extra we do, is replace [[<symbol>]] with links if available
-      std::unordered_map<std::string, std::string> symbol_replacements = {};
-      std::smatch match;
-
-      std::string::const_iterator searchStart(docstring.cbegin());
-      while (
-          std::regex_search(searchStart, docstring.cend(), match, std::regex("\\[{2}(.*)\\]{2}"))) {
-        // Have we already accounted for this symbol?
-        const auto& name = match[1].str();
-        if (symbol_replacements.count(name) != 0) {
-          continue;
-        }
-        // Get this symbols info
-        auto symbol_info =
-            workspace.get_definition_info_from_all_types(name, tracked_file.m_all_types_uri);
-        if (!symbol_info) {
-          symbol_replacements[name] = fmt::format("_{}_", name);
-        } else {
-          // Construct path
-          auto symbol_uri =
-              fmt::format("{}#L{}%2C{}", tracked_file.m_all_types_uri,
-                          symbol_info.value().definition_info->line_idx_to_display + 1,
-                          symbol_info.value().definition_info->pos_in_line);
-          symbol_replacements[name] = fmt::format("[{}]({})", name, symbol_uri);
-        }
-        searchStart = match.suffix().first;
-      }
-      // Replace all symbol occurences
-      for (const auto& [key, val] : symbol_replacements) {
-        docstring = std::regex_replace(docstring, std::regex("\\[{2}" + key + "\\]{2}"), val);
-      }
-
-      markup.m_value = docstring;
-      LSPSpec::Hover hover_resp;
-      hover_resp.m_contents = markup;
-      return hover_resp;
-    } else if (!token_at_pos) {
-      // Check if it's a number, and if so we'll do some numeric conversions
-      if (!is_number(symbol_name.value())) {
-        return {};
-      }
-      lg::debug("hover - numeric match - {}", symbol_name.value());
-      // Construct the body
-      std::string body = "";
-      uint32_t num = std::atoi(symbol_name.value().data());
-      // Assuming it comes in as Decimal
-      body += "| Base    | Value |\n";
-      body += "|---------|-------|\n";
-      body += fmt::format("| Decimal        | `{:d}` |\n", num);
-      body += fmt::format("| Hex            | `{:X}` |\n", num);
-      // TODO - would be nice to format as groups of 4
-      body += fmt::format("| Binary         | `{:b}` |\n", num);
-      if (num >= 16 && (num - 16) % 4 == 0) {
-        uint32_t method_id = (num - 16) / 4;
-        std::string method_name = "not built-in";
-        if (method_id <= 8) {
-          method_name = og_method_names.at(method_id);
-        }
-        body += fmt::format("| Method ID   | `{}` - `{}` |\n", method_id, method_name);
-      }
-      body += fmt::format("| Octal          | `{:o}` |\n", num);
-
-      markup.m_value = body;
-      LSPSpec::Hover hover_resp;
-      hover_resp.m_contents = markup;
-      return hover_resp;
-    }
-  }
-
-  // Otherwise, maybe it's a MIPS instruction
-  if (token_at_pos) {
-    lg::debug("hover - token match - {}", token_at_pos.value());
-    auto& token = token_at_pos.value();
-    std::transform(token.begin(), token.end(), token.begin(),
-                   [](unsigned char c) { return std::tolower(c); });
-    // Find the instruction, there are some edge-cases here where they could be multiple
-    // TODO - havn't addressed `bc` and such instructions!  Those need to be prefixed matched
-    std::vector<std::string> ee_instructions = {};
-    std::vector<std::string> vu_instructions = {};
-    for (const auto& instr : LSPData::MIPS_INSTRUCTION_LIST) {
-      auto mnemonic_lower = instr.mnemonic;
-      std::transform(mnemonic_lower.begin(), mnemonic_lower.end(), mnemonic_lower.begin(),
-                     [](unsigned char c) { return std::tolower(c); });
-      if (mnemonic_lower == token) {
-        if (instr.type == "ee") {
-          ee_instructions.push_back(fmt::format("- _{}_\n\n", instr.description));
-        } else {
-          vu_instructions.push_back(fmt::format("- _{}_\n\n", instr.description));
-        }
-      }
-    }
-
-    // Construct the body
-    std::string body = "";
-    if (!ee_instructions.empty()) {
-      body += "**EE Instructions**\n\n";
-      for (const auto& instr : ee_instructions) {
-        body += instr;
-      }
-      body += "___\n\n";
-    }
-
-    if (!vu_instructions.empty()) {
-      body += "**VU Instructions**\n\n";
-      for (const auto& instr : vu_instructions) {
-        body += instr;
-      }
-      body += "___\n\n";
-    }
-
-    markup.m_value = body;
-    LSPSpec::Hover hover_resp;
-    hover_resp.m_contents = markup;
-    return hover_resp;
-  }
-
-  return {};
-}
-
-std::string truncate_docstring(const std::string& docstring) {
-  std::string truncated = "";
-  const auto lines = str_util::split(docstring);
-  for (const auto& line : lines) {
-    const auto trimmed_line = str_util::ltrim(line);
-    if (str_util::starts_with(trimmed_line, "@")) {
-      break;
-    }
-    truncated += trimmed_line + "\n";
-  }
-  return truncated;
-}
-
-std::optional<json> hover_handler(Workspace& workspace, int /*id*/, json raw_params) {
-  auto params = raw_params.get<LSPSpec::TextDocumentPositionParams>();
-  auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
-
-  if (file_type == Workspace::FileType::OpenGOALIR) {
-    auto tracked_file = workspace.get_tracked_ir_file(params.m_textDocument.m_uri);
-    if (!tracked_file) {
-      return {};
-    }
-    return hover_handler_ir(workspace, params, tracked_file.value());
-  } else if (file_type == Workspace::FileType::OpenGOAL) {
-    auto tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
-    if (!tracked_file) {
-      return {};
-    }
-    // TODO - replace with AST usage instead of figuring out the symbol ourselves
-    const auto symbol = tracked_file->get_symbol_at_position(params.m_position);
-    if (!symbol) {
-      lg::debug("hover - no symbol");
-      return {};
-    }
-    // TODO - there is an issue with docstrings and overridden methods
-    const auto& symbol_info =
-        workspace.get_global_symbol_info(tracked_file.value(), symbol.value());
-    if (!symbol_info) {
-      lg::debug("hover - no symbol info - {}", symbol.value());
-      return {};
-    }
-    LSPSpec::MarkupContent markup;
-    markup.m_kind = "markdown";
-
-    const auto args =
-        Docs::get_args_from_docstring(symbol_info->args(), symbol_info->meta().docstring);
-    std::string signature = "";
-    bool takes_args = true;
-    if (symbol_info->kind() == SymbolInfo::Kind::FUNCTION) {
-      signature += "function ";
-    } else if (symbol_info->kind() == SymbolInfo::Kind::METHOD) {
-      signature += "method ";
-    } else if (symbol_info->kind() == SymbolInfo::Kind::MACRO) {
-      signature += "macro ";
-    } else {
-      takes_args = false;
-    }
-    // TODO - others useful, probably states?
-    signature += symbol.value();
-    if (takes_args) {
-      signature += "(";
-      for (int i = 0; i < (int)args.size(); i++) {
-        const auto& arg = args.at(i);
-        if (i == (int)args.size() - 1) {
-          signature += fmt::format("{}: {}", arg.name, arg.type);
-        } else {
-          signature += fmt::format("{}: {}, ", arg.name, arg.type);
-        }
-      }
-      signature += ")";
-      if (symbol_info->kind() == SymbolInfo::Kind::FUNCTION &&
-          workspace.get_symbol_typespec(tracked_file.value(), symbol.value())) {
-        signature +=
-            fmt::format(": {}", workspace.get_symbol_typespec(tracked_file.value(), symbol.value())
-                                    ->last_arg()
-                                    .base_type());
-      } else if (symbol_info->kind() == SymbolInfo::Kind::METHOD) {
-        signature += fmt::format(": {}", symbol_info->method_info().type.last_arg().base_type());
-      }
-    } else if (workspace.get_symbol_typespec(tracked_file.value(), symbol.value())) {
-      signature += fmt::format(
-          ": {}", workspace.get_symbol_typespec(tracked_file.value(), symbol.value())->base_type());
-    }
-
-    std::string body = fmt::format("```opengoal\n{}\n```\n\n", signature);
-    body += "___\n\n";
-    if (!symbol_info->meta().docstring.empty()) {
-      body += truncate_docstring(symbol_info->meta().docstring) + "\n\n";
-    }
-
-    // TODO - support @see/@returns/[[reference]]
-    for (const auto& arg : args) {
-      std::string param_line = "";
-      if (arg.is_mutated) {
-        param_line += fmt::format("*@param!* `{}: {}`", arg.name, arg.type);
-      } else if (arg.is_optional) {
-        param_line += fmt::format("*@param?* `{}: {}`", arg.name, arg.type);
-      } else if (arg.is_unused) {
-        param_line += fmt::format("*@param_* `{}: {}`", arg.name, arg.type);
-      } else {
-        param_line += fmt::format("*@param* `{}: {}`", arg.name, arg.type);
-      }
-      if (!arg.description.empty()) {
-        param_line += fmt::format(" - {}\n\n", arg.description);
-      } else {
-        param_line += "\n\n";
-      }
-      body += param_line;
-    }
-
-    markup.m_value = body;
-    LSPSpec::Hover hover_resp;
-    hover_resp.m_contents = markup;
-    return hover_resp;
-  }
-
-  return {};
-}
+}  // namespace lsp_handlers

lsp/handlers/text_document/type_hierarchy.cpp

@@ -0,0 +1,144 @@
+#include "type_hierarchy.h"
+
+#include "lsp/lsp_util.h"
+
+namespace lsp_handlers {
+std::optional<json> prepare_type_hierarchy(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::TypeHierarchyPrepareParams>();
+  const auto file_type = workspace.determine_filetype_from_uri(params.m_textDocument.m_uri);
+
+  if (file_type != Workspace::FileType::OpenGOAL) {
+    return nullptr;
+  }
+  auto maybe_tracked_file = workspace.get_tracked_og_file(params.m_textDocument.m_uri);
+  if (!maybe_tracked_file) {
+    return nullptr;
+  }
+  const auto& tracked_file = maybe_tracked_file.value().get();
+  const auto symbol = tracked_file.get_symbol_at_position(params.m_position);
+  if (!symbol) {
+    lg::debug("prepare_type_hierarchy - no symbol");
+    return nullptr;
+  }
+
+  const auto& symbol_info = workspace.get_global_symbol_info(tracked_file, symbol.value());
+  if (!symbol_info) {
+    lg::debug("prepare_type_hierarchy - no symbol info - {}", symbol.value());
+    return nullptr;
+  }
+
+  const auto& def_loc = workspace.get_symbol_def_location(tracked_file, symbol_info.value());
+  if (!def_loc) {
+    return nullptr;
+  }
+
+  auto type_item = LSPSpec::TypeHierarchyItem();
+  type_item.name = symbol.value();
+  // TODO - differentiate between struct and class perhaps
+  type_item.kind = LSPSpec::SymbolKind::Class;
+  if (symbol_info && !symbol_info.value()->m_docstring.empty()) {
+    type_item.detail = symbol_info.value()->m_docstring;
+  }
+  type_item.uri = lsp_util::uri_from_path(def_loc->file_path);
+  // TODO - this range is technically not entirely correct, we'd have to parse the defining file
+  // with an AST to get the true extent of the deftype.  But for this purpose, its not really needed
+  //
+  // HACK - the definition that our compiler stores is the form itself, so we will add
+  // the width of the prefix `(deftype ` to the char_index
+  // TODO - A better way would be to use the AST
+  type_item.range.m_start = {(uint32_t)def_loc->line_idx, (uint32_t)(def_loc->char_idx + 9)};
+  type_item.range.m_end = {(uint32_t)def_loc->line_idx,
+                           (uint32_t)(def_loc->char_idx + 9 + symbol.value().length())};
+  type_item.selectionRange.m_start = {(uint32_t)def_loc->line_idx,
+                                      (uint32_t)(def_loc->char_idx + 8)};
+  type_item.selectionRange.m_end = {(uint32_t)def_loc->line_idx,
+                                    (uint32_t)(def_loc->char_idx + 8 + symbol.value().length())};
+
+  json items = json::array();
+  items.push_back(type_item);
+  return items;
+}
+
+std::optional<json> supertypes_type_hierarchy(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::TypeHierarchySupertypesParams>();
+  const std::optional<GameVersion> game_version =
+      workspace.determine_game_version_from_uri(params.item.uri);
+  if (!game_version) {
+    return nullptr;
+  }
+  const auto& parent_type_path =
+      workspace.get_symbols_parent_type_path(params.item.name, game_version.value());
+  json items = json::array();
+  for (const auto& parent_type : parent_type_path) {
+    if (std::get<0>(parent_type) == params.item.name) {
+      continue;  // skip the item itself
+    }
+    auto type_item = LSPSpec::TypeHierarchyItem();
+    type_item.name = std::get<0>(parent_type);
+    // TODO - differentiate between struct and class perhaps
+    type_item.kind = LSPSpec::SymbolKind::Class;
+    if (!std::get<1>(parent_type).empty()) {
+      type_item.detail = std::get<1>(parent_type);
+    }
+    const auto& def_loc = std::get<2>(parent_type);
+    type_item.uri = def_loc.filename;
+    // TODO - this range is technically not entirely correct, we'd have to parse the defining file
+    // with an AST to get the true entent of the deftype.  But for this purpose, its not really
+    // needed
+    //
+    // HACK - the definition that our compiler stores is the form itself, so we will add
+    // the width of the prefix `(deftype ` to the char_index
+    // TODO - A better way would be to use the AST
+    type_item.range.m_start = {(uint32_t)def_loc.line_idx, (uint32_t)(def_loc.char_idx + 9)};
+    type_item.range.m_end = {(uint32_t)def_loc.line_idx,
+                             (uint32_t)(def_loc.char_idx + 9 + std::get<0>(parent_type).length())};
+    type_item.selectionRange.m_start = {(uint32_t)def_loc.line_idx,
+                                        (uint32_t)(def_loc.char_idx + 8)};
+    type_item.selectionRange.m_end = {
+        (uint32_t)def_loc.line_idx,
+        (uint32_t)(def_loc.char_idx + 8 + std::get<0>(parent_type).length())};
+    items.push_back(type_item);
+  }
+  return items;
+}
+
+std::optional<json> subtypes_type_hierarchy(Workspace& workspace, int /*id*/, json raw_params) {
+  auto params = raw_params.get<LSPSpec::TypeHierarchySupertypesParams>();
+  const std::optional<GameVersion> game_version =
+      workspace.determine_game_version_from_uri(params.item.uri);
+  if (!game_version) {
+    return nullptr;
+  }
+  const auto& parent_type_path =
+      workspace.get_types_subtypes(params.item.name, game_version.value());
+  json items = json::array();
+  for (const auto& parent_type : parent_type_path) {
+    auto type_item = LSPSpec::TypeHierarchyItem();
+    type_item.name = std::get<0>(parent_type);
+    // TODO - differentiate between struct and class perhaps
+    type_item.kind = LSPSpec::SymbolKind::Class;
+    if (!std::get<1>(parent_type).empty()) {
+      type_item.detail = std::get<1>(parent_type);
+    }
+    const auto& def_loc = std::get<2>(parent_type);
+    type_item.uri = def_loc.filename;
+    // TODO - this range is technically not entirely correct, we'd have to parse the defining file
+    // with an AST to get the true entent of the deftype.  But for this purpose, its not really
+    // needed
+    //
+    // HACK - the definition that our compiler stores is the form itself, so we will add
+    // the width of the prefix `(deftype ` to the char_index
+    // TODO - A better way would be to use the AST
+    type_item.range.m_start = {(uint32_t)def_loc.line_idx, (uint32_t)(def_loc.char_idx + 9)};
+    type_item.range.m_end = {(uint32_t)def_loc.line_idx,
+                             (uint32_t)(def_loc.char_idx + 9 + std::get<0>(parent_type).length())};
+    type_item.selectionRange.m_start = {(uint32_t)def_loc.line_idx,
+                                        (uint32_t)(def_loc.char_idx + 8)};
+    type_item.selectionRange.m_end = {
+        (uint32_t)def_loc.line_idx,
+        (uint32_t)(def_loc.char_idx + 8 + std::get<0>(parent_type).length())};
+    items.push_back(type_item);
+  }
+  return items;
+}
+}  // namespace lsp_handlers

lsp/handlers/text_document/type_hierarchy.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <optional>
+
+#include "common/util/json_util.h"
+
+#include "lsp/protocol/common_types.h"
+#include "lsp/protocol/type_hierarchy.h"
+#include "lsp/state/workspace.h"
+
+namespace lsp_handlers {
+
+std::optional<json> prepare_type_hierarchy(Workspace& workspace, int id, json raw_params);
+
+std::optional<json> supertypes_type_hierarchy(Workspace& workspace, int id, json raw_params);
+
+std::optional<json> subtypes_type_hierarchy(Workspace& workspace, int id, json raw_params);
+}  // namespace lsp_handlers

lsp/lsp_util.cpp

@@ -0,0 +1,83 @@
+#include "lsp_util.h"
+
+#include <sstream>
+
+#include "common/util/string_util.h"
+
+#include "fmt/core.h"
+
+namespace lsp_util {
+
+std::string url_encode(const std::string& value) {
+  std::ostringstream escaped;
+  escaped.fill('0');
+  escaped << std::hex;
+
+  for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
+    std::string::value_type c = (*i);
+
+    // Keep alphanumeric and other accepted characters intact
+    if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') {
+      escaped << c;
+      continue;
+    }
+
+    // Any other characters are percent-encoded
+    escaped << std::uppercase;
+    escaped << '%' << std::setw(2) << int((unsigned char)c);
+    escaped << std::nouppercase;
+  }
+
+  return escaped.str();
+}
+
+std::string url_decode(const std::string& input) {
+  std::ostringstream decoded;
+
+  for (std::size_t i = 0; i < input.length(); ++i) {
+    if (input[i] == '%') {
+      // Check if there are enough characters remaining
+      if (i + 2 < input.length()) {
+        // Convert the next two characters after '%' into an integer value
+        std::istringstream hexStream(input.substr(i + 1, 2));
+        int hexValue = 0;
+        hexStream >> std::hex >> hexValue;
+
+        // Append the decoded character to the result
+        decoded << static_cast<char>(hexValue);
+
+        // Skip the next two characters
+        i += 2;
+      }
+    } else if (input[i] == '+') {
+      // Replace '+' with space character ' '
+      decoded << ' ';
+    } else {
+      // Append the character as is
+      decoded << input[i];
+    }
+  }
+
+  return decoded.str();
+}
+
+LSPSpec::DocumentUri uri_from_path(fs::path path) {
+  auto path_str = file_util::convert_to_unix_path_separators(path.string());
+  // vscode works with proper URL encoded URIs for file paths
+  // which means we have to roll our own...
+  path_str = url_encode(path_str);
+  return fmt::format("file:///{}", path_str);
+}
+
+std::string uri_to_path(const LSPSpec::DocumentUri& uri) {
+  auto decoded_uri = url_decode(uri);
+  if (str_util::starts_with(decoded_uri, "file:///")) {
+#ifdef _WIN32
+    decoded_uri = decoded_uri.substr(8);
+#else
+    decoded_uri = decoded_uri.substr(7);
+#endif
+  }
+  return decoded_uri;
+}
+}  // namespace lsp_util

lsp/lsp_util.h

@@ -0,0 +1,13 @@
+#pragma once
+#include <string>
+
+#include "common/util/FileUtil.h"
+
+#include "protocol/common_types.h"
+
+namespace lsp_util {
+std::string url_encode(const std::string& value);
+std::string url_decode(const std::string& input);
+LSPSpec::DocumentUri uri_from_path(fs::path path);
+std::string uri_to_path(const LSPSpec::DocumentUri& uri);
+};  // namespace lsp_util

lsp/main.cpp

@@ -55,6 +55,29 @@ void setup_logging(bool verbose, std::string log_file, bool disable_ansi_colors)
   lg::initialize();
 }
 
+std::string temp_url_encode(const std::string& value) {
+  std::ostringstream escaped;
+  escaped.fill('0');
+  escaped << std::hex;
+
+  for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
+    std::string::value_type c = (*i);
+
+    // Keep alphanumeric and other accepted characters intact
+    if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') {
+      escaped << c;
+      continue;
+    }
+
+    // Any other characters are percent-encoded
+    escaped << std::uppercase;
+    escaped << '%' << std::setw(2) << int((unsigned char)c);
+    escaped << std::nouppercase;
+  }
+
+  return escaped.str();
+}
+
 int main(int argc, char** argv) {
   ArgumentGuard u8_guard(argc, argv);
 
@@ -72,6 +95,7 @@ int main(int argc, char** argv) {
   CLI11_PARSE(app, argc, argv);
 
   AppState appstate;
+
   LSPRouter lsp_router;
   appstate.verbose = verbose;
   try {
@@ -89,6 +113,11 @@ int main(int argc, char** argv) {
   _setmode(_fileno(stdin), _O_BINARY);
 #endif
 
+  // TODO - make the server check for the process id of the extension host and exit itself if that
+  // process goes away (the process id comes on the command line as an argument and in the
+  // initialize request). This is what we do in all our servers since the extension host could die
+  // unexpected as well.
+
   try {
     char c;
     MessageBuffer message_buffer;

lsp/protocol/common_types.cpp

@@ -9,6 +9,11 @@ void LSPSpec::from_json(const json& j, Position& obj) {
   j.at("character").get_to(obj.m_character);
 }
 
+LSPSpec::Range::Range(Position start, Position end) : m_start(start), m_end(end) {}
+
+LSPSpec::Range::Range(uint32_t line, uint32_t character)
+    : m_start({line, character}), m_end({line, character}) {}
+
 void LSPSpec::to_json(json& j, const Range& obj) {
   // TODO - not sure if this works yet, but nice if it does!
   j = json{{"start", obj.m_start}, {"end", obj.m_end}};

lsp/protocol/common_types.h

@@ -28,6 +28,11 @@ void from_json(const json& j, Position& obj);
 struct Range {
   Position m_start;
   Position m_end;
+
+  Range(){};
+  Range(Position start, Position end);
+  // point constructor
+  Range(uint32_t line, uint32_t character);
 };
 void to_json(json& j, const Range& obj);
 void from_json(const json& j, Range& obj);

lsp/protocol/completion.cpp

@@ -1,14 +1,65 @@
 #include "completion.h"
 
 void LSPSpec::to_json(json& j, const CompletionParams& obj) {
-  j = json{{"textDocument", obj.m_textDocument}, {"position", obj.m_position}};
+  json_serialize(textDocument);
+  json_serialize(position);
 }
 
 void LSPSpec::from_json(const json& j, CompletionParams& obj) {
-  j.at("textDocument").get_to(obj.m_textDocument);
-  j.at("position").get_to(obj.m_position);
+  json_deserialize_if_exists(textDocument);
+  json_deserialize_if_exists(position);
 }
 
-void LSPSpec::to_json(json& /*j*/, const CompletionList& /*obj*/) {}
+void LSPSpec::to_json(json& j, const CompletionItemLabelDetails& obj) {
+  json_serialize_optional(detail);
+  json_serialize_optional(description);
+}
 
-void LSPSpec::from_json(const json& /*j*/, CompletionList& /*obj*/) {}
+void LSPSpec::from_json(const json& j, CompletionItemLabelDetails& obj) {
+  json_deserialize_optional_if_exists(detail);
+  json_deserialize_optional_if_exists(description);
+}
+
+void LSPSpec::to_json(json& j, const CompletionItem& obj) {
+  json_serialize(label);
+  json_serialize_optional(labelDetails);
+  json_serialize_optional(kind);
+  json_serialize_optional(tags);
+  json_serialize_optional(detail);
+  json_serialize_optional(documentation);
+  json_serialize_optional(preselect);
+  json_serialize_optional(sortText);
+  json_serialize_optional(filterText);
+  json_serialize_optional(insertText);
+  json_serialize_optional(textEdit);
+  json_serialize_optional(textEditText);
+  json_serialize_optional(additionalTextEdits);
+  json_serialize_optional(commitCharacters);
+}
+
+void LSPSpec::from_json(const json& j, CompletionItem& obj) {
+  json_deserialize_if_exists(label);
+  json_deserialize_optional_if_exists(labelDetails);
+  json_deserialize_optional_if_exists(kind);
+  json_deserialize_optional_if_exists(tags);
+  json_deserialize_optional_if_exists(detail);
+  json_deserialize_optional_if_exists(documentation);
+  json_deserialize_optional_if_exists(preselect);
+  json_deserialize_optional_if_exists(sortText);
+  json_deserialize_optional_if_exists(filterText);
+  json_deserialize_optional_if_exists(insertText);
+  json_deserialize_optional_if_exists(textEdit);
+  json_deserialize_optional_if_exists(textEditText);
+  json_deserialize_optional_if_exists(additionalTextEdits);
+  json_deserialize_optional_if_exists(commitCharacters);
+}
+
+void LSPSpec::to_json(json& j, const CompletionList& obj) {
+  json_serialize(isIncomplete);
+  json_serialize(items);
+}
+
+void LSPSpec::from_json(const json& j, CompletionList& obj) {
+  json_deserialize_if_exists(isIncomplete);
+  json_deserialize_if_exists(items);
+}

lsp/protocol/completion.h

@@ -20,10 +20,10 @@ enum class CompletionTriggerKind {
 
 // TODO - look into inheriting structs?
 struct CompletionParams {
-  /// @brief The text document.
-  TextDocumentIdentifier m_textDocument;
-  /// @brief The position inside the text document.
-  Position m_position;
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+  /// The position inside the text document.
+  Position position;
 };
 
 void to_json(json& j, const CompletionParams& obj);
@@ -40,6 +40,9 @@ struct CompletionItemLabelDetails {
   std::optional<std::string> description;
 };
 
+void to_json(json& j, const CompletionItemLabelDetails& obj);
+void from_json(const json& j, CompletionItemLabelDetails& obj);
+
 /// @brief The kind of a completion entry.
 enum class CompletionItemKind {
   Text = 1,
@@ -95,8 +98,9 @@ struct CompletionItem {
   /// information.
   std::optional<std::string> detail;
   /// A human-readable string that represents a doc-comment.
+  /// TODO - can also be MarkupContent
   std::optional<std::string> documentation;
-  // NOTE - skipped deprecated
+  // NOTE - skipped deprecated (because it's deprecated!)
   /// Select this item when showing.
   ///
   /// *Note* that only one completion item can be selected and that the tool / client decides which
@@ -108,17 +112,104 @@ struct CompletionItem {
   /// A string that should be used when filtering a set of completion items. When omitted the label
   /// is used as the  filter text for this item.
   std::optional<std::string> filterText;
-  // TODO - a lot of other fields...
+  /// A string that should be inserted into a document when selecting
+  /// this completion. When omitted the label is used as the insert text
+  /// for this item.
+  ///
+  /// The `insertText` is subject to interpretation by the client side.
+  /// Some tools might not take the string literally. For example
+  /// VS Code when code complete is requested in this example
+  /// `con<cursor position>` and a completion item with an `insertText` of
+  /// `console` is provided it will only insert `sole`. Therefore it is
+  /// recommended to use `textEdit` instead since it avoids additional client
+  /// side interpretation.
+  std::optional<std::string> insertText;
+  /// The format of the insert text. The format applies to both the
+  /// `insertText` property and the `newText` property of a provided
+  /// `textEdit`. If omitted defaults to `InsertTextFormat.PlainText`.
+  ///
+  /// Please note that the insertTextFormat doesn't apply to
+  /// `additionalTextEdits`.
+  // TODO - std::optional<InsertTextFormat> insertTextFormat;
+  /// How whitespace and indentation is handled during completion
+  /// item insertion. If not provided the client's default value depends on
+  /// the `textDocument.completion.insertTextMode` client capability.
+  ///
+  /// @since 3.16.0
+  /// @since 3.17.0 - support for `textDocument.completion.insertTextMode`
+  // TODO - std::optional<InsertTextMode> insertTextMode;
+  /// An edit which is applied to a document when selecting this completion.
+  /// When an edit is provided the value of `insertText` is ignored.
+  ///
+  /// *Note:* The range of the edit must be a single line range and it must
+  /// contain the position at which completion has been requested.
+  ///
+  /// Most editors support two different operations when accepting a completion
+  /// item. One is to insert a completion text and the other is to replace an
+  /// existing text with a completion text. Since this can usually not be
+  /// predetermined by a server it can report both ranges. Clients need to
+  /// signal support for `InsertReplaceEdit`s via the
+  /// `textDocument.completion.completionItem.insertReplaceSupport` client
+  /// capability property.
+  ///
+  /// *Note 1:* The text edit's range as well as both ranges from an insert
+  /// replace edit must be a [single line] and they must contain the position
+  /// at which completion has been requested.
+  /// *Note 2:* If an `InsertReplaceEdit` is returned the edit's insert range
+  /// must be a prefix of the edit's replace range, that means it must be
+  /// contained and starting at the same position.
+  ///
+  /// @since 3.16.0 additional type `InsertReplaceEdit`
+  /// TODO - can also be InsertReplaceEdit
+  std::optional<TextEdit> textEdit;
+  /// The edit text used if the completion item is part of a CompletionList and
+  /// CompletionList defines an item default for the text edit range.
+  ///
+  /// Clients will only honor this property if they opt into completion list
+  /// item defaults using the capability `completionList.itemDefaults`.
+  ///
+  /// If not provided and a list's default range is provided the label
+  /// property is used as a text.
+  ///
+  /// @since 3.17.0
+  std::optional<std::string> textEditText;
+  /// An optional array of additional text edits that are applied when
+  /// selecting this completion. Edits must not overlap (including the same
+  /// insert position) with the main edit nor with themselves.
+  ///
+  /// Additional text edits should be used to change text unrelated to the
+  /// current cursor position (for example adding an import statement at the
+  /// top of the file if the completion item will insert an unqualified type).
+  std::optional<std::vector<TextEdit>> additionalTextEdits;
+  /// An optional set of characters that when pressed while this completion is
+  /// active will accept it first and then type that character. *Note* that all
+  /// commit characters should have `length=1` and that superfluous characters
+  /// will be ignored.
+  std::optional<std::vector<std::string>> commitCharacters;
+  /// An optional command that is executed *after* inserting this completion.
+  /// *Note* that additional modifications to the current document should be
+  /// described with the additionalTextEdits-property.
+  // TODO - std::optional<Command> command;
+  /// A data entry field that is preserved on a completion item between
+  /// a completion and a completion resolve request.
+  // TODO - LSPAny for data
 };
 
+void to_json(json& j, const CompletionItem& obj);
+void from_json(const json& j, CompletionItem& obj);
+
+// Represents a collection of [completion items](#CompletionItem) to be
+// presented in the editor.
 struct CompletionList {
-  /// This list is not complete. Further typing should result in recomputing this list.
+  /// This list is not complete. Further typing should result in recomputing
+  /// this list.
   ///
-  /// Recomputed lists have all their items replaced (not appended) in the incomplete completion
-  /// sessions.
-  bool m_isIncomplete;
+  /// Recomputed lists have all their items replaced (not appended) in the
+  /// incomplete completion sessions.
+  bool isIncomplete;
+  // TODO - do itemDefaults
   /// The completion items.
-  std::vector<CompletionItem> m_items;
+  std::vector<CompletionItem> items;
 };
 
 void to_json(json& j, const CompletionList& obj);

lsp/protocol/document_synchronization.cpp

@@ -32,3 +32,13 @@ void LSPSpec::to_json(json& j, const DidCloseTextDocumentParams& obj) {
 void LSPSpec::from_json(const json& j, DidCloseTextDocumentParams& obj) {
   j.at("textDocument").get_to(obj.m_textDocument);
 }
+
+void LSPSpec::to_json(json& j, const WillSaveTextDocumentParams& obj) {
+  json_serialize(textDocument);
+  json_serialize(reason);
+}
+
+void LSPSpec::from_json(const json& j, WillSaveTextDocumentParams& obj) {
+  json_deserialize_if_exists(textDocument);
+  json_deserialize_if_exists(reason);
+}

lsp/protocol/document_synchronization.h

@@ -32,4 +32,24 @@ struct DidCloseTextDocumentParams {
 void to_json(json& j, const DidCloseTextDocumentParams& obj);
 void from_json(const json& j, DidCloseTextDocumentParams& obj);
 
+enum class TextDocumentSaveReason {
+  // Manually triggered, e.g. by the user pressing save, by starting debugging, or by an API call.
+  Manual = 1,
+  // Automatic after a delay.
+  AfterDelay = 2,
+  // When the editor lost focus.
+  FocusOut = 3,
+};
+
+// The parameters send in a will save text document notification.
+struct WillSaveTextDocumentParams {
+  // The document that will be saved.
+  TextDocumentIdentifier textDocument;
+  // The 'TextDocumentSaveReason'.
+  TextDocumentSaveReason reason;
+};
+
+void to_json(json& j, const WillSaveTextDocumentParams& obj);
+void from_json(const json& j, WillSaveTextDocumentParams& obj);
+
 }  // namespace LSPSpec

lsp/protocol/progress_report.cpp

@@ -8,7 +8,7 @@ void LSPSpec::from_json(const json& j, WorkDoneProgressCreateParams& obj) {
   json_deserialize_if_exists(token);
 }
 
-void LSPSpec::to_json(json& j, const ProgressPayloadBegin& obj) {
+void LSPSpec::to_json(json& j, const WorkDoneProgressBegin& obj) {
   json_serialize(kind);
   json_serialize(title);
   json_serialize(cancellable);
@@ -16,7 +16,7 @@ void LSPSpec::to_json(json& j, const ProgressPayloadBegin& obj) {
   json_serialize_optional(percentage);
 }
 
-void LSPSpec::from_json(const json& j, ProgressPayloadBegin& obj) {
+void LSPSpec::from_json(const json& j, WorkDoneProgressBegin& obj) {
   json_deserialize_if_exists(kind);
   json_deserialize_if_exists(title);
   json_deserialize_if_exists(cancellable);
@@ -24,56 +24,43 @@ void LSPSpec::from_json(const json& j, ProgressPayloadBegin& obj) {
   json_deserialize_optional_if_exists(percentage);
 }
 
-void LSPSpec::to_json(json& j, const ProgressParamsBegin& obj) {
-  json_serialize(token);
-  json_serialize(value);
-}
-
-void LSPSpec::from_json(const json& j, ProgressParamsBegin& obj) {
-  json_deserialize_if_exists(token);
-  json_deserialize_if_exists(value);
-}
-
-void LSPSpec::to_json(json& j, const ProgressPayloadReport& obj) {
+void LSPSpec::to_json(json& j, const WorkDoneProgressReport& obj) {
   json_serialize(kind);
   json_serialize(cancellable);
   json_serialize_optional(message);
   json_serialize_optional(percentage);
 }
 
-void LSPSpec::from_json(const json& j, ProgressPayloadReport& obj) {
+void LSPSpec::from_json(const json& j, WorkDoneProgressReport& obj) {
   json_deserialize_if_exists(kind);
   json_deserialize_if_exists(cancellable);
   json_deserialize_optional_if_exists(message);
   json_deserialize_optional_if_exists(percentage);
 }
 
-void LSPSpec::to_json(json& j, const ProgressParamsReport& obj) {
-  json_serialize(token);
-  json_serialize(value);
-}
-
-void LSPSpec::from_json(const json& j, ProgressParamsReport& obj) {
-  json_deserialize_if_exists(token);
-  json_deserialize_if_exists(value);
-}
-
-void LSPSpec::to_json(json& j, const ProgressPayloadEnd& obj) {
+void LSPSpec::to_json(json& j, const WorkDoneProgressEnd& obj) {
   json_serialize(kind);
   json_serialize_optional(message);
 }
 
-void LSPSpec::from_json(const json& j, ProgressPayloadEnd& obj) {
+void LSPSpec::from_json(const json& j, WorkDoneProgressEnd& obj) {
   json_deserialize_if_exists(kind);
   json_deserialize_optional_if_exists(message);
 }
 
-void LSPSpec::to_json(json& j, const ProgressParamsEnd& obj) {
+void LSPSpec::to_json(json& j, const ProgressNotificationPayload& obj) {
   json_serialize(token);
-  json_serialize(value);
+  if (obj.beginValue) {
+    j["value"] = obj.beginValue.value();
+  } else if (obj.reportValue) {
+    j["value"] = obj.reportValue.value();
+  } else {
+    j["value"] = obj.endValue.value();
+  }
 }
 
-void LSPSpec::from_json(const json& j, ProgressParamsEnd& obj) {
+void LSPSpec::from_json(const json& j, ProgressNotificationPayload& obj) {
   json_deserialize_if_exists(token);
-  json_deserialize_if_exists(value);
+  // TODO - not needed, but if so -- deserialize 'value', it's possible to figure out which is the
+  // right one
 }

lsp/protocol/progress_report.h

@@ -11,7 +11,7 @@ struct WorkDoneProgressCreateParams {
 void to_json(json& j, const WorkDoneProgressCreateParams& obj);
 void from_json(const json& j, WorkDoneProgressCreateParams& obj);
 
-struct ProgressPayloadBegin {
+struct WorkDoneProgressBegin {
   std::string kind = "begin";
   // Mandatory title of the progress operation. Used to briefly inform about
   // the kind of operation being performed.
@@ -36,20 +36,10 @@ struct ProgressPayloadBegin {
   // that are not following this rule. The value range is [0, 100]
   std::optional<uint32_t> percentage;
 };
-void to_json(json& j, const ProgressPayloadBegin& obj);
-void from_json(const json& j, ProgressPayloadBegin& obj);
+void to_json(json& j, const WorkDoneProgressBegin& obj);
+void from_json(const json& j, WorkDoneProgressBegin& obj);
 
-struct ProgressParamsBegin {
-  // The progress token provided by the client or server.
-  std::string token;
-  // Payload
-  ProgressPayloadBegin value;
-};
-
-void to_json(json& j, const ProgressParamsBegin& obj);
-void from_json(const json& j, ProgressParamsBegin& obj);
-
-struct ProgressPayloadReport {
+struct WorkDoneProgressReport {
   std::string kind = "report";
   // Controls enablement state of a cancel button. This property is only valid
   // if a cancel button got requested in the `WorkDoneProgressBegin` payload.
@@ -71,35 +61,25 @@ struct ProgressPayloadReport {
   // that are not following this rule. The value range is [0, 100]
   std::optional<uint32_t> percentage;
 };
-void to_json(json& j, const ProgressPayloadReport& obj);
-void from_json(const json& j, ProgressPayloadReport& obj);
+void to_json(json& j, const WorkDoneProgressReport& obj);
+void from_json(const json& j, WorkDoneProgressReport& obj);
 
-struct ProgressParamsReport {
-  // The progress token provided by the client or server.
-  std::string token;
-  // Payload
-  ProgressPayloadReport value;
-};
-
-void to_json(json& j, const ProgressParamsReport& obj);
-void from_json(const json& j, ProgressParamsReport& obj);
-
-struct ProgressPayloadEnd {
+struct WorkDoneProgressEnd {
   std::string kind = "end";
   // Optional, a final message indicating to for example indicate the outcome
   // of the operation.
   std::optional<std::string> message;
 };
-void to_json(json& j, const ProgressPayloadEnd& obj);
-void from_json(const json& j, ProgressPayloadEnd& obj);
+void to_json(json& j, const WorkDoneProgressEnd& obj);
+void from_json(const json& j, WorkDoneProgressEnd& obj);
 
-struct ProgressParamsEnd {
-  // The progress token provided by the client or server.
+struct ProgressNotificationPayload {
   std::string token;
-  // Payload
-  ProgressPayloadEnd value;
+  std::optional<WorkDoneProgressBegin> beginValue;
+  std::optional<WorkDoneProgressReport> reportValue;
+  std::optional<WorkDoneProgressEnd> endValue;
 };
+void to_json(json& j, const ProgressNotificationPayload& obj);
+void from_json(const json& j, ProgressNotificationPayload& obj);
 
-void to_json(json& j, const ProgressParamsEnd& obj);
-void from_json(const json& j, ProgressParamsEnd& obj);
 }  // namespace LSPSpec

lsp/protocol/type_hierarchy.cpp

@@ -0,0 +1,50 @@
+#include "type_hierarchy.h"
+
+#include "common/util/json_util.h"
+
+// TODO - there's gotta be a way to share json serialization/deserialization
+// figure it out _soon_
+void LSPSpec::to_json(json& j, const TypeHierarchyPrepareParams& obj) {
+  j = json{{"textDocument", obj.m_textDocument}, {"position", obj.m_position}};
+}
+
+void LSPSpec::from_json(const json& j, TypeHierarchyPrepareParams& obj) {
+  j.at("textDocument").get_to(obj.m_textDocument);
+  j.at("position").get_to(obj.m_position);
+}
+
+void LSPSpec::to_json(json& j, const TypeHierarchyItem& obj) {
+  json_serialize(name);
+  json_serialize(kind);
+  json_serialize_optional(tags);
+  json_serialize_optional(detail);
+  json_serialize(uri);
+  json_serialize(range);
+  json_serialize(selectionRange);
+}
+
+void LSPSpec::from_json(const json& j, TypeHierarchyItem& obj) {
+  json_deserialize_if_exists(name);
+  json_deserialize_if_exists(kind);
+  json_deserialize_optional_if_exists(tags);
+  json_deserialize_optional_if_exists(detail);
+  json_deserialize_if_exists(uri);
+  json_deserialize_if_exists(range);
+  json_deserialize_if_exists(selectionRange);
+}
+
+void LSPSpec::to_json(json& j, const TypeHierarchySupertypesParams& obj) {
+  json_serialize(item);
+}
+
+void LSPSpec::from_json(const json& j, TypeHierarchySupertypesParams& obj) {
+  json_deserialize_if_exists(item);
+}
+
+void LSPSpec::to_json(json& j, const TypeHierarchySubtypesParams& obj) {
+  json_serialize(item);
+}
+
+void LSPSpec::from_json(const json& j, TypeHierarchySubtypesParams& obj) {
+  json_deserialize_if_exists(item);
+}

lsp/protocol/type_hierarchy.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "common_types.h"
+
+#include "lsp/protocol/document_symbols.h"
+
+namespace LSPSpec {
+
+struct TypeHierarchyPrepareParams : TextDocumentPositionParams {};
+
+void to_json(json& j, const TypeHierarchyPrepareParams& obj);
+void from_json(const json& j, TypeHierarchyPrepareParams& obj);
+
+struct TypeHierarchyItem {
+  /// The name of this item.
+  std::string name;
+  /// The kind of this item.
+  SymbolKind kind;
+  /// Tags for this item.
+  std::optional<std::vector<SymbolTag>> tags;
+  /// More detail for this item, e.g. the signature of a function.
+  std::optional<std::string> detail;
+  /// The resource identifier of this item.
+  DocumentUri uri;
+  /// The range enclosing this symbol not including leading/trailing whitespace
+  /// but everything else, e.g. comments and code.
+  Range range;
+  /// The range that should be selected and revealed when this symbol is being
+  /// picked, e.g. the name of a function. Must be contained by the
+  /// `range` of this
+  Range selectionRange;
+  /// A data entry field that is preserved between a type hierarchy prepare and
+  /// supertypes or subtypes requests. It could also be used to identify the
+  /// type hierarchy in the server, helping improve the performance on
+  /// resolving supertypes and subtypes.
+  // ANY data;
+};
+
+void to_json(json& j, const TypeHierarchyItem& obj);
+void from_json(const json& j, TypeHierarchyItem& obj);
+
+struct TypeHierarchySupertypesParams {
+  TypeHierarchyItem item;
+};
+
+void to_json(json& j, const TypeHierarchySupertypesParams& obj);
+void from_json(const json& j, TypeHierarchySupertypesParams& obj);
+
+struct TypeHierarchySubtypesParams {
+  TypeHierarchyItem item;
+};
+
+void to_json(json& j, const TypeHierarchySubtypesParams& obj);
+void from_json(const json& j, TypeHierarchySubtypesParams& obj);
+
+}  // namespace LSPSpec

lsp/state/app.h

@@ -2,6 +2,7 @@
 
 #include "lsp/state/workspace.h"
 
+// TODO - remove this, not really benefiting (never going to have multiple appstates)
 struct AppState {
   Workspace workspace;
   bool verbose;

lsp/state/lsp_requester.cpp

@@ -42,35 +42,49 @@ void LSPRequester::send_notification(const json& params, const std::string& meth
   std::cout << request.c_str() << std::flush;
 }
 
-void LSPRequester::send_progress_create_request(const std::string& token,
-                                                const std::string& title) {
-  LSPSpec::WorkDoneProgressCreateParams params;
-  params.token = token;
-  send_request(params, "window/workDoneProgress/create");
-  LSPSpec::ProgressPayloadBegin beginPayload;
+void LSPRequester::send_progress_create_request(const std::string& title,
+                                                const std::string& message,
+                                                const int percentage) {
+  const std::string token = fmt::format("opengoal/{}", title);
+  LSPSpec::WorkDoneProgressCreateParams createRequest;
+  createRequest.token = token;
+  send_request(createRequest, "window/workDoneProgress/create");
+  LSPSpec::WorkDoneProgressBegin beginPayload;
   beginPayload.title = title;
-  LSPSpec::ProgressParamsBegin beginParams;
-  beginParams.token = token;
-  beginParams.value = beginPayload;
-  send_notification(beginParams, "$/progress");
+  beginPayload.cancellable = false;  // TODO - maybe one day
+  beginPayload.message = message;
+  if (percentage > 0) {
+    beginPayload.percentage = percentage;
+  }
+  LSPSpec::ProgressNotificationPayload notification;
+  notification.token = token;
+  notification.beginValue = beginPayload;
+  send_notification(notification, "$/progress");
 }
 
-void LSPRequester::send_progress_update_request(const std::string& token,
-                                                const std::string& message) {
-  LSPSpec::ProgressPayloadReport reportPayload;
+void LSPRequester::send_progress_update_request(const std::string& title,
+                                                const std::string& message,
+                                                const int percentage) {
+  const std::string token = fmt::format("opengoal/{}", title);
+  LSPSpec::WorkDoneProgressReport reportPayload;
+  reportPayload.cancellable = false;  // TODO - maybe one day
   reportPayload.message = message;
-  LSPSpec::ProgressParamsReport reportParams;
-  reportParams.token = token;
-  reportParams.value = reportPayload;
-  send_notification(reportParams, "$/progress");
+  if (percentage > 0) {
+    reportPayload.percentage = percentage;
+  }
+  LSPSpec::ProgressNotificationPayload notification;
+  notification.token = token;
+  notification.reportValue = reportPayload;
+  send_notification(notification, "$/progress");
 }
 
-void LSPRequester::send_progress_finish_request(const std::string& token,
+void LSPRequester::send_progress_finish_request(const std::string& title,
                                                 const std::string& message) {
-  LSPSpec::ProgressPayloadEnd endPayload;
+  const std::string token = fmt::format("opengoal/{}", title);
+  LSPSpec::WorkDoneProgressEnd endPayload;
   endPayload.message = message;
-  LSPSpec::ProgressParamsEnd endParams;
-  endParams.token = token;
-  endParams.value = endPayload;
-  send_notification(endParams, "$/progress");
+  LSPSpec::ProgressNotificationPayload notification;
+  notification.token = token;
+  notification.endValue = endPayload;
+  send_notification(notification, "$/progress");
 }

lsp/state/lsp_requester.h

@@ -9,9 +9,13 @@
 
 class LSPRequester {
  public:
-  void send_progress_create_request(const std::string& token, const std::string& title);
-  void send_progress_update_request(const std::string& token, const std::string& message);
-  void send_progress_finish_request(const std::string& token, const std::string& message);
+  void send_progress_create_request(const std::string& title,
+                                    const std::string& message,
+                                    const int percentage);
+  void send_progress_update_request(const std::string& title,
+                                    const std::string& message,
+                                    const int percentage);
+  void send_progress_finish_request(const std::string& title, const std::string& message);
 
  private:
   void send_request(const json& payload, const std::string& method);

lsp/state/workspace.cpp

@@ -1,86 +1,23 @@
 #include "workspace.h"
 
-#include <iomanip>
 #include <regex>
-#include <sstream>
 
 #include "common/log/log.h"
+#include "common/util/FileUtil.h"
+#include "common/util/ast_util.h"
 #include "common/util/string_util.h"
 
+#include "lsp/lsp_util.h"
 #include "lsp/protocol/common_types.h"
+#include "tree_sitter/api.h"
 
-std::string url_encode(const std::string& value) {
-  std::ostringstream escaped;
-  escaped.fill('0');
-  escaped << std::hex;
-
-  for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
-    std::string::value_type c = (*i);
-
-    // Keep alphanumeric and other accepted characters intact
-    if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') {
-      escaped << c;
-      continue;
-    }
-
-    // Any other characters are percent-encoded
-    escaped << std::uppercase;
-    escaped << '%' << std::setw(2) << int((unsigned char)c);
-    escaped << std::nouppercase;
-  }
-
-  return escaped.str();
+// Declare the `tree_sitter_opengoal` function, which is
+// implemented by the `tree-sitter-opengoal` library.
+extern "C" {
+extern const TSLanguage* tree_sitter_opengoal();
 }
 
-std::string url_decode(const std::string& input) {
-  std::ostringstream decoded;
-
-  for (std::size_t i = 0; i < input.length(); ++i) {
-    if (input[i] == '%') {
-      // Check if there are enough characters remaining
-      if (i + 2 < input.length()) {
-        // Convert the next two characters after '%' into an integer value
-        std::istringstream hexStream(input.substr(i + 1, 2));
-        int hexValue = 0;
-        hexStream >> std::hex >> hexValue;
-
-        // Append the decoded character to the result
-        decoded << static_cast<char>(hexValue);
-
-        // Skip the next two characters
-        i += 2;
-      }
-    } else if (input[i] == '+') {
-      // Replace '+' with space character ' '
-      decoded << ' ';
-    } else {
-      // Append the character as is
-      decoded << input[i];
-    }
-  }
-
-  return decoded.str();
-}
-
-LSPSpec::DocumentUri uri_from_path(fs::path path) {
-  auto path_str = file_util::convert_to_unix_path_separators(path.string());
-  // vscode works with proper URL encoded URIs for file paths
-  // which means we have to roll our own...
-  path_str = url_encode(path_str);
-  return fmt::format("file:///{}", path_str);
-}
-
-std::string uri_to_path(LSPSpec::DocumentUri uri) {
-  auto decoded_uri = url_decode(uri);
-  if (str_util::starts_with(decoded_uri, "file:///")) {
-#ifdef _WIN32
-    decoded_uri = decoded_uri.substr(8);
-#else
-    decoded_uri = decoded_uri.substr(7);
-#endif
-  }
-  return decoded_uri;
-}
+const TSLanguage* g_opengoalLang = tree_sitter_opengoal();
 
 Workspace::Workspace(){};
 Workspace::~Workspace(){};
@@ -111,18 +48,22 @@ Workspace::FileType Workspace::determine_filetype_from_uri(const LSPSpec::Docume
   return FileType::Unsupported;
 }
 
-std::optional<WorkspaceOGFile> Workspace::get_tracked_og_file(const LSPSpec::URI& file_uri) {
-  if (m_tracked_og_files.find(file_uri) == m_tracked_og_files.end()) {
-    return {};
+std::optional<std::reference_wrapper<WorkspaceOGFile>> Workspace::get_tracked_og_file(
+    const LSPSpec::URI& file_uri) {
+  auto it = m_tracked_og_files.find(file_uri);
+  if (it == m_tracked_og_files.end()) {
+    return std::nullopt;
   }
-  return m_tracked_og_files[file_uri];
+  return std::ref(it->second);
 }
 
-std::optional<WorkspaceIRFile> Workspace::get_tracked_ir_file(const LSPSpec::URI& file_uri) {
-  if (m_tracked_ir_files.count(file_uri) == 0) {
-    return {};
+std::optional<std::reference_wrapper<WorkspaceIRFile>> Workspace::get_tracked_ir_file(
+    const LSPSpec::URI& file_uri) {
+  auto it = m_tracked_ir_files.find(file_uri);
+  if (it == m_tracked_ir_files.end()) {
+    return std::nullopt;
   }
-  return m_tracked_ir_files[file_uri];
+  return std::ref(it->second);
 }
 
 std::optional<DefinitionMetadata> Workspace::get_definition_info_from_all_types(
@@ -143,18 +84,34 @@ std::optional<DefinitionMetadata> Workspace::get_definition_info_from_all_types(
 //
 // This is bad because jak 2 now uses some code from the jak1 folder, and also wouldn't be able to
 // be determined (jak1 or jak2?) if we had a proper 'common' folder(s).
-std::optional<GameVersion> determine_game_version_from_uri(const LSPSpec::DocumentUri& uri) {
-  const auto path = uri_to_path(uri);
+std::optional<GameVersion> Workspace::determine_game_version_from_uri(
+    const LSPSpec::DocumentUri& uri) {
+  const auto path = lsp_util::uri_to_path(uri);
   if (str_util::contains(path, "goal_src/jak1")) {
     return GameVersion::Jak1;
   } else if (str_util::contains(path, "goal_src/jak2")) {
     return GameVersion::Jak2;
+  } else if (str_util::contains(path, "goal_src/jak3")) {
+    return GameVersion::Jak3;
   }
   return {};
 }
 
-std::optional<SymbolInfo> Workspace::get_global_symbol_info(const WorkspaceOGFile& file,
-                                                            const std::string& symbol_name) {
+std::vector<std::shared_ptr<symbol_info::SymbolInfo>> Workspace::get_symbols_starting_with(
+    const GameVersion game_version,
+    const std::string& symbol_prefix) {
+  if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+    lg::debug("Compiler not instantiated for game version - {}",
+              version_to_game_name(game_version));
+    return {};
+  }
+  const auto& compiler = m_compiler_instances[game_version].get();
+  return compiler->lookup_symbol_info_by_prefix(symbol_prefix);
+}
+
+std::optional<std::shared_ptr<symbol_info::SymbolInfo>> Workspace::get_global_symbol_info(
+    const WorkspaceOGFile& file,
+    const std::string& symbol_name) {
   if (m_compiler_instances.find(file.m_game_version) == m_compiler_instances.end()) {
     lg::debug("Compiler not instantiated for game version - {}",
               version_to_game_name(file.m_game_version));
@@ -162,19 +119,21 @@ std::optional<SymbolInfo> Workspace::get_global_symbol_info(const WorkspaceOGFil
   }
   const auto& compiler = m_compiler_instances[file.m_game_version].get();
   const auto symbol_infos = compiler->lookup_exact_name_info(symbol_name);
-  if (!symbol_infos || symbol_infos->empty()) {
+  if (symbol_infos.empty()) {
     return {};
-  } else if (symbol_infos->size() > 1) {
+  } else if (symbol_infos.size() > 1) {
     // TODO - handle this (overriden methods is the main issue here)
-    lg::debug("Found symbol info, but found multiple infos - {}", symbol_infos->size());
+    lg::debug("Found symbol info, but found multiple infos - {}", symbol_infos.size());
     return {};
   }
-  const auto& symbol = symbol_infos->at(0);
+  const auto& symbol = symbol_infos.at(0);
   return symbol;
 }
 
-std::optional<TypeSpec> Workspace::get_symbol_typespec(const WorkspaceOGFile& file,
-                                                       const std::string& symbol_name) {
+// TODO - consolidate what is needed into `SymbolInfo`
+std::optional<std::pair<TypeSpec, Type*>> Workspace::get_symbol_typeinfo(
+    const WorkspaceOGFile& file,
+    const std::string& symbol_name) {
   if (m_compiler_instances.find(file.m_game_version) == m_compiler_instances.end()) {
     lg::debug("Compiler not instantiated for game version - {}",
               version_to_game_name(file.m_game_version));
@@ -183,32 +142,122 @@ std::optional<TypeSpec> Workspace::get_symbol_typespec(const WorkspaceOGFile& fi
   const auto& compiler = m_compiler_instances[file.m_game_version].get();
   const auto typespec = compiler->lookup_typespec(symbol_name);
   if (typespec) {
-    return typespec;
+    // NOTE - for some reason calling with the symbol's typespec and the symbol itself produces
+    // different results!
+    const auto full_type_info = compiler->type_system().lookup_type_no_throw(symbol_name);
+    if (full_type_info != nullptr) {
+      return std::make_pair(typespec.value(), full_type_info);
+    }
   }
   return {};
 }
 
-std::optional<Docs::DefinitionLocation> Workspace::get_symbol_def_location(
+std::optional<symbol_info::DefinitionLocation> Workspace::get_symbol_def_location(
     const WorkspaceOGFile& file,
-    const SymbolInfo& symbol_info) {
-  if (m_compiler_instances.find(file.m_game_version) == m_compiler_instances.end()) {
-    lg::debug("Compiler not instantiated for game version - {}",
-              version_to_game_name(file.m_game_version));
+    const std::shared_ptr<symbol_info::SymbolInfo> symbol_info) {
+  const auto& def_loc = symbol_info->m_def_location;
+  if (!def_loc) {
     return {};
   }
-  const auto& compiler = m_compiler_instances[file.m_game_version].get();
-  std::optional<Docs::DefinitionLocation> def_loc;
-  const auto& goos_info = compiler->get_goos().reader.db.get_short_info_for(symbol_info.src_form());
-  if (goos_info) {
-    Docs::DefinitionLocation new_def_loc;
-    new_def_loc.filename = uri_from_path(goos_info->filename);
-    new_def_loc.line_idx = goos_info->line_idx_to_display;
-    new_def_loc.char_idx = goos_info->pos_in_line;
-    def_loc = new_def_loc;
-  }
   return def_loc;
 }
 
+std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>>
+Workspace::get_symbols_parent_type_path(const std::string& symbol_name,
+                                        const GameVersion game_version) {
+  if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+    lg::debug("Compiler not instantiated for game version - {}",
+              version_to_game_name(game_version));
+    return {};
+  }
+
+  // name, docstring, def_loc
+  std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>> parents = {};
+
+  const auto& compiler = m_compiler_instances[game_version].get();
+  const auto parent_path = compiler->type_system().get_path_up_tree(symbol_name);
+  for (const auto& parent : parent_path) {
+    const auto symbol_infos = compiler->lookup_exact_name_info(parent);
+    if (symbol_infos.empty()) {
+      continue;
+    }
+    std::shared_ptr<symbol_info::SymbolInfo> symbol_info;
+    if (symbol_infos.size() > 1) {
+      for (const auto& info : symbol_infos) {
+        if (info->m_kind == symbol_info::Kind::TYPE) {
+          symbol_info = info;
+        }
+      }
+    } else {
+      symbol_info = symbol_infos.at(0);
+    }
+    if (!symbol_info) {
+      continue;
+    }
+    const auto& def_loc = symbol_info->m_def_location;
+    if (!def_loc) {
+      continue;
+    }
+    Docs::DefinitionLocation new_def_loc;
+    new_def_loc.filename = lsp_util::uri_from_path(def_loc->file_path);
+    new_def_loc.line_idx = def_loc->line_idx;
+    new_def_loc.char_idx = def_loc->char_idx;
+    parents.push_back({parent, symbol_info->m_docstring, new_def_loc});
+  }
+  return parents;
+}
+
+std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>>
+Workspace::get_types_subtypes(const std::string& symbol_name, const GameVersion game_version) {
+  if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+    lg::debug("Compiler not instantiated for game version - {}",
+              version_to_game_name(game_version));
+    return {};
+  }
+
+  // name, docstring, def_loc
+  std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>> subtypes = {};
+
+  const auto& compiler = m_compiler_instances[game_version].get();
+  const auto subtype_names =
+      compiler->type_system().search_types_by_parent_type_strict(symbol_name);
+  for (const auto& subtype_name : subtype_names) {
+    const auto symbol_infos = compiler->lookup_exact_name_info(subtype_name);
+    if (symbol_infos.empty()) {
+      continue;
+    } else if (symbol_infos.size() > 1) {
+      continue;
+    }
+    const auto& symbol_info = symbol_infos.at(0);
+    const auto& def_loc = symbol_info->m_def_location;
+    if (!def_loc) {
+      continue;
+    }
+    Docs::DefinitionLocation new_def_loc;
+    new_def_loc.filename = lsp_util::uri_from_path(def_loc->file_path);
+    new_def_loc.line_idx = def_loc->line_idx;
+    new_def_loc.char_idx = def_loc->char_idx;
+    subtypes.push_back({subtype_name, symbol_info->m_docstring, new_def_loc});
+  }
+  return subtypes;
+}
+
+std::unordered_map<std::string, s64> Workspace::get_enum_entries(const std::string& enum_name,
+                                                                 const GameVersion game_version) {
+  if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+    lg::debug("Compiler not instantiated for game version - {}",
+              version_to_game_name(game_version));
+    return {};
+  }
+
+  const auto& compiler = m_compiler_instances[game_version].get();
+  const auto enum_info = compiler->type_system().try_enum_lookup(enum_name);
+  if (!enum_info) {
+    return {};
+  }
+  return enum_info->entries();
+}
+
 void Workspace::start_tracking_file(const LSPSpec::DocumentUri& file_uri,
                                     const std::string& language_id,
                                     const std::string& content) {
@@ -225,33 +274,51 @@ void Workspace::start_tracking_file(const LSPSpec::DocumentUri& file_uri,
       }
     }
   } else if (language_id == "opengoal") {
+    if (m_tracked_og_files.find(file_uri) != m_tracked_og_files.end()) {
+      lg::debug("Already tracking - {}", file_uri);
+      return;
+    }
     auto game_version = determine_game_version_from_uri(file_uri);
     if (!game_version) {
       lg::debug("Could not determine game version from path - {}", file_uri);
       return;
     }
-    // TODO - this should happen on a separate thread so the LSP is not blocking during this lengthy
-    // step
+
     if (m_compiler_instances.find(*game_version) == m_compiler_instances.end()) {
       lg::debug(
           "first time encountering a OpenGOAL file for game version - {}, initializing a compiler",
           version_to_game_name(*game_version));
-      const auto project_path = file_util::try_get_project_path_from_path(uri_to_path(file_uri));
+      const auto project_path =
+          file_util::try_get_project_path_from_path(lsp_util::uri_to_path(file_uri));
       lg::debug("Detected project path - {}", project_path.value());
       if (!file_util::setup_project_path(project_path)) {
         lg::debug("unable to setup project path, not initializing a compiler");
         return;
       }
-      m_requester.send_progress_create_request("indexing-jak2", "Indexing - Jak 2");
+      const std::string progress_title =
+          fmt::format("Compiling {}", version_to_game_name_external(game_version.value()));
+      m_requester.send_progress_create_request(progress_title, "compiling project", -1);
       m_compiler_instances.emplace(game_version.value(),
                                    std::make_unique<Compiler>(game_version.value()));
-      // TODO - if this fails, annotate some errors, adjust progress
-      m_compiler_instances.at(*game_version)->run_front_end_on_string("(make-group \"all-code\")");
-      m_requester.send_progress_finish_request("indexing-jak2", "Indexed - Jak 2");
+      try {
+        // TODO - this should happen on a separate thread so the LSP is not blocking during this
+        // lengthy step
+        // TODO - make this a setting (disable indexing)
+        // TODO - ask water if there is a fancy way to reduce memory usage (disabling coloring,
+        // etc?)
+        m_compiler_instances.at(*game_version)
+            ->run_front_end_on_string("(make-group \"all-code\")");
+        m_requester.send_progress_finish_request(progress_title, "indexed");
+      } catch (std::exception& e) {
+        // TODO - If it fails, annotate errors (DIAGNOSTIC TODO)
+        m_requester.send_progress_finish_request(progress_title, "failed");
+        lg::debug("error when {}", progress_title);
+      }
     }
-    //  TODO - otherwise, just `ml` the file instead of rebuilding the entire thing
-    //  TODO - if the file fails to `ml`, annotate some errors
-    m_tracked_og_files[file_uri] = WorkspaceOGFile(content, *game_version);
+    m_tracked_og_files.emplace(file_uri, WorkspaceOGFile(file_uri, content, *game_version));
+    m_tracked_og_files[file_uri].update_symbols(
+        m_compiler_instances.at(*game_version)
+            ->lookup_symbol_info_by_file(lsp_util::uri_to_path(file_uri)));
   }
 }
 
@@ -260,7 +327,7 @@ void Workspace::update_tracked_file(const LSPSpec::DocumentUri& file_uri,
   lg::debug("potentially updating - {}", file_uri);
   // Check if the file is already tracked or not, this is done because change events don't give
   // language details it's assumed you are keeping track of that!
-  if (m_tracked_ir_files.count(file_uri) != 0) {
+  if (m_tracked_ir_files.find(file_uri) != m_tracked_ir_files.end()) {
     lg::debug("updating tracked IR file - {}", file_uri);
     WorkspaceIRFile file(content);
     m_tracked_ir_files[file_uri] = file;
@@ -274,52 +341,210 @@ void Workspace::update_tracked_file(const LSPSpec::DocumentUri& file_uri,
       all_types_file->m_game_version = file.m_game_version;
       all_types_file->update_type_system();
     }
-  }
-
-  if (m_tracked_all_types_files.count(file_uri) != 0) {
+  } else if (m_tracked_all_types_files.find(file_uri) != m_tracked_all_types_files.end()) {
     lg::debug("updating tracked all types file - {}", file_uri);
     // If the all-types file has changed, re-parse it
     // NOTE - this assumes its still for the same game version!
     m_tracked_all_types_files[file_uri]->update_type_system();
-  }
-};
-
-void Workspace::stop_tracking_file(const LSPSpec::DocumentUri& file_uri) {
-  if (m_tracked_ir_files.count(file_uri) != 0) {
-    m_tracked_ir_files.erase(file_uri);
-  }
-  if (m_tracked_all_types_files.count(file_uri) != 0) {
-    m_tracked_all_types_files.erase(file_uri);
+  } else if (m_tracked_og_files.find(file_uri) != m_tracked_og_files.end()) {
+    lg::debug("updating tracked OG file - {}", file_uri);
+    m_tracked_og_files[file_uri].parse_content(content);
+    // re-`ml` the file
+    const auto game_version = m_tracked_og_files[file_uri].m_game_version;
+    if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+      lg::debug("No compiler initialized for - {}", version_to_game_name(game_version));
+      return;
+    }
   }
 }
 
-WorkspaceOGFile::WorkspaceOGFile(const std::string& content, const GameVersion& game_version)
-    : m_content(content), m_game_version(game_version) {
-  const auto line_ending = file_util::get_majority_file_line_endings(content);
-  m_lines = str_util::split_string(content, line_ending);
-  lg::info("Added new OG file. {} lines with {} symbols and {} diagnostics", m_lines.size(),
-           m_symbols.size(), m_diagnostics.size());
+void Workspace::tracked_file_will_save(const LSPSpec::DocumentUri& file_uri) {
+  lg::debug("file will be saved - {}", file_uri);
+  if (m_tracked_og_files.find(file_uri) != m_tracked_og_files.end()) {
+    // goalc is not an incremental compiler (yet) so I believe it will be a better UX
+    // to re-compile on the file save, rather than as the user is typing
+    const auto game_version = m_tracked_og_files[file_uri].m_game_version;
+    if (m_compiler_instances.find(game_version) == m_compiler_instances.end()) {
+      lg::debug("No compiler initialized for - {}", version_to_game_name(game_version));
+      return;
+    }
+    CompilationOptions options;
+    options.filename = lsp_util::uri_to_path(file_uri);
+    // re-compile the file
+    m_compiler_instances.at(game_version)->asm_file(options);
+    // Update symbols for this specific file
+    const auto symbol_infos =
+        m_compiler_instances.at(game_version)->lookup_symbol_info_by_file(options.filename);
+    m_tracked_og_files[file_uri].update_symbols(symbol_infos);
+  }
+}
+
+void Workspace::update_global_index(const GameVersion game_version){
+    // TODO - project wide indexing potentially (ie. finding references)
+};
+
+void Workspace::stop_tracking_file(const LSPSpec::DocumentUri& file_uri) {
+  m_tracked_ir_files.erase(file_uri);
+  m_tracked_all_types_files.erase(file_uri);
+  m_tracked_og_files.erase(file_uri);
+}
+
+WorkspaceOGFile::WorkspaceOGFile(const LSPSpec::DocumentUri& uri,
+                                 const std::string& content,
+                                 const GameVersion& game_version)
+    : m_uri(uri), m_game_version(game_version), version(0) {
+  const auto [line_count, line_ending] =
+      file_util::get_majority_file_line_endings_and_count(content);
+  m_line_count = line_count;
+  m_line_ending = line_ending;
+  lg::info("Added new OG file. {} symbols and {} diagnostics", m_symbols.size(),
+           m_diagnostics.size());
+  parse_content(content);
+}
+
+void WorkspaceOGFile::parse_content(const std::string& content) {
+  m_content = content;
+  auto parser = ts_parser_new();
+  if (ts_parser_set_language(parser, g_opengoalLang)) {
+    // Get the AST for the current state of the file
+    // TODO - eventually, we should consider doing partial updates of the AST
+    // but right now the LSP just receives the entire document so that's a larger change.
+    m_ast.reset(ts_parser_parse_string(parser, NULL, m_content.c_str(), m_content.length()),
+                TreeSitterTreeDeleter());
+  }
+  ts_parser_delete(parser);
+}
+
+void WorkspaceOGFile::update_symbols(
+    const std::vector<std::shared_ptr<symbol_info::SymbolInfo>>& symbol_infos) {
+  m_symbols.clear();
+  // TODO - sorting by definition location would be nice (maybe VSCode already does this?)
+  for (const auto& symbol_info : symbol_infos) {
+    LSPSpec::DocumentSymbol lsp_sym;
+    lsp_sym.m_name = symbol_info->m_name;
+    lsp_sym.m_detail = symbol_info->m_docstring;
+    switch (symbol_info->m_kind) {
+      case symbol_info::Kind::CONSTANT:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Constant;
+        break;
+      case symbol_info::Kind::FUNCTION:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Function;
+        break;
+      case symbol_info::Kind::GLOBAL_VAR:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Variable;
+        break;
+      case symbol_info::Kind::MACRO:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Operator;
+        break;
+      case symbol_info::Kind::METHOD:
+        lsp_sym.m_name = fmt::format("{}::{}", symbol_info->m_type, symbol_info->m_name);
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Method;
+        break;
+      case symbol_info::Kind::TYPE:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Class;
+        break;
+      default:
+        lsp_sym.m_kind = LSPSpec::SymbolKind::Object;
+        break;
+    }
+    if (symbol_info->m_def_location) {
+      lsp_sym.m_range = LSPSpec::Range(symbol_info->m_def_location->line_idx,
+                                       symbol_info->m_def_location->char_idx);
+    } else {
+      lsp_sym.m_range = LSPSpec::Range(0, 0);
+    }
+    // TODO - would be nice to make this accurate but we don't store that info yet
+    lsp_sym.m_selectionRange = lsp_sym.m_range;
+    if (symbol_info->m_kind == symbol_info::Kind::TYPE) {
+      std::vector<LSPSpec::DocumentSymbol> type_symbols = {};
+      for (const auto& field : symbol_info->m_type_fields) {
+        LSPSpec::DocumentSymbol field_sym;
+        field_sym.m_name = field.name;
+        field_sym.m_detail = field.description;
+        if (field.is_array) {
+          field_sym.m_kind = LSPSpec::SymbolKind::Array;
+        } else {
+          field_sym.m_kind = LSPSpec::SymbolKind::Field;
+        }
+        // TODO - we don't store the line number for fields
+        field_sym.m_range = lsp_sym.m_range;
+        field_sym.m_selectionRange = lsp_sym.m_selectionRange;
+        type_symbols.push_back(field_sym);
+      }
+      for (const auto& method : symbol_info->m_type_methods) {
+        LSPSpec::DocumentSymbol method_sym;
+        method_sym.m_name = method.name;
+        method_sym.m_kind = LSPSpec::SymbolKind::Method;
+        // TODO - we don't store the line number for fields
+        method_sym.m_range = lsp_sym.m_range;
+        method_sym.m_selectionRange = lsp_sym.m_selectionRange;
+        type_symbols.push_back(method_sym);
+      }
+      for (const auto& state : symbol_info->m_type_states) {
+        LSPSpec::DocumentSymbol state_sym;
+        state_sym.m_name = state.name;
+        state_sym.m_kind = LSPSpec::SymbolKind::Event;
+        // TODO - we don't store the line number for fields
+        state_sym.m_range = lsp_sym.m_range;
+        state_sym.m_selectionRange = lsp_sym.m_selectionRange;
+        type_symbols.push_back(state_sym);
+      }
+      lsp_sym.m_children = type_symbols;
+    }
+    m_symbols.push_back(lsp_sym);
+  }
 }
 
 std::optional<std::string> WorkspaceOGFile::get_symbol_at_position(
     const LSPSpec::Position position) const {
-  // Split the line on typical word boundaries
-  std::string line = m_lines.at(position.m_line);
-  std::smatch matches;
-  std::regex regex("[\\w\\.\\-_!<>*?]+");
-  std::regex_token_iterator<std::string::iterator> rend;
-
-  std::regex_token_iterator<std::string::iterator> match(line.begin(), line.end(), regex);
-  while (match != rend) {
-    auto match_start = std::distance(line.begin(), match->first);
-    auto match_end = match_start + match->length();
-    if (position.m_character >= match_start && position.m_character <= match_end) {
-      return match->str();
+  if (m_ast) {
+    TSNode root_node = ts_tree_root_node(m_ast.get());
+    TSNode found_node =
+        ts_node_descendant_for_point_range(root_node, {position.m_line, position.m_character},
+                                           {position.m_line, position.m_character});
+    if (!ts_node_has_error(found_node)) {
+      uint32_t start = ts_node_start_byte(found_node);
+      uint32_t end = ts_node_end_byte(found_node);
+      const std::string node_str = m_content.substr(start, end - start);
+      lg::debug("AST SAP - {}", node_str);
+      const std::string node_name = ts_node_type(found_node);
+      if (node_name == "sym_name") {
+        return node_str;
+      }
+    } else {
+      // found_node = ts_node_child(found_node, 0);
+      // TODO - maybe get this one (but check if has an error)
+      return {};
     }
-    match++;
+  }
+  return {};
+}
+
+std::vector<OpenGOALFormResult> WorkspaceOGFile::search_for_forms_that_begin_with(
+    std::vector<std::string> prefix) const {
+  std::vector<OpenGOALFormResult> results = {};
+  if (!m_ast) {
+    return results;
   }
 
-  return {};
+  TSNode root_node = ts_tree_root_node(m_ast.get());
+  std::vector<TSNode> found_nodes = {};
+  ast_util::search_for_forms_that_begin_with(m_content, root_node, prefix, found_nodes);
+
+  for (const auto& node : found_nodes) {
+    std::vector<std::string> tokens = {};
+    for (size_t i = 0; i < ts_node_child_count(node); i++) {
+      const auto child_node = ts_node_child(node, i);
+      const auto contents = ast_util::get_source_code(m_content, child_node);
+      tokens.push_back(contents);
+    }
+    const auto start_point = ts_node_start_point(node);
+    const auto end_point = ts_node_end_point(node);
+    results.push_back(
+        {tokens, {start_point.row, start_point.column}, {end_point.row, end_point.column}});
+  }
+
+  return results;
 }
 
 WorkspaceIRFile::WorkspaceIRFile(const std::string& content) {
@@ -356,7 +581,7 @@ void WorkspaceIRFile::find_all_types_path(const std::string& line) {
       const auto& game_version = matches[1];
       const auto& all_types_path = matches[2];
       lg::debug("Found DTS Path - {} : {}", game_version.str(), all_types_path.str());
-      auto all_types_uri = uri_from_path(fs::path(all_types_path.str()));
+      auto all_types_uri = lsp_util::uri_from_path(fs::path(all_types_path.str()));
       lg::debug("DTS URI - {}", all_types_uri);
       if (valid_game_version(game_version.str())) {
         m_game_version = game_name_to_version(game_version.str());
@@ -381,8 +606,6 @@ void WorkspaceIRFile::find_function_symbol(const uint32_t line_num_zero_based,
       lg::info("Adding Symbol - {}", match.str());
       LSPSpec::DocumentSymbol new_symbol;
       new_symbol.m_name = match.str();
-      // TODO - function doc-string
-      // new_symbol.m_detail = ...
       new_symbol.m_kind = LSPSpec::SymbolKind::Function;
       LSPSpec::Range symbol_range;
       symbol_range.m_start = {line_num_zero_based, 0};

lsp/state/workspace.h

@@ -2,6 +2,7 @@
 
 #include <optional>
 #include <string>
+#include <tuple>
 #include <unordered_map>
 
 #include "common/util/FileUtil.h"
@@ -14,20 +15,49 @@
 #include "lsp/protocol/document_symbols.h"
 #include "lsp/state/lsp_requester.h"
 
+#include "third-party/tree-sitter/tree-sitter/lib/src/tree.h"
+
+// TODO -
+// https://sourcegraph.com/github.com/ensisoft/detonator@36f626caf957d0734865a8f5641be6170d997f45/-/blob/editor/app/lua-tools.cpp?L116:15-116:30
+
+struct TreeSitterTreeDeleter {
+  void operator()(TSTree* ptr) const { ts_tree_delete(ptr); }
+};
+
+struct OpenGOALFormResult {
+  std::vector<std::string> tokens;
+  std::pair<int, int> start_point;
+  std::pair<int, int> end_point;
+};
+
+struct OGGlobalIndex {
+  std::unordered_map<std::string, Docs::SymbolDocumentation> global_symbols = {};
+  std::unordered_map<std::string, Docs::FileDocumentation> per_file_symbols = {};
+};
+
 class WorkspaceOGFile {
  public:
   WorkspaceOGFile(){};
-  WorkspaceOGFile(const std::string& content, const GameVersion& game_version);
-  // TODO - make private
-  int32_t version;
-  // TODO - keep an AST of the file instead
+  WorkspaceOGFile(const LSPSpec::DocumentUri& uri,
+                  const std::string& content,
+                  const GameVersion& game_version);
+  LSPSpec::DocumentUri m_uri;
   std::string m_content;
-  std::vector<std::string> m_lines;
+  int m_line_count = 0;
+  std::string m_line_ending;
+  GameVersion m_game_version;
   std::vector<LSPSpec::DocumentSymbol> m_symbols;
   std::vector<LSPSpec::Diagnostic> m_diagnostics;
-  GameVersion m_game_version;
 
+  void parse_content(const std::string& new_content);
+  void update_symbols(const std::vector<std::shared_ptr<symbol_info::SymbolInfo>>& symbol_infos);
   std::optional<std::string> get_symbol_at_position(const LSPSpec::Position position) const;
+  std::vector<OpenGOALFormResult> search_for_forms_that_begin_with(
+      std::vector<std::string> prefix) const;
+
+ private:
+  int32_t version;
+  std::shared_ptr<TSTree> m_ast;
 };
 
 class WorkspaceIRFile {
@@ -93,23 +123,40 @@ class Workspace {
   // and it's a lot faster to check the end of a string, then multiple tracked file maps
   FileType determine_filetype_from_languageid(const std::string& language_id);
   FileType determine_filetype_from_uri(const LSPSpec::DocumentUri& file_uri);
+  std::optional<GameVersion> determine_game_version_from_uri(const LSPSpec::DocumentUri& uri);
 
   void start_tracking_file(const LSPSpec::DocumentUri& file_uri,
                            const std::string& language_id,
                            const std::string& content);
   void update_tracked_file(const LSPSpec::DocumentUri& file_uri, const std::string& content);
+  void tracked_file_will_save(const LSPSpec::DocumentUri& file_uri);
+  void update_global_index(const GameVersion game_version);
   void stop_tracking_file(const LSPSpec::DocumentUri& file_uri);
-  std::optional<WorkspaceOGFile> get_tracked_og_file(const LSPSpec::URI& file_uri);
-  std::optional<WorkspaceIRFile> get_tracked_ir_file(const LSPSpec::URI& file_uri);
+  std::optional<std::reference_wrapper<WorkspaceOGFile>> get_tracked_og_file(
+      const LSPSpec::URI& file_uri);
+  std::optional<std::reference_wrapper<WorkspaceIRFile>> get_tracked_ir_file(
+      const LSPSpec::URI& file_uri);
   std::optional<DefinitionMetadata> get_definition_info_from_all_types(
       const std::string& symbol_name,
       const LSPSpec::DocumentUri& all_types_uri);
-  std::optional<SymbolInfo> get_global_symbol_info(const WorkspaceOGFile& file,
-                                                   const std::string& symbol_name);
-  std::optional<TypeSpec> get_symbol_typespec(const WorkspaceOGFile& file,
-                                              const std::string& symbol_name);
-  std::optional<Docs::DefinitionLocation> get_symbol_def_location(const WorkspaceOGFile& file,
-                                                                  const SymbolInfo& symbol_info);
+  std::vector<std::shared_ptr<symbol_info::SymbolInfo>> get_symbols_starting_with(
+      const GameVersion game_version,
+      const std::string& symbol_prefix);
+  std::optional<std::shared_ptr<symbol_info::SymbolInfo>> get_global_symbol_info(
+      const WorkspaceOGFile& file,
+      const std::string& symbol_name);
+  std::optional<std::pair<TypeSpec, Type*>> get_symbol_typeinfo(const WorkspaceOGFile& file,
+                                                                const std::string& symbol_name);
+  std::optional<symbol_info::DefinitionLocation> get_symbol_def_location(
+      const WorkspaceOGFile& file,
+      const std::shared_ptr<symbol_info::SymbolInfo> symbol_info);
+  std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>>
+  get_symbols_parent_type_path(const std::string& symbol_name, const GameVersion game_version);
+  std::vector<std::tuple<std::string, std::string, Docs::DefinitionLocation>> get_types_subtypes(
+      const std::string& symbol_name,
+      const GameVersion game_version);
+  std::unordered_map<std::string, s64> get_enum_entries(const std::string& enum_name,
+                                                        const GameVersion game_version);
 
  private:
   LSPRequester m_requester;
@@ -126,5 +173,7 @@ class Workspace {
   // and then we can track projects instead of games
   //
   // Until that decoupling happens, things like this will remain fairly clunky.
+  // TODO - change this to a shared_ptr so it can more easily be passed around functions
   std::unordered_map<GameVersion, std::unique_ptr<Compiler>> m_compiler_instances;
+  std::unordered_map<GameVersion, OGGlobalIndex> m_global_indicies;
 };

test/common/formatter/corpus/blank-lines.test.gc

@@ -11,4 +11,4 @@ Separate Top Level
 
 (println "test")
 
-(println "test")
+(println "test")

third-party/tree-sitter/tree-sitter-opengoal/grammar.js

@@ -137,8 +137,7 @@ module.exports = grammar({
     [],
 
   inline: $ =>
-    [$._kwd_unqualified,
-    $._sym_unqualified],
+    [$._sym_unqualified],
 
   rules: {
     // THIS MUST BE FIRST -- even though this doesn't look like it matters
@@ -206,7 +205,7 @@ module.exports = grammar({
       seq(field('numberOfArgs', $._format_token), '*'),
       '?',
       "Newline",
-      seq(repeat(choice($._format_token, ',')), /[$mrRbBdDgGxXeEoOsStTfF]/),
+      seq(repeat(choice($._format_token, ',')), /[$mrRbBdDgGxXeEoOsStTfHhJjKkLlNnVwWyYzZ]/),
     ),
     format_specifier: $ =>
       prec.left(seq(

third-party/tree-sitter/tree-sitter-opengoal/grammar.json

@@ -646,7 +646,7 @@
             },
             {
               "type": "PATTERN",
-              "value": "[$mrRbBdDgGxXeEoOsStTfF]"
+              "value": "[$mrRbBdDgGxXeEoOsStTfHhJjKkLlNnVwWyYzZ]"
             }
           ]
         }
@@ -1041,9 +1041,7 @@
   "precedences": [],
   "externals": [],
   "inline": [
-    "ReferenceError",
     "_sym_unqualified"
   ],
   "supertypes": []
 }
-

third-party/tree-sitter/tree-sitter-opengoal/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t);
+extern void *(*ts_current_calloc)(size_t, size_t);
+extern void *(*ts_current_realloc)(void *, size_t);
+extern void (*ts_current_free)(void *);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

third-party/tree-sitter/tree-sitter-opengoal/tree_sitter/array.h

@@ -0,0 +1,287 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  (_array__grow((Array *)(self), count, array_elem_size(self)), \
+   memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)), \
+   (self)->size += (count))
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(default : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_

third-party/tree-sitter/tree-sitter-opengoal/tree_sitter/parser.h

@@ -13,9 +13,8 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-typedef uint16_t TSStateId;
-
 #ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
@@ -130,9 +129,16 @@ struct TSLanguage {
  *  Lexer Macros
  */
 
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
+  UNUSED                        \
   bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
@@ -166,7 +172,7 @@ struct TSLanguage {
  *  Parse Table Macros
  */
 
-#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
 
 #define STATE(id) id
 
@@ -176,7 +182,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value            \
+      .state = (state_value)          \
     }                                 \
   }}
 
@@ -184,7 +190,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value,           \
+      .state = (state_value),         \
       .repetition = true              \
     }                                 \
   }}

third-party/tree-sitter/tree-sitter/.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

third-party/tree-sitter/tree-sitter/.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+tab_width = 8
+end_of_line = lf
+insert_final_newline = true
+
+[*.rs]
+indent_size = 4
+
+[Makefile]
+indent_style = tab
+indent_size = 8

third-party/tree-sitter/tree-sitter/.gitattributes

@@ -1,3 +1,5 @@
+* text=auto eol=lf
+
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 

third-party/tree-sitter/tree-sitter/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,41 @@
+name: Bug Report
+description: Report a problem
+labels: [bug]
+body:
+  - type: textarea
+    attributes:
+      label: "Problem"
+      description: "Describe the current behavior. May include logs, images, or videos."
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Steps to reproduce"
+      placeholder: |
+        git clone --depth=1 https://github.com/tree-sitter/tree-sitter-ruby
+        cd tree-sitter-ruby
+        tree-sitter generate
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Expected behavior"
+      description: "Describe the behavior you expect."
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "Tree-sitter version (tree-sitter --version)"
+      placeholder: "tree-sitter 0.20.9"
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "Operating system/version"
+      placeholder: "macOS 11.5"
+    validations:
+      required: true

third-party/tree-sitter/tree-sitter/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

third-party/tree-sitter/tree-sitter/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,23 @@
+name: Feature request
+description: Request an enhancement
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before requesting: search [existing feature requests](https://github.com/tree-sitter/tree-sitter/labels/enhancement).
+
+  - type: textarea
+    attributes:
+      label: "Problem"
+      description: "Describe the problem to be solved."
+      placeholder: "No smurf icons available. Smurfs are useful because ..."
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: "Expected behavior"
+      description: "Describe what the new feature or behavior would look like. How does it solve the problem? Is it worth the cost?"
+    validations:
+      required: false

third-party/tree-sitter/tree-sitter/.github/actions/cache/action.yml

@@ -0,0 +1,24 @@
+name: 'Cache'
+description: "This action caches fixtures"
+outputs:
+  cache-hit:
+    description: 'Cache hit'
+    value: ${{ steps.cache_output.outputs.cache-hit }}
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v4
+      id: cache_fixtures
+      with:
+        path: |
+          test/fixtures/grammars
+          target/release/tree-sitter-*.wasm
+        key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
+          'cli/src/generate/**',
+          'script/generate-fixtures*',
+          'test/fixtures/grammars/*/**/src/*.c',
+          '.github/actions/cache/action.yml') }}
+
+    - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT
+      shell: bash
+      id: cache_output

third-party/tree-sitter/tree-sitter/.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "build(deps)"
+    groups:
+      cargo:
+        patterns:
+        - "*"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "ci"
+    groups:
+      actions:
+        patterns:
+        - "*"

third-party/tree-sitter/tree-sitter/.github/scripts/close_unresponsive.js

@@ -0,0 +1,58 @@
+function labeledEvent(data) {
+  return (
+    data.event === "labeled" && data.label.name === "more-information-needed"
+  );
+}
+
+const numberOfDaysLimit = 30;
+const close_message = `This has been closed since a request for information has \
+not been answered for ${numberOfDaysLimit} days. It can be reopened when the \
+requested information is provided.`;
+
+module.exports = async ({ github, context }) => {
+  const owner = context.repo.owner;
+  const repo = context.repo.repo;
+
+  const issues = await github.rest.issues.listForRepo({
+    owner: owner,
+    repo: repo,
+    labels: "more-information-needed",
+  });
+  const numbers = issues.data.map((e) => e.number);
+
+  for (const number of numbers) {
+    const events = await github.paginate(
+      github.rest.issues.listEventsForTimeline,
+      {
+        owner: owner,
+        repo: repo,
+        issue_number: number,
+      },
+      (response) => response.data.filter(labeledEvent),
+    );
+
+    const latest_response_label = events[events.length - 1];
+
+    const created_at = new Date(latest_response_label.created_at);
+    const now = new Date();
+    const diff = now - created_at;
+    const diffDays = diff / (1000 * 60 * 60 * 24);
+
+    if (diffDays > numberOfDaysLimit) {
+      github.rest.issues.update({
+        owner: owner,
+        repo: repo,
+        issue_number: number,
+        state_reason: "not_planned",
+        state: "closed",
+      });
+
+      github.rest.issues.createComment({
+        owner: owner,
+        repo: repo,
+        issue_number: number,
+        body: close_message,
+      });
+    }
+  }
+};

third-party/tree-sitter/tree-sitter/.github/scripts/cross.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
 
-set -x
+# set -x
 set -e
 
-if [ "$CROSS" != 1 ]; then
+if [ "$BUILD_CMD" != "cross" ]; then
+    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
+    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
+    exit 111
+fi
+
+if [ -z "$CROSS_IMAGE" ]; then
+    echo "The CROSS_IMAGE env var should be provided" >&2
     exit 111
 fi
 

third-party/tree-sitter/tree-sitter/.github/scripts/make.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-set -x
+# set -x
 set -e
 
-if [ "$CROSS" = 1 ]; then
+if [ "$BUILD_CMD" == "cross" ]; then
     if [ -z "$CC" ]; then
         echo "make.sh: CC is not set" >&2
         exit 111

third-party/tree-sitter/tree-sitter/.github/scripts/remove_response_label.js

@@ -0,0 +1,19 @@
+module.exports = async ({ github, context }) => {
+  const commenter = context.actor;
+  const issue = await github.rest.issues.get({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: context.issue.number,
+  });
+  const author = issue.data.user.login;
+  const labels = issue.data.labels.map((e) => e.name);
+
+  if (author === commenter && labels.includes("more-information-needed")) {
+    github.rest.issues.removeLabel({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: context.issue.number,
+      name: "more-information-needed",
+    });
+  }
+};

third-party/tree-sitter/tree-sitter/.github/scripts/reviewers_remove.js

@@ -0,0 +1,16 @@
+module.exports = async ({ github, context }) => {
+  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+  });
+
+  const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+  github.rest.pulls.removeRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+    reviewers: reviewers,
+  });
+};

third-party/tree-sitter/tree-sitter/.github/scripts/tree-sitter.sh

@@ -1,11 +1,27 @@
 #!/bin/bash
 
-set -x
+# set -x
 set -e
 
+if [ -z "$ROOT" ]; then
+    echo "The ROOT env var should be set to absolute path of a repo root folder" >&2
+    exit 111
+fi
+
+if [ -z "$TARGET" ]; then
+    echo "The TARGET env var should be equal to a \`cargo build --target <TARGET>\` command value" >&2
+    exit 111
+fi
+
 tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
 
-if [ "$CROSS" = 1 ]; then
+if [ "$BUILD_CMD" == "cross" ]; then
+    if [ -z "$CROSS_RUNNER" ]; then
+        echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2
+        echo "that is available in a docker image used by the cross tool under the hood" >&2
+        exit 111
+    fi
+
     cross.sh $CROSS_RUNNER "$tree_sitter" "$@"
 else
     "$tree_sitter" "$@"

third-party/tree-sitter/tree-sitter/.github/workflows/CICD.yml

@@ -1,69 +0,0 @@
-name: CICD
-
-on:
-  workflow_dispatch:
-  pull_request:
-  push:
-    branches:
-      - master
-      - check/*
-
-concurrency:
-  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
-  cancel-in-progress: true
-
-jobs:
-  init:
-    name: Init
-    runs-on: ubuntu-latest
-    steps:
-      - name: Get PR head ref
-        if: ${{ github.event_name == 'pull_request' }}
-        id: ref
-        run: |
-          echo "ref=refs/pull/${{ github.event.pull_request.number }}/head" >> $GITHUB_OUTPUT
-    outputs:
-      ref: >-
-        ${{
-          (github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/v'))
-          && steps.ref.outputs.ref
-          || github.ref
-        }}
-
-  fast_checks:
-    name: Fast checks
-    uses: ./.github/workflows/fast_checks.yml
-
-  full_checks:
-    name: Full Rust checks
-    needs: fast_checks
-    uses: ./.github/workflows/full_rust_checks.yml
-
-  min_version:
-    name: Minimum supported rust version
-    needs: fast_checks
-    uses: ./.github/workflows/msrv.yml
-    with:
-      package: tree-sitter-cli
-
-  build:
-    name: Build & Test
-    needs: [init, fast_checks]
-    uses: ./.github/workflows/build.yml
-    with:
-      ref: ${{ needs.init.outputs.ref }}
-
-  release:
-    name: Release
-    needs: [init, fast_checks, full_checks, min_version, build]
-    if: >
-      github.event.pull_request.head.repo.full_name == github.repository &&
-      startsWith(github.head_ref, 'release/v')
-    uses: ./.github/workflows/release.yml
-    with:
-      ref: ${{ needs.init.outputs.ref }}
-
-  publish:
-    name: Publish
-    needs: release
-    uses: ./.github/workflows/publish.yml

third-party/tree-sitter/tree-sitter/.github/workflows/build.yml

@@ -8,160 +8,190 @@ env:
 on:
   workflow_call:
     inputs:
-      ref:
-        default: ${{ github.ref }}
-        type: string
+      run_test:
+        default: true
+        type: boolean
 
 jobs:
   build:
-    name: ${{ matrix.job.name }} (${{ matrix.job.target }}) (${{ matrix.job.os }})
-    runs-on: ${{ matrix.job.os }}
+    name: ${{ matrix.platform }} (${{ matrix.target }}) (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
-        job:
-          - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest  , use-cross: true }
-          - { name: linux-arm     , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest  , use-cross: true }
-          - { name: linux-x64     , target: x86_64-unknown-linux-gnu    , os: ubuntu-latest                    }
-          - { name: linux-x86     , target: i686-unknown-linux-gnu      , os: ubuntu-latest  , use-cross: true }
-          - { name: windows-x64   , target: x86_64-pc-windows-msvc      , os: windows-latest                   }
-          - { name: windows-x86   , target: i686-pc-windows-msvc        , os: windows-latest                   }
-          - { name: macos-x64     , target: x86_64-apple-darwin         , os: macos-latest                     }
+        platform:
+        - linux-arm64       #
+        - linux-arm         #
+        - linux-x64         #
+        - linux-x86         #
+        - linux-powerpc64   #
+        - windows-arm64     #
+        - windows-x64       # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
+        - windows-x86       #     -- // --
+        - macos-arm64       #
+        - macos-x64         #
+
+        include:
+        # When adding a new `target`:
+        # 1. Define a new platform alias above
+        # 2. Add a new record to a matrix map in `cli/npm/install.js`
+        - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu           , os: ubuntu-latest  , use-cross: true   }
+        - { platform: linux-arm         , target: arm-unknown-linux-gnueabi           , os: ubuntu-latest  , use-cross: true   }
+        - { platform: linux-x64         , target: x86_64-unknown-linux-gnu            , os: ubuntu-20.04   , enable-wasm: true } #2272
+        - { platform: linux-x86         , target: i686-unknown-linux-gnu              , os: ubuntu-latest  , use-cross: true   }
+        - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu         , os: ubuntu-latest  , use-cross: true   }
+        - { platform: windows-arm64     , target: aarch64-pc-windows-msvc             , os: windows-latest                     }
+        - { platform: windows-x64       , target: x86_64-pc-windows-msvc              , os: windows-latest , enable-wasm: true }
+        - { platform: windows-x86       , target: i686-pc-windows-msvc                , os: windows-latest                     }
+        - { platform: macos-arm64       , target: aarch64-apple-darwin                , os: macos-14       , enable-wasm: true }
+        - { platform: macos-x64         , target: x86_64-apple-darwin                 , os: macos-latest   , enable-wasm: true }
+
+        # Cross compilers for C library
+        - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc             , ar: aarch64-linux-gnu-ar           }
+        - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc             , ar: arm-linux-gnueabi-ar           }
+        - { platform: linux-x86         , cc: i686-linux-gnu-gcc                , ar: i686-linux-gnu-ar              }
+        - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc           , ar: powerpc64-linux-gnu-ar         }
+
+        # See #2041 tree-sitter issue
+        - { platform: windows-x64   , rust-test-threads: 1 }
+        - { platform: windows-x86   , rust-test-threads: 1 }
+
+        # CLI only build
+        - { platform: windows-arm64 , cli-only: true }
 
     env:
       BUILD_CMD: cargo
+      EMSCRIPTEN_VERSION: ""
+      EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
 
     defaults:
       run:
         shell: bash
 
     steps:
-    - name: Checkout source code
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ inputs.ref }}
+    - uses: actions/checkout@v4
 
     - name: Read Emscripten version
       run: |
-        echo "EMSCRIPTEN_VERSION=$(cat cli/emscripten-version)" >> $GITHUB_ENV
+        echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV
 
     - name: Install Emscripten
-      uses: mymindstorm/setup-emsdk@v12
+      if: ${{ !matrix.cli-only && !matrix.use-cross }}
+      uses: mymindstorm/setup-emsdk@v14
       with:
         version: ${{ env.EMSCRIPTEN_VERSION }}
 
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@stable
-      with:
-        targets: ${{ matrix.job.target }}
+    - run: rustup toolchain install stable --profile minimal
+    - run: rustup target add ${{ matrix.target }}
+    - uses: Swatinem/rust-cache@v2
 
     - name: Install cross
-      if: matrix.job.use-cross
+      if: ${{ matrix.use-cross }}
       uses: taiki-e/install-action@v2
       with:
         tool: cross
 
     - name: Build custom cross image
-      if: ${{ matrix.job.use-cross && matrix.job.os == 'ubuntu-latest' }}
+      if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
       run: |
         cd ..
 
-        target="${{ matrix.job.target }}"
+        target="${{ matrix.target }}"
         image=ghcr.io/cross-rs/$target:custom
-        echo "CROSS_IMAGE=$image"                                  >> $GITHUB_ENV
+        echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
 
-        echo "[target.$target]"                                    >> Cross.toml
-        echo "image = \"$image\""                                  >> Cross.toml
-        echo "CROSS_CONFIG=$PWD/Cross.toml"                        >> $GITHUB_ENV
+        echo "[target.$target]"                                >> Cross.toml
+        echo "image = \"$image\""                              >> Cross.toml
+        echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
 
-        echo "FROM ghcr.io/cross-rs/$target:edge"                  >> Dockerfile
-        echo "ENV DEBIAN_FRONTEND=noninteractive"                  >> Dockerfile
-        echo "RUN apt-get update && apt-get install -y nodejs"     >> Dockerfile
+        echo "FROM ghcr.io/cross-rs/$target:edge"              >> Dockerfile
+        echo "ENV DEBIAN_FRONTEND=noninteractive"              >> Dockerfile
+        echo "RUN apt-get update && apt-get install -y nodejs" >> Dockerfile
         docker build -t $image .
-        docker images
-        docker run --rm $image env
 
-        cd -
-
-    - name: Setup extra env
+    - name: Setup env extras
+      env:
+        RUST_TEST_THREADS: ${{ matrix.rust-test-threads || '' }}
+        USE_CROSS: ${{ matrix.use-cross }}
+        TARGET: ${{ matrix.target }}
+        CC: ${{ matrix.cc }}
+        AR: ${{ matrix.ar }}
+        IS_WINDOWS: ${{ contains(matrix.os, 'windows') }}
+        ENABLE_WASM: ${{ matrix.enable-wasm }}
       run: |
         PATH="$PWD/.github/scripts:$PATH"
-        echo "PATH=$PATH" >> $GITHUB_ENV
-        echo "ROOT=$PWD" >> $GITHUB_ENV
+        echo "$PWD/.github/scripts" >> $GITHUB_PATH
+
         echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
-
-        export TARGET=${{ matrix.job.target }}
         echo "TARGET=$TARGET" >> $GITHUB_ENV
+        echo "ROOT=$PWD" >> $GITHUB_ENV
 
-        USE_CROSS="${{ matrix.job.use-cross }}"
+        [ -n "$RUST_TEST_THREADS" ] && \
+        echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
+
+        [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
+        [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
+
+        [ "$IS_WINDOWS" = "false" ] && echo "CFLAGS=-Werror" >> $GITHUB_ENV
+
+        if [ "$ENABLE_WASM" == "true" ]; then
+          echo "CLI_FEATURES=wasm" >> $GITHUB_ENV
+        fi
 
         if [ "$USE_CROSS" == "true" ]; then
           echo "BUILD_CMD=cross" >> $GITHUB_ENV
-
-          export CROSS=1; echo "CROSS=$CROSS" >> $GITHUB_ENV
-
-          runner=$(cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
+          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
           [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
-          echo "runner: $runner"
-
-          case "$TARGET" in
-            i686-unknown-linux-gnu)      CC=i686-linux-gnu-gcc              AR=i686-linux-gnu-ar                  ;;
-            aarch64-unknown-linux-gnu)   CC=aarch64-linux-gnu-gcc           AR=aarch64-linux-gnu-ar               ;;
-            arm-unknown-linux-gnueabihf) CC=arm-unknown-linux-gnueabihf-gcc AR=arm-unknown-linux-gnueabihf-gcc-ar ;;
-          esac
-
-          [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
-          [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
         fi
 
-        case "$TARGET" in
-          *-windows-*)
-            echo "RUST_TEST_THREADS=1" >> $GITHUB_ENV # See #2041 tree-sitter issue
-            ;;
-        esac
-
     - name: Build C library
-      if: "!contains(matrix.job.os, 'windows')" # Requires an additional adapted Makefile for `cl.exe` compiler
-      run: make.sh CFLAGS="-Werror" -j
+      if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
+      run: make.sh -j
 
     - name: Build wasm library
+      if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times
       run: script/build-wasm
 
     - name: Build CLI
-      run: $BUILD_CMD build --release --target=${{ matrix.job.target }}
+      run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${CLI_FEATURES}
 
-    - name: Fetch fixtures
-      run: script/fetch-fixtures
+    - run: script/fetch-fixtures
+
+    - uses: ./.github/actions/cache
+      id: cache
 
     - name: Generate fixtures
+      if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host
       run: script/generate-fixtures
 
     - name: Generate WASM fixtures
-      if: "!matrix.job.use-cross"
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step
       run: script/generate-fixtures-wasm
 
     - name: Run main tests
-      run: $BUILD_CMD test --target=${{ matrix.job.target }}
+      if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host
+      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${CLI_FEATURES}
 
     - name: Run wasm tests
-      if: "!matrix.job.use-cross" # TODO: Install Emscripten into custom cross images
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step
       run: script/test-wasm
 
     - name: Run benchmarks
-      if: "!matrix.job.use-cross" # It doesn't make sense to benchmark something in an emulator
-      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.job.target }}
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense
+      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
 
     - name: Upload CLI artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: tree-sitter.${{ matrix.job.name }}
-        path: target/${{ matrix.job.target }}/release/tree-sitter${{ contains(matrix.job.target, 'windows') && '.exe' || '' }}
+        name: tree-sitter.${{ matrix.platform }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }}
         if-no-files-found: error
         retention-days: 7
 
     - name: Upload WASM artifacts
-      if: ${{ matrix.job.name == 'linux-x64' }}
-      uses: actions/upload-artifact@v3
+      if: ${{ matrix.platform == 'linux-x64' }}
+      uses: actions/upload-artifact@v4
       with:
         name: tree-sitter.wasm
         path: |

third-party/tree-sitter/tree-sitter/.github/workflows/checks.yml

@@ -0,0 +1,24 @@
+name: Full Rust codebase checks
+
+on:
+  workflow_call:
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - run: rustup toolchain install stable --profile minimal
+    - uses: Swatinem/rust-cache@v2
+
+    - run: make lint
+
+  check_c_warnings:
+    name: Check C warnings
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Make C library to check that it's able to compile without warnings
+      run: make -j CFLAGS="-Werror"

third-party/tree-sitter/tree-sitter/.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'push' }}
+
+jobs:
+  checks:
+    uses: ./.github/workflows/checks.yml
+
+  sanitize:
+    uses: ./.github/workflows/sanitize.yml
+
+  build:
+    uses: ./.github/workflows/build.yml

third-party/tree-sitter/tree-sitter/.github/workflows/fast_checks.yml

@@ -1,31 +0,0 @@
-name: Fast checks to fail fast on any simple code issues
-
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-
-on:
-  workflow_call:
-
-jobs:
-  check_rust_formatting:
-    name: Check Rust formating
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Checkout source code
-      uses: actions/checkout@v3
-
-    - name: Run cargo fmt
-      run: cargo fmt -- --check
-
-  check_c_warnings:
-    name: Check C warnings
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Checkout source code
-      uses: actions/checkout@v3
-
-    - name: Make C library to check that it's able to compile without warnings
-      run: make -j CFLAGS="-Werror"

third-party/tree-sitter/tree-sitter/.github/workflows/full_rust_checks.yml

@@ -1,32 +0,0 @@
-name: Full Rust codebase checks
-
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-
-on:
-  workflow_call:
-
-jobs:
-  run:
-    name: Run checks
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Checkout source code
-      uses: actions/checkout@v3
-
-    - name: Install rust toolchain
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: stable
-        components: clippy, rustfmt
-
-    - name: Run cargo fmt
-      run: cargo fmt -- --check
-
-    # - name: Run clippy
-    #   run: cargo clippy --all-targets
-
-    - name: Run cargo check
-      run: cargo check --workspace --examples --tests --benches --bins

third-party/tree-sitter/tree-sitter/.github/workflows/msrv.yml

@@ -1,42 +0,0 @@
-name: Minimum supported rust version
-
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-
-on:
-  workflow_call:
-    inputs:
-      package:
-        description: Target cargo package name
-        required: true
-        type: string
-
-
-jobs:
-  run:
-    name: Run checks
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Checkout source code
-      uses: actions/checkout@v3
-
-    - name: Get the MSRV from the package metadata
-      id: msrv
-      run: cargo metadata --no-deps --format-version 1 | jq -r '"version=" + (.packages[] | select(.name == "${{ inputs.package }}").rust_version)' >> $GITHUB_OUTPUT
-
-    - name: Install rust toolchain (v${{ steps.msrv.outputs.version }})
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: ${{ steps.msrv.outputs.version }}
-        components: clippy, rustfmt
-
-    - name: Run cargo fmt
-      run: cargo fmt -- --check
-
-    # - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix)
-    #   run: cargo clippy --all-targets
-
-    # - name: Run main tests
-    #   run: cargo test

third-party/tree-sitter/tree-sitter/.github/workflows/publish.yml

@@ -1,21 +0,0 @@
-name: Publish to registries
-
-on:
-  workflow_call:
-
-jobs:
-  crates_io:
-    name: Publish to Crates.io
-    runs-on: ubuntu-latest
-    steps:
-      - name: Publish packages
-        run: |
-          echo "::warning::TODO: add a Crates.io publish logic"
-
-  npm:
-    name: Publish to npmjs.com
-    runs-on: ubuntu-latest
-    steps:
-      - name: Publish packages
-        run: |
-          echo "::warning::TODO: add a npmjs.com publish logic"

third-party/tree-sitter/tree-sitter/.github/workflows/release.yml

@@ -1,52 +1,27 @@
 name: Release
-
 on:
-  workflow_call:
-    inputs:
-      ref:
-        default: ${{ github.ref }}
-        type: string
+  workflow_dispatch:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
 
 jobs:
-  permissions:
-    name: Check permissions
-    runs-on: ubuntu-latest
-    outputs:
-      release_allowed: ${{ steps.maintainer.outputs.is_maintainer == 'true' }}
-    steps:
-
-    - name: Is maintainer
-      id: maintainer
-      env:
-        GH_TOKEN: ${{ github.token }}
-        repo: ${{ github.repository }}
-        actor: ${{ github.actor }}
-      run: |
-        maintainer=$(
-          gh api "/repos/${repo}/collaborators" |
-          jq ".[] | {login, maintainer: .permissions | .maintain} | select(.login == \"${actor}\") | .maintainer"
-        );
-        if [ "$maintainer" == "true" ]; then
-          echo "@${actor} has maintainer level permissions :rocket:" >> $GITHUB_STEP_SUMMARY;
-          echo "is_maintainer=true" >> $GITHUB_OUTPUT
-        fi
+  build:
+    uses: ./.github/workflows/build.yml
+    with:
+      run_test: false
 
   release:
     name: Release
-    needs: permissions
-    if: needs.permissions.outputs.release_allowed
     runs-on: ubuntu-latest
+    needs: build
     permissions:
       contents: write
     steps:
-
-      - name: Checkout source code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.ref }}
+      - uses: actions/checkout@v4
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -66,36 +41,60 @@ jobs:
           rm -rf artifacts
           ls -l target/
 
-      - name: Get tag name from a release/v* branch name
-        id: tag_name
-        env:
-          tag: ${{ github.head_ref }}
-        run: echo "tag=${tag#release/}" >> $GITHUB_OUTPUT
-
-      - name: Add a release tag
-        env:
-          ref: ${{ inputs.ref }}
-          tag: ${{ steps.tag_name.outputs.tag }}
-          message: "Release ${{ steps.tag_name.outputs.tag }}"
-        run: |
-          git config user.name "${GITHUB_ACTOR}"
-          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-          git tag -a "$tag" HEAD -m "$message"
-          git push origin "$tag"
-
       - name: Create release
         uses: softprops/action-gh-release@v1
         with:
-          name: ${{ steps.tag_name.outputs.tag }}
-          tag_name: ${{ steps.tag_name.outputs.tag }}
+          name: ${{ github.ref_name }}
+          tag_name: ${{ github.ref_name }}
           fail_on_unmatched_files: true
           files: |
             target/tree-sitter-*.gz
             target/tree-sitter.wasm
             target/tree-sitter.js
 
-      - name: Merge release PR
+  crates_io:
+    name: Publish CLI to Crates.io
+    runs-on: ubuntu-latest
+    needs: release
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Publish crates to Crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+  npm:
+    name: Publish lib to npmjs.com
+    runs-on: ubuntu-latest
+    needs: release
+    strategy:
+      fail-fast: false
+      matrix:
+        directory: ["cli/npm", "lib/binding_web"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wasm
+        if: matrix.directory == 'lib/binding_web'
+        run: ./script/build-wasm
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Publish lib to npmjs.com
         env:
-          GH_TOKEN: ${{ github.token }}
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
         run: |
-          gh pr merge ${{ github.event.pull_request.html_url }} --match-head-commit $(git rev-parse HEAD) --merge --delete-branch
+          cd ${{ matrix.directory }}
+          npm publish

third-party/tree-sitter/tree-sitter/.github/workflows/response.yml

@@ -0,0 +1,35 @@
+name: no_response
+on:
+  schedule:
+    - cron: '30 1 * * *' # Run every day at 01:30
+  workflow_dispatch:
+  issue_comment:
+
+jobs:
+  close:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const script = require('./.github/scripts/close_unresponsive.js')
+            await script({github, context})
+
+  remove_label:
+    if: github.event_name == 'issue_comment'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const script = require('./.github/scripts/remove_response_label.js')
+            await script({github, context})

third-party/tree-sitter/tree-sitter/.github/workflows/reviewers_remove.yml

@@ -0,0 +1,17 @@
+name: "reviewers: remove"
+on:
+  pull_request_target:
+    types: [converted_to_draft, closed]
+jobs:
+  remove-reviewers:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Remove reviewers'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const script = require('./.github/scripts/reviewers_remove.js')
+            await script({github, context})

third-party/tree-sitter/tree-sitter/.github/workflows/sanitize.yml

@@ -0,0 +1,53 @@
+name: Sanitize
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+
+jobs:
+  check_undefined_behaviour:
+    name: Sanitizer checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Install UBSAN library
+      run: sudo apt-get update -y && sudo apt-get install -y libubsan1
+
+    - run: rustup toolchain install stable --profile minimal
+    - uses: Swatinem/rust-cache@v2
+
+    - name: Build CLI
+      run: cargo build --release
+
+    - run: script/fetch-fixtures
+
+    - uses: ./.github/actions/cache
+      id: cache
+
+    - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      run: script/generate-fixtures
+
+    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
+      env:
+        UBSAN_OPTIONS: halt_on_error=1
+        CFLAGS: -fsanitize=undefined
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
+      run: cargo test -- --test-threads 1
+
+    - name: Run main tests with address sanitizer (ASAN)
+      env:
+        ASAN_OPTIONS: halt_on_error=1
+        CFLAGS: -fsanitize=address
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -Zsanitizer=address --cfg=sanitizing
+      run: |
+        rustup install nightly
+        rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
+        cargo +nightly test -Z build-std --target x86_64-unknown-linux-gnu -- --test-threads 1

third-party/tree-sitter/tree-sitter/.gitignore

@@ -7,6 +7,7 @@ log*.html
 
 fuzz-results
 
+/tree-sitter.pc
 test/fixtures/grammars/*
 !test/fixtures/grammars/.gitkeep
 package-lock.json
@@ -24,4 +25,6 @@ docs/assets/js/tree-sitter.js
 *.obj
 *.exp
 *.lib
-*.wasm
+*.wasm
+.swiftpm
+zig-*