tooling: breaking off my tooling changes in the big batch PR so they can be used (#850)

Taskfile.yml

@@ -19,7 +19,7 @@ tasks:
       - rm ./decompiler_out/**/*disasm.gc
   decomp-file:
     cmds:
-      - python ./scripts/next-decomp-file.py --file "{{.FILE}}"
+      - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
       - task: decomp
   decomp-list:
     cmds:
@@ -29,7 +29,7 @@ tasks:
   # python -m pip install -U watchdog[watchmedo]
   decomp-watch:
     cmds:
-      - watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILE="{{.FILE}}"' ./decompiler/config/
+      - watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/
   clean-all-types:
     cmds:
       - python ./scripts/cleanup-all-types.py
@@ -47,11 +47,11 @@ tasks:
   add-reference-test:
     cmds:
       - task: decomp-file
-      - python ./scripts/add-reference-test.py --file "{{.FILE}}"
+      - python ./scripts/add-reference-test.py --file "{{.FILES}}"
       - task: offline-tests
   add-reference-test-no-decomp:
     cmds:
-      - python ./scripts/add-reference-test.py --file "{{.FILE}}"
+      - python ./scripts/add-reference-test.py --file "{{.FILES}}"
       - task: offline-tests
   update-reference-tests:
     cmds:
@@ -62,17 +62,21 @@ tasks:
       - task: offline-tests
   find-label-types:
     cmds:
-      - python ./scripts/next-decomp-file.py --file "{{.FILE}}"
+      - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
       - task: decomp
-      - python ./scripts/find-label-types.py --file "{{.FILE}}"
+      - python ./scripts/find-label-types.py --file "{{.FILES}}"
   type-test:
     cmds:
       - ./out/build/Release/bin/goalc-test.exe --gtest_brief=0 --gtest_filter="*TypeConsistency*"
+  update-gsrc:
+    cmds:
+      - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
+      - task: decomp
+      - python ./scripts/update-goal-src.py --files "{{.FILES}}"
+      - task: type-test
   # TODO - make a script that checks for function/method splits
-  # TODO - make a script to cleanup common cruft from finished file
   file-good?:
     cmds:
       - task: type-test
       - task: offline-tests
       - task: find-label-types
-

decompiler/IR2/FormExpressionAnalysis.cpp

@@ -657,7 +657,8 @@ void SimpleExpressionElement::update_from_stack_div_s(const Env& env,
         GenericOperator::make_fixed(FixedOperatorKind::DIVISION), args.at(0), args.at(1));
     result->push_back(new_form);
   } else {
-    throw std::runtime_error(fmt::format("Floating point division attempted on invalid types."));
+    throw std::runtime_error(
+        fmt::format("Floating point division attempted on invalid types at OP: [{}].", m_my_idx));
   }
 }
 
@@ -789,7 +790,8 @@ void SimpleExpressionElement::update_from_stack_float_1(const Env& env,
         pool.alloc_element<GenericElement>(GenericOperator::make_fixed(kind), args.at(0));
     result->push_back(new_form);
   } else {
-    throw std::runtime_error(fmt::format("Floating point division attempted on invalid types."));
+    throw std::runtime_error(
+        fmt::format("Floating point division attempted on invalid types at OP: [{}].", m_my_idx));
   }
 }
 

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -652,7 +652,7 @@ void ObjectFileDB::ir2_insert_lets(int seg) {
         combined_stats += insert_lets(func, func.ir2.env, *func.ir2.form_pool, func.ir2.top_form);
       } catch (const std::exception& e) {
         func.warnings.general_warning(
-            fmt::format("Error while inserting lets: {}\n Make sure that the return type is not "
+            fmt::format("Error while inserting lets: {}. Make sure that the return type is not "
                         "none if something is actually returned.",
                         e.what()));
       }

scripts/next-decomp-file.py

@@ -3,38 +3,39 @@ from jak1_file_list import file_list
 
 import argparse
 parser = argparse.ArgumentParser()
-parser.add_argument("--file")
+parser.add_argument("--files")
 parser.add_argument("--list", type=int)
 args = parser.parse_args()
 
-def update_file(file, hack):
+def update_json_file(decomp_list):
   new_file_contents = []
-  print("Next file to decompile is - " + file[0])
-  print("Target is - " + "goal_src/" + file[4] + "/" + file[0] + ".gc")
-  print("Uses the following CGO / DGO - " + str(file[3]))
+  print("Decompiling - " + ','.join(decomp_list))
   # TODO, update the CGO/DGO
   with open("decompiler\config\jak1_ntsc_black_label.jsonc", "r") as config_file:
     cfg_lines = config_file.readlines()
     for line in cfg_lines:
-      if "allowed_objects" in line and hack == False:
-        line = "  \"allowed_objects\": [\"" + file[0] + "\"],\n"
-      elif "allowed_objects" in line and hack == True:
-        line = "  \"allowed_objects\": [\"" + file[1] + "\"],\n"
+      if "allowed_objects" in line:
+        line = "  \"allowed_objects\": [\"" + ','.join(decomp_list) + "\"],\n"
       new_file_contents.append(line)
   if len(new_file_contents) > 0:
     with open("decompiler\config\jak1_ntsc_black_label.jsonc", "w") as f:
       f.writelines(new_file_contents)
       print("Allowed objects list updated")
 
-if args.file:
-  for file in file_list:
-    if file[0] == args.file:
-      update_file(file, False)
-      break
-    elif file[1] == args.file:
-      # NOTE - kinda a hack, assumes -ag files always come after
-      update_file(file, True)
-      break
+if args.files:
+  input_list = args.files.split(",")
+  decomp_list = []
+  for inFile in input_list:
+    for file in file_list:
+      if file[0] == inFile:
+        decomp_list.append(file[0])
+        break
+      elif file[1] == inFile:
+        # NOTE - kinda a hack, assumes -ag files always come after
+        decomp_list.append(file[1])
+        break
+  if len(decomp_list) > 0:
+    update_json_file(decomp_list)
 else:
   list_eligible = []
   for file in file_list:

scripts/update-goal-src.py

@@ -0,0 +1,92 @@
+## Given a list of files(comma delimited), decompile it, then place it under the specified placeholder (if it exists)
+## if the placeholder doesn't exist, error out
+## the placeholder is `;; DECOMP BEGINS`
+
+from jak1_file_list import file_list
+import os
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument("--files")
+args = parser.parse_args()
+
+files = args.files.split(",")
+
+throw_error = False
+
+for file in files:
+  disasm_path = "./decompiler_out/jak1/{}_disasm.gc".format(file)
+  if not os.path.exists(disasm_path):
+    print("{} doesn't exist!".format(disasm_path))
+    throw_error = True
+    continue
+
+  src_path = ""
+  for f in file_list:
+    if f[2] != 3:
+      continue
+    if f[0] == file:
+      src_path = f[4]
+      break
+
+  if not os.path.exists("./goal_src/{}".format(src_path)):
+    print("{} couldn't find in /goal_src!".format(file))
+    throw_error = True
+    continue
+
+  file_path = "./goal_src/{}/{}.gc".format(src_path, file)
+  new_lines = []
+  with open(file_path) as f:
+    lines = f.readlines()
+    found_placeholder = False
+    for line in lines:
+      if ";; decomp begins" in line.lower():
+        found_placeholder = True
+        new_lines.append(line.upper())
+        break
+      new_lines.append(line)
+    if found_placeholder == False:
+      print("No placeholder found in {}, skipping".format(file_path))
+      throw_error = True
+      continue
+
+  # finally...update the file
+  lines_to_ignore = [
+    ";;-*-Lisp-*-",
+    "(in-package goal)",
+    ";; definition",
+    ";; INFO:",
+    ";; failed to figure",
+    ";; Used lq/sq"
+  ]
+  def skippable_line(line):
+    for prefix in lines_to_ignore:
+      if line.startswith(prefix):
+        return True
+    return False
+  with open(disasm_path) as f:
+    lines = f.readlines()
+    in_inspect_method = False
+    for i, line in enumerate(lines):
+      # strip inspect methods
+      if line.startswith("(defmethod inspect") or (line.startswith("(defmethod") and (i + 1 < len(lines) and "inspect" in lines[i+1])):
+        in_inspect_method = True
+        continue
+      if in_inspect_method and line == "\n":
+        in_inspect_method = False
+      elif in_inspect_method:
+        continue
+      # strip comments we dont care about
+      if skippable_line(line):
+        continue
+      # otherwise, add it to the file
+      new_lines.append(line)
+
+  # write the damn thing
+  os.remove(file_path)
+  with open(file_path, "w") as f:
+    f.writelines(new_lines)
+
+  print("Copied - {}!".format(file))
+
+if throw_error:
+  exit(1)