decomp: Copy over new docstrings back to jak 2's common code, write some scripts to help automate this (#3366)

scripts/ci/docs/append-file-docs.py

@@ -42,3 +42,4 @@ def append_file_docs(game_name):
 
 append_file_docs("jak1")
 append_file_docs("jak2")
+append_file_docs("jak3")

scripts/gsrc/copy-common-naming.py

@@ -0,0 +1,191 @@
+# The goal of this script is fairly straight forward
+# Given a object file name, decompile it for both jak 2 and jak 3 WITHOUT variable casts
+# Get all function definitions and compare the function bodies.
+# - if the function bodies are the same, copy the variables from whichever game has them defined to the other
+#   - also, if it's a function and a docstring exists on one side but not the other, copy the docstring to the other side's all-types file
+
+import argparse
+import glob
+import json
+import os
+
+from utils import decompile_file, is_file_in_game
+
+
+parser = argparse.ArgumentParser("copy-common-naming")
+parser.add_argument("--file", help="The name of the file", type=str)
+parser.add_argument("--decompiler", help="The path to the decompiler", type=str)
+parser.add_argument("--update-names-from-refs", help="The decomp config version", action='store_true')
+args = parser.parse_args()
+
+def find_all_function_defs(lines):
+  store = {}
+  in_function_def = False
+  in_docstring = False
+  passed_potential_docstring = False
+  current_function_name = None
+  for line in lines:
+    if line.startswith("; .function") and "top-level" not in line:
+      current_function_name = line.split(".function")[1].strip()
+      store[current_function_name] = {
+        "docstring": [],
+        "definition": [],
+      }
+      passed_potential_docstring = False
+      in_docstring = False
+      continue
+    if current_function_name is not None and line.startswith(";;-*-OpenGOAL-Start-*-"):
+      in_function_def = True
+      continue
+    if current_function_name is not None and line.startswith(";;-*-OpenGOAL-End-*-"):
+      in_function_def = False
+      continue
+    if line.strip() == "":
+      continue
+    if in_function_def:
+      if not passed_potential_docstring and line.strip().startswith("\""):
+        in_docstring = True
+      if in_docstring:
+        store[current_function_name]["docstring"].append(line.strip())
+        if line.strip().endswith("\""):
+          in_docstring = False
+      else:
+        store[current_function_name]["definition"].append(line)
+        if len(store[current_function_name]["definition"]) > 1 and line.startswith("   "):
+          passed_potential_docstring = True
+  return store
+
+def get_var_casts_for_game(game_name):
+  return json.load(open("./decompiler/config/{}/ntsc_v1/var_names.jsonc".format(game_name), "r"))
+
+def save_var_casts_for_game(game_name, casts):
+  with open("./decompiler/config/{}/ntsc_v1/var_names.jsonc".format(game_name), "w") as f:
+    json.dump(casts, f, indent=2)
+
+def get_all_types_for_game(game_name):
+  return open("./decompiler/config/{}/all-types.gc".format(game_name), "r").readlines()
+
+jak2_alltypes = get_all_types_for_game("jak2")
+jak3_alltypes = get_all_types_for_game("jak3")
+
+file_stats = ""
+
+def update_file_var_name_casts(file_name, modify_alltypes):
+  global file_stats
+  # Check if the file exists in both games
+  if not is_file_in_game("jak3", file_name) or not is_file_in_game("jak2", file_name):
+    print("File not found in both games")
+    return
+
+  # Decompile the file for both games
+  decompile_file(args.decompiler, "jak3/jak3_config.jsonc", "ntsc_v1", "[\"{}\"]".format(file_name), True)
+  decompile_file(args.decompiler, "jak2/jak2_config.jsonc", "ntsc_v1", "[\"{}\"]".format(file_name), True)
+
+  # Go grab the contents of each file
+  jak2_file_contents = open("./decompiler_out/jak2/{}_ir2.asm".format(file_name), "r").readlines()
+  jak3_file_contents = open("./decompiler_out/jak3/{}_ir2.asm".format(file_name), "r").readlines()
+
+  # Read in the function definitions for each file to find which ones match
+
+  jak2_function_defs = find_all_function_defs(jak2_file_contents)
+  jak3_function_defs = find_all_function_defs(jak3_file_contents)
+
+  # print(jak2_function_defs["vector-xz-cross!"])
+  # print()
+  # print(jak3_function_defs["vector-xz-cross!"])
+
+  # Compare functions to see which ones are eligible
+  matching_func_names = []
+  for func_name in jak2_function_defs:
+    if func_name in jak3_function_defs and jak2_function_defs[func_name]["definition"] == jak3_function_defs[func_name]["definition"]:
+      matching_func_names.append(func_name)
+
+  # print(matching_func_names)
+  file_stats = file_stats + "Found {} matching functions in {}\n".format(len(matching_func_names), file_name)
+
+  # Go grab the var casts for each game
+  jak2_var_casts = get_var_casts_for_game("jak2")
+  jak3_var_casts = get_var_casts_for_game("jak3")
+
+  # For each eligible matching function, copy over the var casts
+  # The assumption is if it exists in jak 3 it's better (done more recently) so we use that
+  # else, use jak 2's if it exists
+  for func_name in matching_func_names:
+    if func_name in jak3_var_casts:
+      jak2_var_casts[func_name] = jak3_var_casts[func_name]
+    elif func_name in jak2_var_casts:
+      jak3_var_casts[func_name] = jak2_var_casts[func_name]
+
+  save_var_casts_for_game("jak2", jak2_var_casts)
+  save_var_casts_for_game("jak3", jak3_var_casts)
+
+  # Automatically copy docstrings for functions (methods are way to annoying to do with hack scripts now)
+  if modify_alltypes:
+    for func_name in matching_func_names:
+      if func_name.startswith("("):
+        continue
+      # handle the case where the jak 3 version has a docstring, but jak 2 does not
+      if len(jak3_function_defs[func_name]["docstring"]) != 0 and len(jak2_function_defs[func_name]["docstring"]) == 0:
+        for line_no, line in enumerate(jak2_alltypes):
+          line = jak2_alltypes[line_no]
+          if line.startswith("(define-extern {}".format(func_name)):
+            jak2_alltypes[line_no] = line.replace("(define-extern {} ".format(func_name), "(define-extern {}\n  {}\n  ".format(func_name, "\n  ".join(jak3_function_defs[func_name]["docstring"])))
+            break
+      # handle the case where jak 2 has a docstring but jak 3 does not
+      elif len(jak2_function_defs[func_name]["docstring"]) != 0 and len(jak3_function_defs[func_name]["docstring"]) == 0:
+        for line_no, line in enumerate(jak3_alltypes):
+          line = jak3_alltypes[line_no]
+          if line.startswith("(define-extern {}".format(func_name)):
+            jak3_alltypes[line_no] = line.replace("(define-extern {} ".format(func_name), "(define-extern {}\n  {}\n  ".format(func_name, "\n  ".join(jak2_function_defs[func_name]["docstring"])))
+            break
+
+if args.update_names_from_refs:
+  reference_test_files = glob.glob("./test/decompiler/reference/jak3/**/*_REF.gc", recursive=True)
+  for file_no, reference_test_file in enumerate(reference_test_files):
+    file_name = os.path.basename(reference_test_file).split("_REF.gc")[0]
+    print("({}/{}) Checking Var Name Casts for {}...".format(file_no+1, len(reference_test_files), file_name))
+    update_file_var_name_casts(file_name, False)
+else:
+  update_file_var_name_casts(args.file, True)
+
+print(file_stats)
+
+def get_type_docstrings_from_alltypes(lines):
+  store = {}
+  awaiting_next_docstring = True
+  current_type_name = None
+  for line in lines:
+    if line.startswith("(deftype"):
+      current_type_name = line.split("deftype ")[1].split("(")[0].strip()
+      awaiting_next_docstring = False
+      store[current_type_name] = []
+      continue
+    if line.strip().startswith("(") and not line.strip().endswith("\""):
+      awaiting_next_docstring = True
+      continue
+    if not awaiting_next_docstring:
+      store[current_type_name].append(line.strip())
+  return store
+
+jak2_type_docs = get_type_docstrings_from_alltypes(jak2_alltypes)
+jak3_type_docs = get_type_docstrings_from_alltypes(jak3_alltypes)
+
+# If a docstring exists in jak3 but not in jak2, copy it back
+new_jak2_alltypes = []
+for line_no, line in enumerate(jak2_alltypes):
+  line = jak2_alltypes[line_no]
+  new_jak2_alltypes.append(line)
+  if line.startswith("(deftype "):
+    current_type_name = line.split("deftype ")[1].split("(")[0].strip()
+    if current_type_name in jak3_type_docs and len(jak2_type_docs[current_type_name]) == 0:
+      for docstring_line in jak3_type_docs[current_type_name]:
+        new_jak2_alltypes.append("  " + docstring_line + "\n")
+jak2_alltypes = new_jak2_alltypes
+
+# Save all-types
+def get_all_types_for_game(game_name, lines):
+  with open("./decompiler/config/{}/all-types.gc".format(game_name), "w") as f:
+    f.writelines(lines)
+
+get_all_types_for_game("jak2", jak2_alltypes)
+get_all_types_for_game("jak3", jak3_alltypes)

scripts/gsrc/find-missing-refs.py

@@ -23,8 +23,6 @@ for file in file_list:
       gsrc_length = len(fp.readlines())
 
     if gsrc_length > 15:
-      if file_name == "enemy-h":
-        print(file_name)
       # check if ref exists
       ref_path = get_ref_path_from_filename("jak2", file_name, "./test/decompiler/reference/")
       if not os.path.exists(ref_path):

scripts/gsrc/requirements.txt

@@ -1,3 +0,0 @@
-rapidfuzz
-GitPython
-colorama

scripts/gsrc/update-from-decomp.py

@@ -36,13 +36,9 @@
 # - there are likely ways to make this more efficient
 
 import argparse
-import os
 from code_retention.all_types_retention import update_alltypes_named_blocks
 from code_retention.code_retention import is_line_start_of_form, has_form_ended
 from utils import get_gsrc_path_from_filename
-import shutil
-from pathlib import Path
-import subprocess
 
 parser = argparse.ArgumentParser("update-from-decomp")
 parser.add_argument("--game", help="The name of the game", type=str)
@@ -64,7 +60,6 @@ comments = []
 debug_lines = []
 decomp_ignore_forms = ["defmethod inspect"]
 decomp_ignore_errors = False
-update_with_merge = False
 
 with open(gsrc_path) as f:
     lines_temp = f.readlines()
@@ -78,26 +73,8 @@ with open(gsrc_path) as f:
             decomp_ignore_errors = True
         if "og:ignore-form" in line:
             decomp_ignore_forms.append(line.partition("ignore-form:")[2].strip())
-        if "og:update-with-merge" in line:
-            update_with_merge = True
         lines.append(line)
 
-# If we are going to `update_with_merge` then make a backup of the file, and
-# an empty file to use as the common ancestor.
-#
-# This means that all changes will be flagged as a conflict and will not be able to be
-# merged into the repo without being explicitly resolved
-if update_with_merge:
-    subprocess.run(
-        [
-            "git",
-            "restore",
-            gsrc_path
-        ]
-    )
-    shutil.copyfile(gsrc_path, gsrc_path.replace(".gc", ".before.gc"))
-    Path(gsrc_path.replace(".gc", ".empty.gc")).touch()
-
 if args.debug:
     with open(gsrc_path, "w") as f:
         f.writelines(debug_lines)
@@ -207,29 +184,3 @@ with open(gsrc_path, "w") as f:
     while i + lines_to_ignore < len(final_lines):
         f.write(final_lines[i])
         i = i + 1
-
-# If we need to merge, now is the time!
-if update_with_merge:
-    shutil.move(gsrc_path, gsrc_path.replace(".gc", ".after.gc"))
-    shutil.move(gsrc_path.replace(".gc", ".before.gc"), gsrc_path)
-    subprocess.run(
-        [
-            "git",
-            "merge-file",
-            gsrc_path,
-            gsrc_path.replace(".gc", ".empty.gc"),
-            gsrc_path.replace(".gc", ".after.gc"),
-            "-L",
-            "Before Updating",
-            "-L",
-            "ignored",
-            "-L",
-            "After Updating",
-        ]
-    )
-    if os.path.exists(gsrc_path.replace(".gc", ".empty.gc")):
-        os.remove(gsrc_path.replace(".gc", ".empty.gc"))
-    if os.path.exists(gsrc_path.replace(".gc", ".before.gc")):
-        os.remove(gsrc_path.replace(".gc", ".before.gc"))
-    if os.path.exists(gsrc_path.replace(".gc", ".after.gc")):
-        os.remove(gsrc_path.replace(".gc", ".after.gc"))

scripts/gsrc/update-gsrc-via-refs.py

@@ -1,8 +1,9 @@
 # Updates files in gsrc if they are modified in the reference test folder
 # Uses git
-import subprocess
 from git import Repo
 
+from utils import decompile_file
+
 repo = Repo("./")
 
 import argparse
@@ -45,19 +46,7 @@ else:
 
 all_names = str(file_names).replace("'", "\"").replace("{", "[").replace("}", "]");
 print("Decompiling - {}".format(all_names))
-# Decompile file
-subprocess.run(
-    [
-        args.decompiler,
-        "./decompiler/config/{}".format(args.decompiler_config),
-        "./iso_data",
-        "./decompiler_out",
-        "--version",
-        args.version,
-        "--config-override",
-        '{{"levels_extract": false, "process_art_groups": false, "decompile_code": true, "allowed_objects": {}}}'.format(all_names),
-    ]
-)
+decompile_file(args.decompiler, args.decompiler_config, args.version, all_names, False)
 
 for file_name in file_names:
     print("Updating - {}".format(file_name))

scripts/gsrc/utils.py

@@ -1,5 +1,6 @@
 import json
 import os
+import subprocess
 
 jak1_files = None
 jak2_files = None
@@ -21,6 +22,15 @@ def get_file_list(game_name):
     case "jak3":
       return jak3_files
 
+def is_file_in_game(game_name, file_name):
+  file_list = get_file_list(game_name)
+  for f in file_list:
+    if f[2] != 3 and f[2] != 5:
+      continue
+    if f[0] == file_name:
+      return True
+  return False
+
 def get_gsrc_path_from_filename(game_name, file_name):
   file_list = get_file_list(game_name)
   src_path = ""
@@ -53,3 +63,21 @@ def get_ref_path_from_filename(game_name, file_name, ref_folder):
     exit(1)
   path = os.path.join(ref_folder, game_name, src_path, "{}_REF.gc".format(file_name))
   return path
+
+def decompile_file(decompiler_path, decompiler_config, game_version, file_names, omit_var_casts=False):
+  decompiler_args = '{{"levels_extract": false, "process_art_groups": false, "decompile_code": true, "allowed_objects": {}}}'.format(file_names)
+  if omit_var_casts:
+    decompiler_args = '{{"levels_extract": false, "process_art_groups": false, "decompile_code": true, "ignore_var_name_casts": true, "allowed_objects": {}}}'.format(file_names)
+  subprocess.run(
+    [
+        decompiler_path,
+        "./decompiler/config/{}".format(decompiler_config),
+        "./iso_data",
+        "./decompiler_out",
+        "--version",
+        game_version,
+        "--config-override",
+        decompiler_args,
+    ],
+    stdout = subprocess.DEVNULL
+)