[decomp] better handling of animation code and art files (#1352)

* update refs * [decompiler] read and process art groups * finish decompiler art group selection & detect in `ja-group?` * make art stuff work on offline tests! * [decompiler] detect `ja-group!` (primitive) * corrections. * more * use new feature on skel groups! * find `loop!` as well * fully fledged `ja` macro & decomp + `loop` detect * fancy fixed point printing! * update source * `:num! max` (i knew i should've done this) * Update jak1_ntsc_black_label.jsonc * hi imports * make compiling the game work * fix `defskelgroup` * clang * update refs * fix chan * fix seek and finalboss * fix tests * delete unused function * track let rewrite stats * reorder `rewrite_let` * Update .gitattributes * fix bug with `:num! max` * Update robotboss-part.gc * Update goal-lib.gc * document `ja` * get rid of pc fixes thing * use std::abs
2026-07-11 07:25:37 -04:00 · 2022-05-20 02:30:14 +01:00
parent 971a69e15c
commit 0212aa10c9
786 changed files with 21658 additions and 49501 deletions
@@ -65,6 +65,9 @@ class LinkedObjectFile {
  u32 read_data_word(const DecompilerLabel& label);
  const DecompilerLabel& get_label_by_name(const std::string& name) const;
  std::string get_goal_string_by_label(const DecompilerLabel& label) const;
+  std::string get_goal_string_by_label(int label_id) const {
+    return get_goal_string_by_label(labels.at(label_id));
+  }
  std::string get_goal_string(int seg, int word_idx, bool with_quotes = true) const;
  bool is_string(int seg, int byte_idx) const;

@@ -667,124 +667,94 @@ std::string ObjectFileDB::process_game_count_file() {
  return result;
 }

+namespace {
+void get_art_info(ObjectFileDB& db, ObjectFileData& obj) {
+  if (obj.obj_version == 4) {
+    const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
+    if (words.at(0).kind() == LinkedWord::Kind::TYPE_PTR &&
+        words.at(0).symbol_name() == "art-group") {
+      // fmt::print("art-group {}:\n", obj.to_unique_name());
+      auto name = obj.linked_data.get_goal_string_by_label(words.at(2).label_id());
+      int length = words.at(3).data;
+      // fmt::print("  length: {}\n", length);
+      std::unordered_map<int, std::string> art_group_elts;
+      for (int i = 0; i < length; ++i) {
+        const auto& word = words.at(8 + i);
+        if (word.kind() == LinkedWord::Kind::SYM_PTR && word.symbol_name() == "#f") {
+          continue;
+        }
+        const auto& label = obj.linked_data.labels.at(word.label_id());
+        auto elt_name =
+            obj.linked_data.get_goal_string_by_label(words.at(label.offset / 4 + 1).label_id());
+        std::string elt_type = words.at(label.offset / 4 - 1).symbol_name();
+        std::string unique_name = elt_name;
+        if (elt_type == "art-joint-geo") {
+          // the skeleton!
+          unique_name += "-jg";
+        } else if (elt_type == "merc-ctrl" || elt_type == "shadow-geo") {
+          // (maybe mesh-geo as well but that doesnt exist)
+          // the skin!
+          unique_name += "-mg";
+        } else if (elt_type == "art-joint-anim") {
+          // the animations!
+          unique_name += "-ja";
+        } else {
+          // the something idk!
+          throw std::runtime_error(
+              fmt::format("unknown art elt type {} in {}", elt_type, obj.to_unique_name()));
+        }
+        art_group_elts[i] = unique_name;
+        // fmt::print("  {}: {} ({}) -> {}\n", i, elt_name, elt_type, unique_name);
+      }
+      db.dts.art_group_info[obj.to_unique_name()] = art_group_elts;
+    }
+  }
+}
+}  // namespace
+
 /*!
- * This is the main decompiler routine which runs after we've identified functions.
+ * Get information from art groups.
 */
-void ObjectFileDB::analyze_functions_ir1(const Config& config) {
-  lg::info("- Analyzing Functions...");
+void ObjectFileDB::extract_art_info() {
+  lg::info("Processing art groups...");
  Timer timer;

-  int total_functions = 0;
-
-  // Step 1 - analyze the "top level" or "login" code for each object file.
-  // this will give us type definitions, method definitions, and function definitions...
-  lg::info("  - Processing top levels...");
-
-  timer.start();
-  for_each_obj([&](ObjectFileData& data) {
-    if (data.linked_data.segments == 3) {
-      // the top level segment should have a single function
-      ASSERT(data.linked_data.functions_by_seg.at(2).size() == 1);
-
-      auto& func = data.linked_data.functions_by_seg.at(2).front();
-      ASSERT(func.guessed_name.empty());
-      func.guessed_name.set_as_top_level(data.to_unique_name());
-      func.find_global_function_defs(data.linked_data, dts);
-      func.find_type_defs(data.linked_data, dts);
-      func.find_method_defs(data.linked_data, dts);
+  for_each_obj([&](ObjectFileData& obj) {
+    try {
+      get_art_info(*this, obj);
+    } catch (std::runtime_error& e) {
+      lg::warn("Error when extracting art group info: {}", e.what());
    }
  });

-  // check for function uniqueness.
-  std::unordered_set<std::string> unique_names;
-  std::unordered_map<std::string, std::unordered_set<std::string>> duplicated_functions;
+  lg::info("Processed art groups: in {:.2f} ms\n", timer.getMs());
+}

-  int uid = 1;
-  for_each_obj([&](ObjectFileData& data) {
-    int func_in_obj = 0;
-    for (int segment_id = 0; segment_id < int(data.linked_data.segments); segment_id++) {
-      for (auto& func : data.linked_data.functions_by_seg.at(segment_id)) {
-        func.guessed_name.unique_id = uid++;
-        func.guessed_name.id_in_object = func_in_obj++;
-        func.guessed_name.object_name = data.to_unique_name();
-        auto name = func.name();
+/*!
+ * Write out the art group information.
+ */
+void ObjectFileDB::dump_art_info(const std::string& output_dir) {
+  lg::info("Writing art group info...");
+  Timer timer;

-        if (unique_names.find(name) != unique_names.end()) {
-          duplicated_functions[name].insert(data.to_unique_name());
-        }
-
-        unique_names.insert(name);
-
-        if (config.hacks.asm_functions_by_name.find(name) !=
-            config.hacks.asm_functions_by_name.end()) {
-          func.warnings.info("Flagged as asm by config");
-          func.suspected_asm = true;
-        }
-      }
+  if (!dts.art_group_info.empty()) {
+    file_util::create_dir_if_needed(file_util::combine_path(output_dir, "import"));
+  }
+  for (const auto& [ag_name, info] : dts.art_group_info) {
+    auto ag_fname = ag_name + ".gc";
+    auto filename = file_util::get_file_path({output_dir, "import", ag_fname});
+    std::string result = ";;-*-Lisp-*-\n";
+    result += "(in-package goal)\n\n";
+    result += fmt::format(";; {} - art group OpenGOAL import file\n", ag_fname);
+    result += ";; THIS FILE IS AUTOMATICALLY GENERATED!\n\n";
+    for (const auto& [idx, elt_name] : info) {
+      result += print_art_elt_for_dump(ag_name, elt_name, idx);
    }
-  });
+    result += "\n";
+    file_util::write_text_file(filename, result);
+  }

-  for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
-    (void)segment_id;
-    auto name = func.name();
-
-    if (duplicated_functions.find(name) != duplicated_functions.end()) {
-      duplicated_functions[name].insert(data.to_unique_name());
-      func.warnings.info("Exists in multiple non-identical object files");
-    }
-  });
-
-  int total_trivial_cfg_functions = 0;
-  int total_named_functions = 0;
-
-  int asm_funcs = 0;
-
-  std::map<int, std::vector<std::string>> unresolved_by_length;
-
-  timer.start();
-  int total_basic_blocks = 0;
-
-  // Main Pass over each function...
-  for_each_function_def_order([&](Function& func, int segment_id, ObjectFileData& data) {
-    total_functions++;
-
-    // first, find basic blocks.
-    auto blocks = find_blocks_in_function(data.linked_data, segment_id, func);
-    total_basic_blocks += blocks.size();
-    func.basic_blocks = blocks;
-
-    // analyze the proluge
-    if (!func.suspected_asm) {
-      // first, find the prologue/epilogue
-      func.analyze_prologue(data.linked_data);
-    }
-
-    if (!func.suspected_asm) {
-      // run analysis
-
-      // build a control flow graph, just looking at branch instructions.
-      func.cfg = build_cfg(data.linked_data, segment_id, func, {}, {});
-
-      // convert individual basic blocks to sequences of IR Basic Ops
-      for (auto& block : func.basic_blocks) {
-        if (block.end_word > block.start_word) {
-          auto label_id =
-              data.linked_data.get_label_at(segment_id, (func.start_word + block.start_word) * 4);
-          if (label_id != -1) {
-            block.label_name = data.linked_data.get_label_name(label_id);
-          }
-        }
-      }
-    } else {
-      asm_funcs++;
-    }
-  });
-
-  lg::info("Found {} functions ({} with no control flow)", total_functions,
-           total_trivial_cfg_functions);
-  lg::info("Named {}/{} functions ({:.3f}%)", total_named_functions, total_functions,
-           100.f * float(total_named_functions) / float(total_functions));
-  lg::info("Excluding {} asm functions", asm_funcs);
+  lg::info("Written art group info: in {:.2f} ms\n", timer.getMs());
 }

 void ObjectFileDB::dump_raw_objects(const std::string& output_dir) {
@@ -796,4 +766,10 @@ void ObjectFileDB::dump_raw_objects(const std::string& output_dir) {
    file_util::write_binary_file(dest, data.data.data(), data.data.size());
  });
 }
+
+std::string print_art_elt_for_dump(const std::string& group_name,
+                                   const std::string& name,
+                                   int idx) {
+  return fmt::format("(def-art-elt {} {} {})\n", group_name, name, idx);
+}
 }  // namespace decompiler
@@ -10,9 +10,10 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include "LinkedObjectFile.h"
-#include "decompiler/util/DecompilerTypeSystem.h"
 #include "common/common_types.h"
+#include "LinkedObjectFile.h"
+#include "third-party/fmt/core.h"
+#include "decompiler/util/DecompilerTypeSystem.h"
 #include "decompiler/data/TextureDB.h"
 #include "decompiler/analysis/symbol_def_map.h"
 #include "common/util/Assert.h"
@@ -46,6 +47,28 @@ struct ObjectFileData {
  std::string output_with_skips;
 };

+/*!
+ * Stats structure for let rewriting.
+ */
+struct LetRewriteStats {
+  int dotimes;
+  int countdown;
+  int abs;
+  int abs2;
+  int unused;
+  int ja;
+  int case_no_else;
+  int case_with_else;
+  int set_vector;
+  int set_vector2;
+  int send_event;
+
+  int total() const {
+    return dotimes + countdown + abs + abs2 + unused + ja + case_no_else + case_with_else +
+           set_vector + set_vector2 + send_event;
+  }
+};
+
 class ObjectFileDB {
 public:
  ObjectFileDB(const std::vector<std::string>& _dgos,
@@ -59,6 +82,8 @@ class ObjectFileDB {
  void process_labels();
  void find_code(const Config& config);
  void find_and_write_scripts(const std::string& output_dir);
+  void extract_art_info();
+  void dump_art_info(const std::string& output_dir);
  void dump_raw_objects(const std::string& output_dir);

  void write_object_file_words(const std::string& output_dir, bool dump_data, bool dump_code);
@@ -67,7 +92,6 @@ class ObjectFileDB {
                         bool disassemble_code,
                         bool print_hex);

-  void analyze_functions_ir1(const Config& config);
  void analyze_functions_ir2(
      const std::string& output_dir,
      const Config& config,
@@ -209,10 +233,13 @@ class ObjectFileDB {
  SymbolMapBuilder map_builder;

  struct {
+    LetRewriteStats let;
    uint32_t total_dgo_bytes = 0;
    uint32_t total_obj_files = 0;
    uint32_t unique_obj_files = 0;
    uint32_t unique_obj_bytes = 0;
  } stats;
 };
+
+std::string print_art_elt_for_dump(const std::string& group_name, const std::string& name, int idx);
 }  // namespace decompiler
@@ -58,26 +58,28 @@ void ObjectFileDB::analyze_functions_ir2(
    ir2_do_segment_analysis_phase1(MAIN_SEGMENT, config, data);
    ir2_setup_labels(config, data);
    ir2_do_segment_analysis_phase2(TOP_LEVEL_SEGMENT, config, data);
-    try {
-      if (data.linked_data.functions_by_seg.size() == 3) {
+    if (data.linked_data.functions_by_seg.size() == 3) {
+      enum { DEFPART, DEFSTATE, DEFSKELGROUP } step = DEFPART;
+      try {
        run_defpartgroup(data.linked_data.functions_by_seg.at(TOP_LEVEL_SEGMENT).front());
-      }
-    } catch (const std::exception& e) {
-      lg::error("Failed to find defpartgroups: {}", e.what());
-    }
-    try {
-      if (data.linked_data.functions_by_seg.size() == 3) {
+        step = DEFSTATE;
        run_defstate(data.linked_data.functions_by_seg.at(TOP_LEVEL_SEGMENT).front(), skip_states);
-      }
-    } catch (const std::exception& e) {
-      lg::error("Failed to find defstates: {}", e.what());
-    }
-    try {
-      if (data.linked_data.functions_by_seg.size() == 3) {
+        step = DEFSKELGROUP;
        run_defskelgroups(data.linked_data.functions_by_seg.at(TOP_LEVEL_SEGMENT).front());
+
+      } catch (const std::exception& e) {
+        switch (step) {
+          case DEFPART:
+            lg::error("Failed to find defpartgroups: {}", e.what());
+            break;
+          case DEFSTATE:
+            lg::error("Failed to find defstates: {}", e.what());
+            break;
+          case DEFSKELGROUP:
+            lg::error("Failed to find defskelgroups: {}", e.what());
+            break;
+        }
      }
-    } catch (const std::exception& e) {
-      lg::error("Failed to find defskelgroups: {}", e.what());
    }
    ir2_do_segment_analysis_phase2(DEBUG_SEGMENT, config, data);
    ir2_do_segment_analysis_phase2(MAIN_SEGMENT, config, data);
@@ -104,6 +106,20 @@ void ObjectFileDB::analyze_functions_ir2(
    fmt::print("Done in {:.2f}ms\n", file_timer.getMs());
  });

+  int total = stats.let.total();
+  lg::info("LET REWRITE STATS: {} total", total);
+  lg::info("  dotimes: {}", stats.let.dotimes);
+  lg::info("  countdown: {}", stats.let.countdown);
+  lg::info("  abs: {}", stats.let.abs);
+  lg::info("  abs2: {}", stats.let.abs2);
+  lg::info("  ja: {}", stats.let.ja);
+  lg::info("  set_vector: {}", stats.let.set_vector);
+  lg::info("  set_vector2: {}", stats.let.set_vector2);
+  lg::info("  case_no_else: {}", stats.let.case_no_else);
+  lg::info("  case_with_else: {}", stats.let.case_with_else);
+  lg::info("  unused: {}", stats.let.unused);
+  lg::info("  send_event: {}", stats.let.send_event);
+
  if (config.generate_symbol_definition_map) {
    lg::info("Generating symbol definition map...");
    map_builder.build_map();
@@ -403,6 +419,7 @@ Value try_lookup(const std::unordered_map<Key, Value>& map, const Key& key) {
 * - NOTE: this will update register info usage more accurately for functions.
 */
 void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config, ObjectFileData& data) {
+  auto obj_name = data.to_unique_name();
  for_each_function_in_seg_in_obj(seg, data, [&](Function& func) {
    if (!func.suspected_asm) {
      TypeSpec ts;
@@ -414,9 +431,11 @@ void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config, ObjectF
        auto register_casts =
            try_lookup(config.register_type_casts_by_function_by_atomic_op_idx, func_name);
        func.ir2.env.set_type_casts(register_casts);
+
        auto stack_casts =
            try_lookup(config.stack_type_casts_by_function_by_stack_offset, func_name);
        func.ir2.env.set_stack_casts(stack_casts);
+
        if (config.hacks.pair_functions_by_name.find(func_name) !=
            config.hacks.pair_functions_by_name.end()) {
          func.ir2.env.set_sloppy_pair_typing();
@@ -426,8 +445,18 @@ void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config, ObjectF
            config.hacks.reject_cond_to_value.end()) {
          func.ir2.env.aggressively_reject_cond_to_value_rewrite = true;
        }
+
        func.ir2.env.set_stack_structure_hints(
            try_lookup(config.stack_structure_hints_by_function, func_name));
+
+        if (config.art_groups_by_function.find(func_name) != config.art_groups_by_function.end()) {
+          func.ir2.env.set_art_group(config.art_groups_by_function.at(func_name));
+        } else if (config.art_groups_by_file.find(obj_name) != config.art_groups_by_file.end()) {
+          func.ir2.env.set_art_group(config.art_groups_by_file.at(obj_name));
+        } else {
+          func.ir2.env.set_art_group(obj_name + "-ag");
+        }
+
        if (run_type_analysis_ir2(ts, dts, func)) {
          func.ir2.env.types_succeeded = true;
        } else {
@@ -446,7 +475,7 @@ void ObjectFileDB::ir2_register_usage_pass(int seg, ObjectFileData& data) {
    if (!func.suspected_asm && func.ir2.atomic_ops_succeeded) {
      func.ir2.env.set_reg_use(analyze_ir2_register_usage(func));

-      auto block_0_start = func.ir2.env.reg_use().block.at(0).input;
+      auto& block_0_start = func.ir2.env.reg_use().block.at(0).input;
      std::vector<Register> dep_regs;
      for (auto x : block_0_start) {
        dep_regs.push_back(x);
@@ -572,12 +601,14 @@ void ObjectFileDB::ir2_insert_lets(int seg, ObjectFileData& data) {
  for_each_function_in_seg_in_obj(seg, data, [&](Function& func) {
    if (func.ir2.expressions_succeeded) {
      try {
-        insert_lets(func, func.ir2.env, *func.ir2.form_pool, func.ir2.top_form);
+        insert_lets(func, func.ir2.env, *func.ir2.form_pool, func.ir2.top_form, stats.let);
      } catch (const std::exception& e) {
-        func.warnings.general_warning(
-            fmt::format("Error while inserting lets: {}. Make sure that the return type is not "
-                        "none if something is actually returned.",
-                        e.what()));
+        const auto err = fmt::format(
+            "Error while inserting lets: {}. Make sure that the return type is not "
+            "none if something is actually returned.",
+            e.what());
+        lg::warn(err);
+        func.warnings.general_warning(err);
      }
    }
  });