[Decompiler] Write IR2 to file and implement some Atomic Op conversions (#187)

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -989,7 +989,7 @@ goos::Object LinkedObjectFile::to_form_script(int seg, int word_idx, std::vector
 /*!
  * Is the thing pointed to a string?
  */
-bool LinkedObjectFile::is_string(int seg, int byte_idx) {
+bool LinkedObjectFile::is_string(int seg, int byte_idx) const {
   if (byte_idx % 4) {
     return false;  // must be aligned pointer.
   }

decompiler/ObjectFile/LinkedObjectFile.h

@@ -63,6 +63,8 @@ class LinkedObjectFile {
 
   u32 read_data_word(const DecompilerLabel& label);
   std::string get_goal_string_by_label(const DecompilerLabel& label) const;
+  std::string get_goal_string(int seg, int word_idx, bool with_quotes = true) const;
+  bool is_string(int seg, int byte_idx) const;
 
   struct Stats {
     uint32_t total_code_bytes = 0;
@@ -129,8 +131,6 @@ class LinkedObjectFile {
   goos::Object to_form_script(int seg, int word_idx, std::vector<bool>& seen);
   goos::Object to_form_script_object(int seg, int byte_idx, std::vector<bool>& seen);
   bool is_empty_list(int seg, int byte_idx);
-  bool is_string(int seg, int byte_idx);
-  std::string get_goal_string(int seg, int word_idx, bool with_quotes = true) const;
 
   std::vector<std::unordered_map<int, int>> label_per_seg_by_offset;
 };

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -126,19 +126,19 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
         "consistent naming when doing a partial decompilation.");
   }
 
-  lg::info("-Loading DGOs...");
+  lg::info("-Loading {} DGOs...", _dgos.size());
   for (auto& dgo : _dgos) {
     get_objs_from_dgo(dgo);
   }
 
-  lg::info("-Loading plain object files...");
+  lg::info("-Loading {} plain object files...", object_files.size());
   for (auto& obj : object_files) {
     auto data = file_util::read_binary_file(obj);
     auto name = obj_filename_to_name(obj);
     add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO");
   }
 
-  lg::info("-Loading streaming object files...");
+  lg::info("-Loading {} streaming object files...", str_files.size());
   for (auto& obj : str_files) {
     StrFileReader reader(obj);
     // name from the file name
@@ -153,15 +153,7 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
     }
   }
 
-  lg::info("ObjectFileDB Initialized:");
-  lg::info("Total DGOs: {}", int(_dgos.size()));
-  lg::info("Total data: {} bytes", stats.total_dgo_bytes);
-  lg::info("Total objs: {}", stats.total_obj_files);
-  lg::info("Unique objs: {}", stats.unique_obj_files);
-  lg::info("Unique data: {} bytes", stats.unique_obj_bytes);
-  lg::info("Total {:.2f} ms ({:.3f} MB/sec, {:.2f} obj/sec)", timer.getMs(),
-           stats.total_dgo_bytes / ((1u << 20u) * timer.getSeconds()),
-           stats.total_obj_files / timer.getSeconds());
+  lg::info("ObjectFileDB Initialized\n");
 }
 
 void ObjectFileDB::load_map_file(const std::string& map_data) {
@@ -481,7 +473,7 @@ std::string ObjectFileDB::generate_obj_listing() {
  * Process all of the linking data of all objects.
  */
 void ObjectFileDB::process_link_data() {
-  lg::info("- Processing Link Data...");
+  lg::info("Processing Link Data...");
   Timer process_link_timer;
 
   LinkedObjectFile::Stats combined_stats;
@@ -491,25 +483,7 @@ void ObjectFileDB::process_link_data() {
     combined_stats.add(obj.linked_data.stats);
   });
 
-  lg::info("Processed Link Data:");
-  lg::info(" Code {} bytes", combined_stats.total_code_bytes);
-  lg::info(" v2 Code {} bytes", combined_stats.total_v2_code_bytes);
-  lg::info(" v2 Link Data {} bytes", combined_stats.total_v2_link_bytes);
-  lg::info(" v2 Pointers {}", combined_stats.total_v2_pointers);
-  lg::info(" v2 Pointer Seeks {}", combined_stats.total_v2_pointer_seeks);
-  lg::info(" v2 Symbols {}", combined_stats.total_v2_symbol_count);
-  lg::info(" v2 Symbol Links {}", combined_stats.total_v2_symbol_links);
-
-  lg::info(" v3 Code {} bytes", combined_stats.v3_code_bytes);
-  lg::info(" v3 Link Data {} bytes", combined_stats.v3_link_bytes);
-  lg::info(" v3 Pointers {}", combined_stats.v3_pointers);
-  lg::info("   Split {}", combined_stats.v3_split_pointers);
-  lg::info("   Word  {}", combined_stats.v3_word_pointers);
-  lg::info(" v3 Pointer Seeks {}", combined_stats.v3_pointer_seeks);
-  lg::info(" v3 Symbols {}", combined_stats.v3_symbol_count);
-  lg::info(" v3 Offset Symbol Links {}", combined_stats.v3_symbol_link_offset);
-  lg::info(" v3 Word Symbol Links {}", combined_stats.v3_symbol_link_word);
-
+  lg::info("Processed Link Data");
   lg::info(" Total {} ms\n", process_link_timer.getMs());
   // printf("\n");
 }
@@ -518,15 +492,14 @@ void ObjectFileDB::process_link_data() {
  * Process all of the labels generated from linking and give them reasonable names.
  */
 void ObjectFileDB::process_labels() {
-  lg::info("- Processing Labels...");
+  lg::info("Processing Labels...");
   Timer process_label_timer;
   uint32_t total = 0;
   for_each_obj([&](ObjectFileData& obj) { total += obj.linked_data.set_ordered_label_names(); });
 
   lg::info("Processed Labels:");
   lg::info(" Total {} labels", total);
-  lg::info(" Total {} ms", process_label_timer.getMs());
-  // printf("\n");
+  lg::info(" Total {} ms\n", process_label_timer.getMs());
 }
 
 /*!
@@ -636,7 +609,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir,
  * Find code/data zones, identify functions, and disassemble
  */
 void ObjectFileDB::find_code() {
-  lg::info("- Finding code in object files...");
+  lg::info("Finding code in object files...");
   LinkedObjectFile::Stats combined_stats;
   Timer timer;
 
@@ -670,8 +643,7 @@ void ObjectFileDB::find_code() {
   auto total_ops = combined_stats.code_bytes / 4;
   lg::info(" Decoded {} / {} ({:.3f} %)", combined_stats.decoded_ops, total_ops,
            100.f * (float)combined_stats.decoded_ops / total_ops);
-  lg::info(" Total {:.3f} ms", timer.getMs());
-  // printf("\n");
+  lg::info(" Total {:.3f} ms\n", timer.getMs());
 }
 
 /*!
@@ -767,7 +739,7 @@ std::string ObjectFileDB::process_game_count_file() {
 /*!
  * This is the main decompiler routine which runs after we've identified functions.
  */
-void ObjectFileDB::analyze_functions() {
+void ObjectFileDB::analyze_functions_ir1() {
   lg::info("- Analyzing Functions...");
   Timer timer;
 

decompiler/ObjectFile/ObjectFileDB.h

@@ -65,7 +65,15 @@ class ObjectFileDB {
                          const std::string& file_suffix = "");
 
   void write_debug_type_analysis(const std::string& output_dir, const std::string& suffix = "");
-  void analyze_functions();
+  void analyze_functions_ir1();
+  void analyze_functions_ir2(const std::string& output_dir);
+  void ir2_top_level_pass();
+  void ir2_basic_block_pass();
+  void ir2_atomic_op_pass();
+  void ir2_write_results(const std::string& output_dir);
+  std::string ir2_to_file(ObjectFileData& data);
+  std::string ir2_function_to_string(ObjectFileData& data, Function& function, int seg);
+
   void process_tpages();
   void analyze_expressions();
   std::string process_game_count_file();

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -0,0 +1,347 @@
+/*!
+ * @file ObjectFileDB_IR2.cpp
+ * This runs the IR2 analysis passes.
+ */
+
+#include "ObjectFileDB.h"
+#include "common/log/log.h"
+#include "common/util/Timer.h"
+#include "common/util/FileUtil.h"
+#include "decompiler/Function/TypeInspector.h"
+
+namespace decompiler {
+
+/*!
+ * Main IR2 analysis pass.
+ * At this point, we assume that the files are loaded and we've run find_code to locate all
+ * functions, but nothing else.
+ */
+void ObjectFileDB::analyze_functions_ir2(const std::string& output_dir) {
+  lg::info("Using IR2 analysis...");
+  lg::info("Processing top-level functions...");
+  ir2_top_level_pass();
+  lg::info("Processing basic blocks and control flow graph...");
+  ir2_basic_block_pass();
+  lg::info("Converting to atomic ops...");
+  ir2_atomic_op_pass();
+  lg::info("Writing results...");
+  ir2_write_results(output_dir);
+}
+
+void ObjectFileDB::ir2_top_level_pass() {
+  Timer timer;
+  int total_functions = 0;
+  int total_named_global_functions = 0;
+  int total_methods = 0;
+  int total_top_levels = 0;
+  int total_unknowns = 0;
+
+  for_each_obj([&](ObjectFileData& data) {
+    if (data.linked_data.segments == 3) {
+      // the top level segment should have a single function
+      assert(data.linked_data.functions_by_seg.at(2).size() == 1);
+
+      auto& func = data.linked_data.functions_by_seg.at(2).front();
+      assert(func.guessed_name.empty());
+      func.guessed_name.set_as_top_level();
+      func.find_global_function_defs(data.linked_data, dts);
+      func.find_type_defs(data.linked_data, dts);
+      func.find_method_defs(data.linked_data, dts);
+    }
+  });
+
+  // check for function uniqueness.
+  std::unordered_set<std::string> unique_names;
+  std::unordered_map<std::string, std::unordered_set<std::string>> duplicated_functions;
+
+  int uid = 1;
+  for_each_obj([&](ObjectFileData& data) {
+    int func_in_obj = 0;
+    for (int segment_id = 0; segment_id < int(data.linked_data.segments); segment_id++) {
+      for (auto& func : data.linked_data.functions_by_seg.at(segment_id)) {
+        func.guessed_name.unique_id = uid++;
+        func.guessed_name.id_in_object = func_in_obj++;
+        func.guessed_name.object_name = data.to_unique_name();
+        auto name = func.guessed_name.to_string();
+
+        switch (func.guessed_name.kind) {
+          case FunctionName::FunctionKind::METHOD:
+            total_methods++;
+            break;
+          case FunctionName::FunctionKind::GLOBAL:
+            total_named_global_functions++;
+            break;
+          case FunctionName::FunctionKind::TOP_LEVEL_INIT:
+            total_top_levels++;
+            break;
+          case FunctionName::FunctionKind::UNIDENTIFIED:
+            total_unknowns++;
+            break;
+          default:
+            assert(false);
+        }
+        total_functions++;
+
+        if (unique_names.find(name) != unique_names.end()) {
+          duplicated_functions[name].insert(data.to_unique_name());
+        }
+
+        unique_names.insert(name);
+
+        if (get_config().asm_functions_by_name.find(name) !=
+            get_config().asm_functions_by_name.end()) {
+          func.warnings += ";; flagged as asm by config\n";
+          func.suspected_asm = true;
+        }
+      }
+    }
+  });
+
+  for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
+    (void)segment_id;
+    auto name = func.guessed_name.to_string();
+
+    if (duplicated_functions.find(name) != duplicated_functions.end()) {
+      duplicated_functions[name].insert(data.to_unique_name());
+      func.warnings += ";; this function exists in multiple non-identical object files\n";
+    }
+  });
+
+  lg::info("Found a total of {} functions in {:.2f} ms", total_functions, timer.getMs());
+  lg::info("{:4d} unknown {:.2f}%", total_unknowns, 100.f * total_unknowns / total_functions);
+  lg::info("{:4d} global  {:.2f}%", total_named_global_functions,
+           100.f * total_named_global_functions / total_functions);
+  lg::info("{:4d} methods {:.2f}%", total_methods, 100.f * total_methods / total_functions);
+  lg::info("{:4d} logins  {:.2f}%\n", total_top_levels, 100.f * total_top_levels / total_functions);
+}
+
+void ObjectFileDB::ir2_basic_block_pass() {
+  Timer timer;
+  // Main Pass over each function...
+  int total_basic_blocks = 0;
+  int total_functions = 0;
+  int functions_with_one_block = 0;
+  int inspect_methods = 0;
+  int suspected_asm = 0;
+  int failed_to_build_cfg = 0;
+
+  for_each_function_def_order([&](Function& func, int segment_id, ObjectFileData& data) {
+    total_functions++;
+
+    // first, find basic blocks.
+    auto blocks = find_blocks_in_function(data.linked_data, segment_id, func);
+    total_basic_blocks += blocks.size();
+    if (blocks.size() == 1) {
+      functions_with_one_block++;
+    }
+    func.basic_blocks = blocks;
+
+    if (!func.suspected_asm) {
+      // find the prologue/epilogue so they can be excluded from basic blocks.
+      func.analyze_prologue(data.linked_data);
+    }
+
+    if (!func.suspected_asm) {
+      // run analysis
+
+      // build a control flow graph, just looking at branch instructions.
+      func.cfg = build_cfg(data.linked_data, segment_id, func);
+      if (!func.cfg->is_fully_resolved()) {
+        lg::warn("Function {} from {} failed to build control flow graph!",
+                 func.guessed_name.to_string(), data.to_unique_name());
+        failed_to_build_cfg++;
+      }
+
+      // if we got an inspect method, inspect it.
+      if (func.is_inspect_method) {
+        auto result = inspect_inspect_method(func, func.method_of_type, dts, data.linked_data);
+        all_type_defs += ";; " + data.to_unique_name() + "\n";
+        all_type_defs += result.print_as_deftype() + "\n";
+        inspect_methods++;
+      }
+    }
+
+    if (func.suspected_asm) {
+      suspected_asm++;
+    }
+  });
+
+  lg::info("Found {} basic blocks in {} functions in {:.2f} ms:", total_basic_blocks,
+           total_functions, timer.getMs());
+  lg::info(" {} functions ({:.2f}%) failed to build control flow graph", failed_to_build_cfg,
+           100.f * failed_to_build_cfg / total_functions);
+  lg::info(" {} functions ({:.2f}%) had exactly one basic block", functions_with_one_block,
+           100.f * functions_with_one_block / total_functions);
+  lg::info(" {} functions ({:.2f}%) were ignored as assembly", suspected_asm,
+           100.f * suspected_asm / total_functions);
+  lg::info(" {} functions ({:.2f}%) were inspect methods\n", inspect_methods,
+           100.f * inspect_methods / total_functions);
+}
+
+void ObjectFileDB::ir2_atomic_op_pass() {
+  Timer timer;
+  int total_functions = 0;
+  int attempted = 0;
+  int successful = 0;
+  for_each_function_def_order([&](Function& func, int segment_id, ObjectFileData& data) {
+    (void)segment_id;
+    total_functions++;
+    if (!func.suspected_asm) {
+      func.ir2.atomic_ops_attempted = true;
+      attempted++;
+      try {
+        auto ops = convert_function_to_atomic_ops(func, data.linked_data.labels);
+        func.ir2.atomic_ops = std::make_shared<FunctionAtomicOps>(std::move(ops));
+        func.ir2.atomic_ops_succeeded = true;
+        successful++;
+      } catch (std::exception& e) {
+        lg::warn("Function {} from {} could not be converted to atomic ops: {}",
+                 func.guessed_name.to_string(), data.to_unique_name(), e.what());
+      }
+    }
+  });
+
+  lg::info("{}/{}/{} (successful/attempted/total) functions converted to Atomic Ops in {:.2f} ms",
+           successful, attempted, total_functions, timer.getMs());
+  lg::info("{:.2f}% were attempted, {:.2f}% of attempted succeeded\n",
+           100.f * attempted / total_functions, 100.f * successful / attempted);
+}
+
+void ObjectFileDB::ir2_write_results(const std::string& output_dir) {
+  Timer timer;
+  lg::info("Writing IR2 results to file...");
+  int total_files = 0;
+  int total_bytes = 0;
+  for_each_obj([&](ObjectFileData& obj) {
+    if (obj.linked_data.has_any_functions()) {
+      // todo
+      total_files++;
+      auto file_text = ir2_to_file(obj);
+      total_bytes += file_text.length();
+      auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + "_ir2.asm");
+
+      file_util::write_text_file(file_name, file_text);
+    }
+  });
+  lg::info("Wrote {} files ({:.2f} MB) in {:.2f} ms\n", total_files, total_bytes / float(1 << 20),
+           timer.getMs());
+}
+
+std::string ObjectFileDB::ir2_to_file(ObjectFileData& data) {
+  std::string result;
+
+  const char* segment_names[] = {"main segment", "debug segment", "top-level segment"};
+  assert(data.linked_data.segments <= 3);
+  for (int seg = data.linked_data.segments; seg-- > 0;) {
+    // segment header
+    result += ";------------------------------------------\n;  ";
+    result += segment_names[seg];
+    result += "\n;------------------------------------------\n\n";
+
+    // functions
+    for (auto& func : data.linked_data.functions_by_seg.at(seg)) {
+      result += ir2_function_to_string(data, func, seg);
+    }
+
+    // print data
+    for (size_t i = data.linked_data.offset_of_data_zone_by_seg.at(seg);
+         i < data.linked_data.words_by_seg.at(seg).size(); i++) {
+      for (int j = 0; j < 4; j++) {
+        auto label_id = data.linked_data.get_label_at(seg, i * 4 + j);
+        if (label_id != -1) {
+          result += data.linked_data.labels.at(label_id).name + ":";
+          if (j != 0) {
+            result += " (offset " + std::to_string(j) + ")";
+          }
+          result += "\n";
+        }
+      }
+
+      auto& word = data.linked_data.words_by_seg[seg][i];
+      data.linked_data.append_word_to_string(result, word);
+
+      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+        result += "; " + data.linked_data.get_goal_string(seg, i) + "\n";
+      }
+    }
+  }
+
+  return result;
+}
+
+std::string ObjectFileDB::ir2_function_to_string(ObjectFileData& data, Function& func, int seg) {
+  std::string result;
+  result += ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n";
+  result += "; .function " + func.guessed_name.to_string() + "\n";
+  result += ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n";
+  result += func.prologue.to_string(2) + "\n";
+  if (!func.warnings.empty()) {
+    result += ";;Warnings:\n" + func.warnings + "\n";
+  }
+
+  bool print_atomics = func.ir2.atomic_ops_succeeded;
+  // print each instruction in the function.
+  bool in_delay_slot = false;
+
+  for (int i = 1; i < func.end_word - func.start_word; i++) {
+    // check for a label to print
+    auto label_id = data.linked_data.get_label_at(seg, (func.start_word + i) * 4);
+    if (label_id != -1) {
+      result += data.linked_data.labels.at(label_id).name + ":\n";
+    }
+
+    // check for no misaligned labels in code segments.
+    for (int j = 1; j < 4; j++) {
+      assert(data.linked_data.get_label_at(seg, (func.start_word + i) * 4 + j) == -1);
+    }
+
+    // print the assembly instruction
+    auto& instr = func.instructions.at(i);
+    std::string line = "    " + instr.to_string(data.linked_data.labels);
+
+    //    printf("%d inst %s\n", print_atomics, instr.to_string(data.linked_data.labels).c_str());
+
+    bool printed_comment = false;
+
+    // print atomic op
+    if (print_atomics && func.instr_starts_atomic_op(i)) {
+      if (line.length() < 30) {
+        line.append(30 - line.length(), ' ');
+      }
+      line +=
+          " ;; " + func.get_atomic_op_at_instr(i).to_string(data.linked_data.labels, &func.ir2.env);
+      printed_comment = true;
+    }
+
+    // print linked strings
+    for (int iidx = 0; iidx < instr.n_src; iidx++) {
+      if (instr.get_src(iidx).is_label()) {
+        auto lab = data.linked_data.labels.at(instr.get_src(iidx).get_label());
+        if (data.linked_data.is_string(lab.target_segment, lab.offset)) {
+          if (!printed_comment) {
+            line += " ;; ";
+            printed_comment = true;
+          }
+          line += " " + data.linked_data.get_goal_string(lab.target_segment, lab.offset / 4 - 1);
+        }
+      }
+    }
+    result += line + "\n";
+
+    // print delay slot gap
+    if (in_delay_slot) {
+      result += "\n";
+      in_delay_slot = false;
+    }
+
+    // for next time...
+    if (gOpcodeInfo[(int)instr.kind].has_delay_slot) {
+      in_delay_slot = true;
+    }
+  }
+  result += "\n";
+
+  return result;
+}
+
+}  // namespace decompiler