[decompiler] automatically label things when possible (#784)

* improve label system

* clean up menu

* debug menu working, still need to fix tests

* fix tests and clean up

decompiler/ObjectFile/LinkedObjectFile.h

@@ -14,6 +14,7 @@
 #include "decompiler/Disasm/DecompilerLabel.h"
 #include "decompiler/Function/Function.h"
 #include "common/common_types.h"
+#include "decompiler/IR2/LabelDB.h"
 
 namespace decompiler {
 /*!
@@ -128,6 +129,8 @@ class LinkedObjectFile {
   std::vector<std::vector<Function>> functions_by_seg;
   std::vector<DecompilerLabel> labels;
 
+  std::unique_ptr<LabelDB> label_db;
+
  private:
   goos::Object to_form_script(int seg, int word_idx, std::vector<bool>& seen);
   goos::Object to_form_script_object(int seg, int byte_idx, std::vector<bool>& seen);

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -331,7 +331,7 @@ std::string ObjectFileDB::generate_dgo_listing() {
   for (const auto& name : dgo_names) {
     result += "(\"" + name + "\"\n";
     for (auto& obj_rec : obj_files_by_dgo[name]) {
-      auto obj = lookup_record(obj_rec);
+      auto& obj = lookup_record(obj_rec);
       std::string extension = ".o";
       if (obj.obj_version == 4 || obj.obj_version == 2) {
         extension = ".go";

decompiler/ObjectFile/ObjectFileDB.h

@@ -81,7 +81,9 @@ class ObjectFileDB {
   void ir2_insert_anonymous_functions(int seg);
   void ir2_symbol_definition_map(const std::string& output_dir);
   void ir2_write_results(const std::string& output_dir, const Config& config);
-  void ir2_do_common_segment_analysis(int seg, const Config& config);
+  void ir2_do_segment_analysis_phase1(int seg, const Config& config);
+  void ir2_do_segment_analysis_phase2(int seg, const Config& config);
+  void ir2_setup_labels(const Config& config);
   std::string ir2_to_file(ObjectFileData& data, const Config& config);
   std::string ir2_function_to_string(ObjectFileData& data, Function& function, int seg);
   std::string ir2_final_out(ObjectFileData& data,

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -12,6 +12,7 @@
 #include "decompiler/analysis/type_analysis.h"
 #include "decompiler/analysis/reg_usage.h"
 #include "decompiler/analysis/insert_lets.h"
+#include "decompiler/analysis/label_types.h"
 #include "decompiler/analysis/find_defstates.h"
 #include "decompiler/analysis/variable_naming.h"
 #include "decompiler/analysis/cfg_builder.h"
@@ -40,7 +41,13 @@ void ObjectFileDB::analyze_functions_ir2(const std::string& output_dir,
   lg::info("Processing top-level functions...");
   ir2_top_level_pass(config);
 
-  ir2_do_common_segment_analysis(TOP_LEVEL_SEGMENT, config);
+  ir2_do_segment_analysis_phase1(TOP_LEVEL_SEGMENT, config);
+  ir2_do_segment_analysis_phase1(DEBUG_SEGMENT, config);
+  ir2_do_segment_analysis_phase1(MAIN_SEGMENT, config);
+
+  ir2_setup_labels(config);
+
+  ir2_do_segment_analysis_phase2(TOP_LEVEL_SEGMENT, config);
 
   for_each_obj([&](ObjectFileData& data) {
     try {
@@ -50,8 +57,8 @@ void ObjectFileDB::analyze_functions_ir2(const std::string& output_dir,
     }
   });
 
-  ir2_do_common_segment_analysis(DEBUG_SEGMENT, config);
-  ir2_do_common_segment_analysis(MAIN_SEGMENT, config);
+  ir2_do_segment_analysis_phase2(DEBUG_SEGMENT, config);
+  ir2_do_segment_analysis_phase2(MAIN_SEGMENT, config);
 
   if (config.generate_symbol_definition_map) {
     lg::info("Generating symbol definition map...");
@@ -75,13 +82,20 @@ void ObjectFileDB::analyze_functions_ir2(const std::string& output_dir,
   }
 }
 
-void ObjectFileDB::ir2_do_common_segment_analysis(int seg, const Config& config) {
+void ObjectFileDB::ir2_do_segment_analysis_phase1(int seg, const Config& config) {
+  lg::info("COMMON ANALYSIS 1 {}", seg);
+
   lg::info("Processing basic blocks and control flow graph...");
   ir2_basic_block_pass(seg, config);
   lg::info("Finding stack spills...");
   ir2_stack_spill_slot_pass(seg);
   lg::info("Converting to atomic ops...");
   ir2_atomic_op_pass(seg, config);
+}
+
+void ObjectFileDB::ir2_do_segment_analysis_phase2(int seg, const Config& config) {
+  lg::info("COMMON ANALYSIS 2 {}", seg);
+
   lg::info("Running type analysis...");
   ir2_type_analysis_pass(seg, config);
   lg::info("Register usage analysis...");
@@ -100,6 +114,25 @@ void ObjectFileDB::ir2_do_common_segment_analysis(int seg, const Config& config)
   ir2_insert_lets(seg);
 }
 
+void ObjectFileDB::ir2_setup_labels(const Config& config) {
+  for_each_obj([&](ObjectFileData& data) {
+    if (data.linked_data.segments == 3) {
+      std::unordered_map<std::string, LabelConfigInfo> config_labels;
+      auto config_it = config.label_types.find(data.to_unique_name());
+      if (config_it != config.label_types.end()) {
+        config_labels = config_it->second;
+      }
+      try {
+        data.linked_data.label_db =
+            std::make_unique<LabelDB>(config_labels, data.linked_data.labels, dts);
+        analyze_labels(data.linked_data.label_db.get(), &data.linked_data);
+      } catch (const std::exception& e) {
+        lg::die("Error parsing labels for {}: {}\n", data.to_unique_name(), e.what());
+      }
+    }
+  });
+}
+
 /*!
  * Analyze the top level function of each object.
  * - Find global function definitions
@@ -411,8 +444,6 @@ void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config) {
         auto register_casts =
             try_lookup(config.register_type_casts_by_function_by_atomic_op_idx, func_name);
         func.ir2.env.set_type_casts(register_casts);
-        auto label_types = try_lookup(config.label_types, data.to_unique_name());
-        func.ir2.env.set_label_types(label_types);
         auto stack_casts =
             try_lookup(config.stack_type_casts_by_function_by_stack_offset, func_name);
         func.ir2.env.set_stack_casts(stack_casts);