Add decompiler IR, basic operations, all-types file (#57)

* framework for basic op

* started IR framework

* check in type info file

* add some basic operations to the first pass ir conversion

* use a single condition system

* add more basic op decoding

* more ir

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -483,6 +483,8 @@ void LinkedObjectFile::process_fp_relative_links() {
                 if (pprev_instr && pprev_instr->kind == InstructionKind::LUI) {
                   assert(pprev_instr->get_dst(0).get_reg() == offset_reg);
                   additional_offset = (1 << 16) * pprev_instr->get_imm_src().get_imm();
+                  pprev_instr->get_imm_src().set_label(
+                      get_label_id_for(seg, current_fp + atom.get_imm() + additional_offset));
                 }
                 atom.set_label(
                     get_label_id_for(seg, current_fp + atom.get_imm() + additional_offset));
@@ -554,6 +556,12 @@ std::string LinkedObjectFile::print_disassembly() {
           auto& word = words_by_seg[seg].at(func.start_word + i);
           append_word_to_string(result, word);
         } else {
+          if (func.has_basic_ops() && func.instr_starts_basic_op(i)) {
+            if (line.length() < 40) {
+              line.append(40 - line.length(), ' ');
+            }
+            line += ";; " + func.get_basic_op_at_instr(i)->print(*this);
+          }
           result += line + "\n";
         }
 

decompiler/ObjectFile/LinkedObjectFileCreation.cpp

@@ -8,7 +8,7 @@
 #include <cstring>
 #include "LinkedObjectFileCreation.h"
 #include "decompiler/config.h"
-#include "decompiler/TypeSystem/TypeInfo.h"
+#include "decompiler/util/DecompilerTypeSystem.h"
 
 // There are three link versions:
 // V2 - not really in use anymore, but V4 will resue logic from it (and the game didn't rename the
@@ -86,8 +86,9 @@ static uint32_t c_symlink2(LinkedObjectFile& f,
                            uint32_t link_ptr_offset,
                            SymbolLinkKind kind,
                            const char* name,
-                           int seg_id) {
-  get_type_info().inform_symbol_with_no_type_info(name);
+                           int seg_id,
+                           DecompilerTypeSystem& dts) {
+  dts.add_symbol(name);
   auto initial_offset = code_ptr_offset;
   do {
     auto table_value = data.at(link_ptr_offset);
@@ -130,7 +131,7 @@ static uint32_t c_symlink2(LinkedObjectFile& f,
           word_kind = LinkedWord::EMPTY_PTR;
           break;
         case SymbolLinkKind::TYPE:
-          get_type_info().inform_type(name);
+          dts.add_symbol(name, "type");
           word_kind = LinkedWord::TYPE_PTR;
           break;
         default:
@@ -162,8 +163,9 @@ static uint32_t c_symlink3(LinkedObjectFile& f,
                            uint32_t link_ptr,
                            SymbolLinkKind kind,
                            const char* name,
-                           int seg) {
-  get_type_info().inform_symbol_with_no_type_info(name);
+                           int seg,
+                           DecompilerTypeSystem& dts) {
+  dts.add_symbol(name);
   auto initial_offset = code_ptr;
   do {
     // seek, with a variable length encoding that sucks.
@@ -187,7 +189,7 @@ static uint32_t c_symlink3(LinkedObjectFile& f,
           word_kind = LinkedWord::EMPTY_PTR;
           break;
         case SymbolLinkKind::TYPE:
-          get_type_info().inform_type(name);
+          dts.add_symbol(name, "type");
           word_kind = LinkedWord::TYPE_PTR;
           break;
         default:
@@ -223,7 +225,8 @@ static uint32_t align16(uint32_t in) {
  */
 static void link_v4(LinkedObjectFile& f,
                     const std::vector<uint8_t>& data,
-                    const std::string& name) {
+                    const std::string& name,
+                    DecompilerTypeSystem& dts) {
   // read the V4 header to find where the link data really is
   const auto* header = (const LinkHeaderV4*)&data.at(0);
   uint32_t link_data_offset = header->code_size + sizeof(LinkHeaderV4);  // no basic offset
@@ -358,7 +361,7 @@ static void link_v4(LinkedObjectFile& f,
 
       link_ptr_offset += strlen(s_name) + 1;
       f.stats.total_v2_symbol_count++;
-      link_ptr_offset = c_symlink2(f, data, code_offset, link_ptr_offset, kind, s_name, 0);
+      link_ptr_offset = c_symlink2(f, data, code_offset, link_ptr_offset, kind, s_name, 0, dts);
       if (data.at(link_ptr_offset) == 0)
         break;
     }
@@ -384,7 +387,8 @@ static void assert_string_empty_after(const char* str, int size) {
 
 static void link_v5(LinkedObjectFile& f,
                     const std::vector<uint8_t>& data,
-                    const std::string& name) {
+                    const std::string& name,
+                    DecompilerTypeSystem& dts) {
   auto header = (const LinkHeaderV5*)(&data.at(0));
   if (header->n_segments == 1) {
     printf("abandon %s!\n", name.c_str());
@@ -539,10 +543,10 @@ static void link_v5(LinkedObjectFile& f,
 
           if (std::string("_empty_") == sname) {
             link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
-                                  SymbolLinkKind::EMPTY_LIST, sname, seg_id);
+                                  SymbolLinkKind::EMPTY_LIST, sname, seg_id, dts);
           } else {
             link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
-                                  SymbolLinkKind::SYMBOL, sname, seg_id);
+                                  SymbolLinkKind::SYMBOL, sname, seg_id, dts);
           }
         } else if ((reloc & 0x3f) == 0x3f) {
           assert(false);  // todo, does this ever get hit?
@@ -556,7 +560,7 @@ static void link_v5(LinkedObjectFile& f,
           const char* sname = (const char*)(&data.at(link_ptr));
           link_ptr += strlen(sname) + 1;
           link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
-                                SymbolLinkKind::TYPE, sname, seg_id);
+                                SymbolLinkKind::TYPE, sname, seg_id, dts);
         }
 
         sub_link_ptr = link_ptr;
@@ -586,7 +590,8 @@ static void link_v5(LinkedObjectFile& f,
 
 static void link_v3(LinkedObjectFile& f,
                     const std::vector<uint8_t>& data,
-                    const std::string& name) {
+                    const std::string& name,
+                    DecompilerTypeSystem& dts) {
   auto header = (const LinkHeaderV3*)(&data.at(0));
   assert(name == header->name);
   assert(header->segments == 3);
@@ -739,7 +744,7 @@ static void link_v3(LinkedObjectFile& f,
         // methods todo
 
         s_name = (const char*)(&data.at(link_ptr));
-        get_type_info().inform_type_method_count(s_name, reloc & 0x7f);
+        // get_type_info().inform_type_method_count(s_name, reloc & 0x7f); todo
         kind = SymbolLinkKind::TYPE;
       }
 
@@ -750,7 +755,7 @@ static void link_v3(LinkedObjectFile& f,
 
       link_ptr += strlen(s_name) + 1;
       f.stats.v3_symbol_count++;
-      link_ptr = c_symlink3(f, data, base_ptr, link_ptr, kind, s_name, seg_id);
+      link_ptr = c_symlink3(f, data, base_ptr, link_ptr, kind, s_name, seg_id, dts);
     }
     segment_link_ends[seg_id] = link_ptr;
   }
@@ -775,19 +780,21 @@ static void link_v3(LinkedObjectFile& f,
 /*!
  * Main function to generate LinkedObjectFiles from raw object data.
  */
-LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data, const std::string& name) {
+LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data,
+                                       const std::string& name,
+                                       DecompilerTypeSystem& dts) {
   LinkedObjectFile result;
   const auto* header = (const LinkHeaderCommon*)&data.at(0);
 
   // use appropriate linker
   if (header->version == 3) {
     assert(header->type_tag == 0);
-    link_v3(result, data, name);
+    link_v3(result, data, name, dts);
   } else if (header->version == 4) {
     assert(header->type_tag == 0xffffffff);
-    link_v4(result, data, name);
+    link_v4(result, data, name, dts);
   } else if (header->version == 5) {
-    link_v5(result, data, name);
+    link_v5(result, data, name, dts);
   } else {
     assert(false);
   }

decompiler/ObjectFile/LinkedObjectFileCreation.h

@@ -11,6 +11,9 @@
 
 #include "LinkedObjectFile.h"
 
-LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data, const std::string& name);
+class DecompilerTypeSystem;
+LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data,
+                                       const std::string& name,
+                                       DecompilerTypeSystem& dts);
 
 #endif  // NEXT_LINKEDOBJECTFILECREATION_H

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -18,6 +18,7 @@
 #include "common/util/Timer.h"
 #include "common/util/FileUtil.h"
 #include "decompiler/Function/BasicBlocks.h"
+#include "decompiler/IR/BasicOpBuilder.h"
 
 /*!
  * Get a unique name for this object file.
@@ -64,6 +65,8 @@ ObjectFileData& ObjectFileDB::lookup_record(ObjectFileRecord rec) {
  */
 ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos) {
   Timer timer;
+  printf("- Loading Types...\n");
+  dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
 
   printf("- Initializing ObjectFileDB...\n");
   for (auto& dgo : _dgos) {
@@ -71,7 +74,7 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos) {
   }
 
   printf("ObjectFileDB Initialized:\n");
-  printf(" total dgos: %lld\n", _dgos.size());
+  printf(" total dgos: %d\n", int(_dgos.size()));
   printf(" total data: %d bytes\n", stats.total_dgo_bytes);
   printf(" total objs: %d\n", stats.total_obj_files);
   printf(" unique objs: %d\n", stats.unique_obj_files);
@@ -356,7 +359,7 @@ void ObjectFileDB::process_link_data() {
   LinkedObjectFile::Stats combined_stats;
 
   for_each_obj([&](ObjectFileData& obj) {
-    obj.linked_data = to_linked_object_file(obj.data, obj.record.name);
+    obj.linked_data = to_linked_object_file(obj.data, obj.record.name, dts);
     combined_stats.add(obj.linked_data.stats);
   });
 
@@ -543,7 +546,7 @@ void ObjectFileDB::analyze_functions() {
         auto& func = data.linked_data.functions_by_seg.at(2).front();
         assert(func.guessed_name.empty());
         func.guessed_name.set_as_top_level();
-        func.find_global_function_defs(data.linked_data);
+        func.find_global_function_defs(data.linked_data, dts);
         func.find_method_defs(data.linked_data);
       }
     });
@@ -591,6 +594,8 @@ void ObjectFileDB::analyze_functions() {
   int total_nontrivial_functions = 0;
   int total_resolved_nontrivial_functions = 0;
   int total_named_functions = 0;
+  int total_basic_ops = 0;
+  int total_failed_basic_ops = 0;
 
   std::map<int, std::vector<std::string>> unresolved_by_length;
   if (get_config().find_basic_blocks) {
@@ -601,10 +606,18 @@ void ObjectFileDB::analyze_functions() {
       total_basic_blocks += blocks.size();
       func.basic_blocks = blocks;
 
+      total_functions++;
       if (!func.suspected_asm) {
         func.analyze_prologue(data.linked_data);
         func.cfg = build_cfg(data.linked_data, segment_id, func);
-        total_functions++;
+        for (auto& block : func.basic_blocks) {
+          if (block.end_word > block.start_word) {
+            add_basic_ops_to_block(&func, block, &data.linked_data);
+          }
+        }
+        total_basic_ops += func.get_basic_op_count();
+        total_failed_basic_ops += func.get_failed_basic_op_count();
+
         if (func.cfg->is_fully_resolved()) {
           resolved_cfg_functions++;
         }
@@ -640,11 +653,15 @@ void ObjectFileDB::analyze_functions() {
            total_nontrivial_functions,
            100.f * float(total_resolved_nontrivial_functions) / float(total_nontrivial_functions));
 
-    for (auto& kv : unresolved_by_length) {
-      printf("LEN %d\n", kv.first);
-      for (auto& x : kv.second) {
-        printf("  %s\n", x.c_str());
-      }
-    }
+    int successful_basic_ops = total_basic_ops - total_failed_basic_ops;
+    printf(" %d/%d basic ops converted successfully (%.2f%%)\n", successful_basic_ops,
+           total_basic_ops, 100.f * float(successful_basic_ops) / float(total_basic_ops));
+
+    //    for (auto& kv : unresolved_by_length) {
+    //      printf("LEN %d\n", kv.first);
+    //      for (auto& x : kv.second) {
+    //        printf("  %s\n", x.c_str());
+    //      }
+    //    }
   }
 }

decompiler/ObjectFile/ObjectFileDB.h

@@ -15,6 +15,7 @@
 #include <unordered_map>
 #include <vector>
 #include "LinkedObjectFile.h"
+#include "decompiler/util/DecompilerTypeSystem.h"
 
 /*!
  * A "record" which can be used to identify an object file.
@@ -55,6 +56,7 @@ class ObjectFileDB {
   void write_disassembly(const std::string& output_dir, bool disassemble_objects_without_functions);
   void analyze_functions();
   ObjectFileData& lookup_record(ObjectFileRecord rec);
+  DecompilerTypeSystem dts;
 
  private:
   void get_objs_from_dgo(const std::string& filename);