Recognize auto-generated inspect methods and create deftypes from them (#95)

- Recognize new type definitions/parents/type flags in the decompiler
- Analyze autogenerated inspect methods and dump guesses at fields to a file
- Utility functions for accessing static data by label
- Better ordering in the decompiler to go through functions in the order they appeared in the source
- Added a decent number of types to `all-types.gc` based on the new field analyzer
- Correct a few `int`/`integer` mistakes in `gcommon.gc` (this should really be a warning)
- Correct a few type issues in `gcommon` and `gkernel-h`
- Option in the decompiler to be strict about `define-extern` redefining a type of a symbol
- Add a test to check consistency in types between `all-types.gc` (used by decompiler) and `goal_src` (used by the compiler)

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -794,8 +794,11 @@ std::string LinkedObjectFile::print_disassembly() {
 /*!
  * Hacky way to get a GOAL string object
  */
-std::string LinkedObjectFile::get_goal_string(int seg, int word_idx) {
-  std::string result = "\"";
+std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_quotes) {
+  std::string result;
+  if (with_quotes) {
+    result += "\"";
+  }
   // next should be the size
   if (word_idx + 1 >= int(words_by_seg[seg].size())) {
     return "invalid string!\n";
@@ -819,7 +822,10 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx) {
     memcpy(cword, &word.data, 4);
     result += cword[byte_offset];
   }
-  return result + "\"";
+  if (with_quotes) {
+    result += "\"";
+  }
+  return result;
 }
 
 /*!
@@ -997,4 +1003,16 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
   }
 
   return result;
+}
+
+u32 LinkedObjectFile::read_data_word(const Label& label) {
+  assert(0 == (label.offset % 4));
+  auto& word = words_by_seg.at(label.target_segment).at(label.offset / 4);
+  assert(word.kind == LinkedWord::Kind::PLAIN_DATA);
+  return word.data;
+}
+
+std::string LinkedObjectFile::get_goal_string_by_label(const Label& label) {
+  assert(0 == (label.offset % 4));
+  return get_goal_string(label.target_segment, (label.offset / 4) - 1, false);
 }

decompiler/ObjectFile/LinkedObjectFile.h

@@ -68,6 +68,9 @@ class LinkedObjectFile {
                                          const std::string& extra_name);
   std::string print_asm_function_disassembly(const std::string& my_name);
 
+  u32 read_data_word(const Label& label);
+  std::string get_goal_string_by_label(const Label& label);
+
   struct Stats {
     uint32_t total_code_bytes = 0;
     uint32_t total_v2_code_bytes = 0;
@@ -134,7 +137,7 @@ class LinkedObjectFile {
   goos::Object to_form_script_object(int seg, int byte_idx, std::vector<bool>& seen);
   bool is_empty_list(int seg, int byte_idx);
   bool is_string(int seg, int byte_idx);
-  std::string get_goal_string(int seg, int word_idx);
+  std::string get_goal_string(int seg, int word_idx, bool with_quotes = true);
 
   std::vector<std::unordered_map<int, int>> label_per_seg_by_offset;
 };

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -20,6 +20,7 @@
 #include "decompiler/Function/BasicBlocks.h"
 #include "decompiler/IR/BasicOpBuilder.h"
 #include "decompiler/IR/CfgBuilder.h"
+#include "decompiler/Function/TypeInspector.h"
 #include "third-party/spdlog/include/spdlog/spdlog.h"
 #include "third-party/json.hpp"
 
@@ -625,7 +626,8 @@ void ObjectFileDB::analyze_functions() {
         assert(func.guessed_name.empty());
         func.guessed_name.set_as_top_level();
         func.find_global_function_defs(data.linked_data, dts);
-        func.find_method_defs(data.linked_data);
+        func.find_type_defs(data.linked_data, dts);
+        func.find_method_defs(data.linked_data, dts);
       }
     });
 
@@ -687,10 +689,11 @@ void ObjectFileDB::analyze_functions() {
   int successful_type_analysis = 0;
 
   std::map<int, std::vector<std::string>> unresolved_by_length;
+
   if (get_config().find_basic_blocks) {
     timer.start();
     int total_basic_blocks = 0;
-    for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
+    for_each_function_def_order([&](Function& func, int segment_id, ObjectFileData& data) {
       //      printf("in %s from %s\n", func.guessed_name.to_string().c_str(),
       //             data.to_unique_name().c_str());
       auto blocks = find_blocks_in_function(data.linked_data, segment_id, func);
@@ -718,6 +721,12 @@ void ObjectFileDB::analyze_functions() {
         total_basic_ops += func.get_basic_op_count();
         total_failed_basic_ops += func.get_failed_basic_op_count();
 
+        if (func.is_inspect_method) {
+          auto result = inspect_inspect_method(func, func.method_of_type, dts, data.linked_data);
+          all_type_defs += ";; " + data.to_unique_name() + "\n";
+          all_type_defs += result.print_as_deftype() + "\n";
+        }
+
         // Combine basic ops + CFG to build a nested IR
         func.ir = build_cfg_ir(func, *func.cfg, data.linked_data);
         non_asm_funcs++;
@@ -744,11 +753,13 @@ void ObjectFileDB::analyze_functions() {
             }
             // GOOD!
             func.type = kv->second;
+
             /*
             spdlog::info("Type Analysis on {} {}", func.guessed_name.to_string(),
                          kv->second.print());
             func.run_type_analysis(kv->second, dts, data.linked_data);
              */
+
             if (func.has_typemaps()) {
               successful_type_analysis++;
             }

decompiler/ObjectFile/ObjectFileDB.h

@@ -57,6 +57,7 @@ class ObjectFileDB {
   void analyze_functions();
   ObjectFileData& lookup_record(const ObjectFileRecord& rec);
   DecompilerTypeSystem dts;
+  std::string all_type_defs;
 
  private:
   void load_map_file(const std::string& map_data);
@@ -101,6 +102,23 @@ class ObjectFileDB {
     });
   }
 
+  template <typename Func>
+  void for_each_function_def_order(Func f) {
+    for_each_obj([&](ObjectFileData& data) {
+      //      printf("IN %s\n", data.record.to_unique_name().c_str());
+      for (int i = 0; i < int(data.linked_data.segments); i++) {
+        //        printf("seg %d\n", i);
+        int fn = 0;
+        //        for (auto& goal_func : data.linked_data.functions_by_seg.at(i)) {
+        for (size_t j = data.linked_data.functions_by_seg.at(i).size(); j-- > 0;) {
+          //          printf("fn %d\n", fn);
+          f(data.linked_data.functions_by_seg.at(i).at(j), i, data);
+          fn++;
+        }
+      }
+    });
+  }
+
   // Danger: after adding all object files, we assume that the vector never reallocates.
   std::unordered_map<std::string, std::vector<ObjectFileData>> obj_files_by_name;
   std::unordered_map<std::string, std::vector<ObjectFileRecord>> obj_files_by_dgo;