support type-ref (#613)

common/type_system/TypeSystem.cpp

@@ -38,6 +38,16 @@ TypeSystem::TypeSystem() {
  * throw_on_redefine is set. The type should be fully set up (fields, etc) before running this.
  */
 Type* TypeSystem::add_type(const std::string& name, std::unique_ptr<Type> type) {
+  auto method_kv = m_forward_declared_method_counts.find(name);
+  if (method_kv != m_forward_declared_method_counts.end()) {
+    int method_count = get_next_method_id(type.get());
+    if (method_count != method_kv->second) {
+      throw_typesystem_error(
+          "Type {} was defined with {} methods, but was forward declared with {}\n", name,
+          method_count, method_kv->second);
+    }
+  }
+
   auto kv = m_types.find(name);
   if (kv != m_types.end()) {
     // exists already
@@ -177,6 +187,53 @@ void TypeSystem::forward_declare_type_as(const std::string& new_type,
   }
 }
 
+void TypeSystem::forward_declare_type_method_count(const std::string& name, int num_methods) {
+  auto existing_fwd = m_forward_declared_method_counts.find(name);
+  if (existing_fwd != m_forward_declared_method_counts.end() &&
+      existing_fwd->second != num_methods) {
+    throw_typesystem_error(
+        "Type {} was originally forward declared with {} methods and is now being forward declared "
+        "with {} methods",
+        name, existing_fwd->second, num_methods);
+  }
+
+  auto existing_type = m_types.find(name);
+  if (existing_type != m_types.end()) {
+    int existing_count = get_next_method_id(existing_type->second.get());
+    if (existing_count != num_methods) {
+      throw_typesystem_error(
+          "Type {} was defined with {} methods and is now being forward declared with {} methods",
+          name, existing_fwd->second, num_methods);
+    }
+  }
+
+  m_forward_declared_method_counts[name] = num_methods;
+}
+
+int TypeSystem::get_type_method_count(const std::string& name) const {
+  auto result = try_get_type_method_count(name);
+  if (result) {
+    return *result;
+  }
+  throw_typesystem_error("Tried to find the number of methods on type {}, but it is not defined.",
+                         name);
+  return -1;
+}
+
+std::optional<int> TypeSystem::try_get_type_method_count(const std::string& name) const {
+  auto type_it = m_types.find(name);
+  if (type_it != m_types.end()) {
+    return get_next_method_id(type_it->second.get());
+  }
+
+  auto fwd_it = m_forward_declared_method_counts.find(name);
+  if (fwd_it != m_forward_declared_method_counts.end()) {
+    return fwd_it->second;
+  }
+
+  return {};
+}
+
 /*!
  * Get the runtime type (as a name string) of a TypeSpec.  Gets the runtime type of the primary
  * type of the TypeSpec.

common/type_system/TypeSystem.h

@@ -125,6 +125,9 @@ class TypeSystem {
   Type* add_type(const std::string& name, std::unique_ptr<Type> type);
   void forward_declare_type_as_type(const std::string& name);
   void forward_declare_type_as(const std::string& new_type, const std::string& parent_type);
+  void forward_declare_type_method_count(const std::string& name, int num_methods);
+  int get_type_method_count(const std::string& name) const;
+  std::optional<int> try_get_type_method_count(const std::string& name) const;
   std::string get_runtime_type(const TypeSpec& ts);
 
   DerefInfo get_deref_info(const TypeSpec& ts) const;
@@ -257,6 +260,8 @@ class TypeSystem {
 
   std::unordered_map<std::string, std::unique_ptr<Type>> m_types;
   std::unordered_map<std::string, std::string> m_forward_declared_types;
+  std::unordered_map<std::string, int> m_forward_declared_method_counts;
+
   std::vector<std::unique_ptr<Type>> m_old_types;
 
   bool m_allow_redefinition = false;

decompiler/ObjectFile/LinkedObjectFileCreation.cpp

@@ -760,10 +760,8 @@ static void link_v3(LinkedObjectFile& f,
         link_ptr--;
         s_name = (const char*)(&data.at(link_ptr));
       } else {
-        // methods todo
-
         s_name = (const char*)(&data.at(link_ptr));
-        // get_type_info().inform_type_method_count(s_name, reloc & 0x7f); todo
+        dts.ts.forward_declare_type_method_count(s_name, reloc & 0x7f);
         kind = SymbolLinkKind::TYPE;
       }
 

decompiler/config/all-types.gc

@@ -12736,7 +12736,7 @@
 (deftype entity-actor (entity)
   (
     (nav-mesh nav-mesh :offset-assert 52)
-    (etype basic :offset-assert 56) ;; probably type
+    (etype type :offset-assert 56) ;; probably type
     (task uint8 :offset-assert 60)
     (vis-id uint16 :offset-assert 62)
     (quat quaternion :inline :offset-assert 64)
@@ -12754,9 +12754,9 @@
   )
 
 (deftype entity-info (basic)
-  ((ptype     basic  :offset-assert 4)
+  ((ptype     type   :offset-assert 4)
    (package   basic  :offset-assert 8)
-   (art-group basic  :offset-assert 12)
+   (art-group pair   :offset-assert 12)
    (pool      basic  :offset-assert 16)
    (heap-size int32  :offset-assert 20)
    )
@@ -15988,7 +15988,7 @@
 
 ;; - Unknowns
 
-;;(define-extern *entity-info* object) ;; unknown type
+(define-extern *entity-info* (array entity-info))
 
 
 ;; ----------------------

decompiler/config/jak1_ntsc_black_label/label_types.jsonc

@@ -581,5 +581,9 @@
     ["L227", "uint64", true],
     ["L228", "uint64", true],
     ["L229", "uint64", true]
+  ],
+
+  "entity-table": [
+    ["L8", "(array entity-info)", true]
   ]
 }

decompiler/util/data_decompile.cpp

@@ -270,15 +270,12 @@ goos::Object decompile_structure(const TypeSpec& type,
 
   // check alignment
   auto offset_location = label.offset - type_info->get_offset();
-  //  if ((offset_location % 8) == 2) {
-  //    // TEMP HACK
-  //    lg::error("Data decompile was looking for a structure, but it looks like a pair instead.");
-  //    return decompile_pair(label, labels, words, ts);
-  //  }
   if (offset_location % 8) {
-    throw std::runtime_error(
-        fmt::format("Structure type {} (type offset {}) has alignment {}, which is not valid.",
-                    type_info->get_name(), type_info->get_offset(), (offset_location % 8)));
+    throw std::runtime_error(fmt::format(
+        "Tried to decompile a structure with type type {} (type offset {}) at label {}, but it has "
+        "alignment {}, which is not valid. {}",
+        type_info->get_name(), type_info->get_offset(), label.name, (offset_location % 8),
+        (offset_location & 0b10) ? "Maybe it is actually a pair?" : ""));
   }
 
   // check enough room
@@ -518,6 +515,17 @@ goos::Object decompile_structure(const TypeSpec& type,
           }
         } else if (word.kind == LinkedWord::EMPTY_PTR) {
           field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("'()"));
+        } else if (word.kind == LinkedWord::TYPE_PTR) {
+          if (field.type() != TypeSpec("type")) {
+            throw std::runtime_error(
+                fmt::format("Field {} in type {} offset {} had a reference to type {}, but the "
+                            "type of the field is not type.",
+                            field.name(), actual_type.print(), field.offset(), word.symbol_name));
+          }
+          int method_count = ts.get_type_method_count(word.symbol_name);
+          field_defs_out.emplace_back(
+              field.name(), pretty_print::to_symbol(fmt::format("(type-ref {} :method-count {})",
+                                                                word.symbol_name, method_count)));
         } else {
           throw std::runtime_error(
               fmt::format("Field {} in type {} offset {} did not have a proper reference",

docs/markdown/progress-notes/changelog.md

@@ -164,4 +164,5 @@
 - GOOS supports `string-ref`, `string-length`, `ash`, and characters can now be treated as a signed 8-bit number
 - Fixed a bug where saved xmm registers might be clobbered when calling a C++ function that wasn't `format`.
 - The `declare-type` form now supports any parent type. The type system will do a better job of trying to make things work out when only part of the type hierarchy is defined, and you can now chain type forward declarations. The compiler is stricter and will not accept forward declarations that are possibly incompatible. Instead, forward declare enough types and their parents for the compiler to be able to figure it out.  
-- The `deftype` form is more strict and will throw an error if the type definition is in any way incompatible with existing forward declarations of types.
+- The `deftype` form is more strict and will throw an error if the type definition is in any way incompatible with existing forward declarations of types.
+- Added a `type-ref` form to insert a reference to a type into a static structure and optionally forward declare the number of methods

goal_src/engine/entity/entity-h.gc

@@ -160,7 +160,7 @@
 (declare-type nav-mesh basic)
 (deftype entity-actor (entity)
   ((nav-mesh nav-mesh           :offset-assert 52)
-   (etype    basic              :offset-assert 56)
+   (etype    type               :offset-assert 56)
    (task     uint8              :offset-assert 60)
    (vis-id   uint16             :offset-assert 62)
    (quat     quaternion :inline :offset-assert 64)
@@ -178,9 +178,9 @@
 
 ;; definition of type entity-info
 (deftype entity-info (basic)
-  ((ptype     basic  :offset-assert 4)
+  ((ptype     type   :offset-assert 4)
    (package   basic  :offset-assert 8)
-   (art-group basic  :offset-assert 12)
+   (art-group pair   :offset-assert 12)
    (pool      basic  :offset-assert 16)
    (heap-size int32  :offset-assert 20)
    )

goalc/compiler/StaticObject.cpp

@@ -215,4 +215,13 @@ StaticResult StaticResult::make_symbol(const std::string& name) {
   result.m_symbol = name;
   result.m_ts = TypeSpec("symbol");
   return result;
+}
+
+StaticResult StaticResult::make_type_ref(const std::string& type_name, int method_count) {
+  StaticResult result;
+  result.m_kind = Kind::TYPE;
+  result.m_symbol = type_name;
+  result.m_method_count = method_count;
+  result.m_ts = TypeSpec("type");
+  return result;
 }

goalc/compiler/StaticObject.h

@@ -101,13 +101,13 @@ class StaticResult {
   static StaticResult make_constant_data(const ConstantValue& data, TypeSpec ts);
   static StaticResult make_constant_data(u64 data, const TypeSpec& ts);
   static StaticResult make_symbol(const std::string& name);
-
-  std::string print() const;
+  static StaticResult make_type_ref(const std::string& type_name, int method_count);
 
   const TypeSpec& typespec() const { return m_ts; }
   bool is_reference() const { return m_kind == Kind::STRUCTURE_REFERENCE; }
   bool is_constant_data() const { return m_kind == Kind::CONSTANT_DATA; }
   bool is_symbol() const { return m_kind == Kind::SYMBOL; }
+  bool is_type() const { return m_kind == Kind::TYPE; }
 
   StaticStructure* reference() const {
     assert(is_reference());
@@ -121,10 +121,15 @@ class StaticResult {
   }
 
   const std::string& symbol_name() const {
-    assert(is_symbol());
+    assert(is_symbol() || is_type());
     return m_symbol;
   }
 
+  int method_count() const {
+    assert(is_type());
+    return m_method_count;
+  }
+
   u64 constant_u64() const {
     assert(is_constant_data() && m_constant_data && m_constant_data->size() == 8);
     return m_constant_data->value_64();
@@ -145,10 +150,19 @@ class StaticResult {
   // used for only constant data
   std::optional<ConstantValue> m_constant_data;
 
-  // used for only symbol
+  // used for only symbol and type
   std::string m_symbol;
 
-  enum class Kind { STRUCTURE_REFERENCE, CONSTANT_DATA, SYMBOL, INVALID } m_kind = Kind::INVALID;
+  // used only for type
+  int m_method_count = -1;
+
+  enum class Kind {
+    STRUCTURE_REFERENCE,
+    CONSTANT_DATA,
+    SYMBOL,
+    TYPE,
+    INVALID
+  } m_kind = Kind::INVALID;
 };
 
 class StaticPair : public StaticStructure {

goalc/compiler/compilation/Static.cpp

@@ -205,6 +205,12 @@ void Compiler::compile_static_structure_inline(const goos::Object& form,
           typecheck(form, field_info.type, sr.typespec());
           structure->add_pointer_record(field_offset, sr.reference(),
                                         sr.reference()->get_addr_offset());
+        } else if (sr.is_type()) {
+          if (field_info.type != TypeSpec("type")) {
+            throw_compiler_error(form, "Cannot put a type reference in a field with type {}",
+                                 field_info.type.print());
+          }
+          structure->add_type_record(sr.symbol_name(), field_offset);
         } else {
           throw_compiler_error(form, "Unsupported field value {}.", field_value.print());
         }
@@ -669,6 +675,37 @@ StaticResult Compiler::compile_static(const goos::Object& form_before_macro, Env
           }
         }
       }
+    } else if (first.is_symbol("type-ref")) {
+      auto args = get_va(form, rest);
+      va_check(form, args, {goos::ObjectType::SYMBOL},
+               {{{"method-count", {false, goos::ObjectType::INTEGER}}}});
+
+      auto type_name = args.unnamed.at(0).as_symbol()->name;
+
+      std::optional<int> expected_method_count = m_ts.try_get_type_method_count(type_name);
+      int method_count = -1;
+
+      if (args.has_named("method-count")) {
+        method_count = args.get_named("method-count").as_int();
+        if (expected_method_count && (method_count != *expected_method_count)) {
+          throw_compiler_error(
+              form, "type-ref wanted {} methods for type {}, but the type system thinks it has {}",
+              method_count, type_name, *expected_method_count);
+        }
+      } else {
+        if (!expected_method_count) {
+          throw_compiler_error(
+              form,
+              "Cannot create a static type reference for type {}. The type-ref form did not have a "
+              ":method-count argument and the type system does not know how many methods it has.",
+              type_name);
+        }
+        method_count = *expected_method_count;
+      }
+
+      m_ts.forward_declare_type_method_count(type_name, method_count);
+
+      return StaticResult::make_type_ref(type_name, method_count);
     } else {
       // maybe an enum
       s64 int_out;

goalc/compiler/compilation/Type.cpp

@@ -124,7 +124,7 @@ void Compiler::generate_field_description(const goos::Object& form,
                                           const Field& f) {
   std::string str_template;
   std::vector<RegVal*> format_args = {};
-  if (m_ts.tc(m_ts.make_typespec("type"), f.type())) {
+  if (f.name() == "type" && f.offset() == 0) {
     // type
     return;
   } else if (f.is_array() && !f.is_dynamic()) {

goalc/emitter/ObjectGenerator.cpp

@@ -458,7 +458,7 @@ void ObjectGenerator::emit_link_type_pointer(int seg, const TypeSystem* ts) {
     out.push_back(0);
 
     // method count
-    out.push_back(ts->get_next_method_id(ts->lookup_type(rec.first)));
+    out.push_back(ts->get_type_method_count(rec.first));
 
     // number of links
     push_data<u32>(size, out);

test/decompiler/reference/engine/entity/entity-h_REF.gc

@@ -234,7 +234,7 @@
 ;; definition of type entity-actor
 (deftype entity-actor (entity)
   ((nav-mesh nav-mesh           :offset-assert  52)
-   (etype    basic              :offset-assert  56)
+   (etype    type               :offset-assert  56)
    (task     uint8              :offset-assert  60)
    (vis-id   uint16             :offset-assert  62)
    (quat     quaternion :inline :offset-assert  64)
@@ -252,9 +252,9 @@
 
 ;; definition of type entity-info
 (deftype entity-info (basic)
-  ((ptype     basic  :offset-assert   4)
+  ((ptype     type   :offset-assert   4)
    (package   basic  :offset-assert   8)
-   (art-group basic  :offset-assert  12)
+   (art-group pair   :offset-assert  12)
    (pool      basic  :offset-assert  16)
    (heap-size int32  :offset-assert  20)
    )

test/goalc/source_templates/with_game/test-type-ref.gc

@@ -0,0 +1,24 @@
+
+;; define a type with fields that hold a type.
+(deftype test-type-with-type-field (basic)
+  ((name string)
+   (some-type-1 type)
+   (some-type-2 type))
+  )
+
+(let ((obj (new 'static 'test-type-with-type-field
+                :name "test"
+                :some-type-2 (type-ref some-unknown-type :method-count 20)
+                :some-type-1 (type-ref string :method-count 9)))
+      (old-string string))
+  (format #t "~A ~A ~A ~A ~D ~D~%"
+          (-> obj some-type-1 symbol)
+          (= (-> obj some-type-1) old-string) ;; should not reallocate string
+          (-> obj some-type-1 parent)
+          (-> obj some-type-2 symbol) ;; should fill out type symbol
+          (-> obj some-type-2 allocated-length) ;; and method length
+          (-> obj some-type-2 parent) ;; but other stuff should be 0's
+          )
+  )
+
+

test/goalc/test_with_game.cpp

@@ -621,6 +621,12 @@ TEST_F(WithGameTests, StaticArrayField) {
                           "0\n"});
 }
 
+TEST_F(WithGameTests, TypeReference) {
+  runner.run_static_test(env, testCategory, "test-type-ref.gc",
+                         {"string #t basic some-unknown-type 20 0\n"
+                          "0\n"});
+}
+
 TEST_F(WithGameTests, StaticFieldInlineArray) {
   runner.run_static_test(env, testCategory, "test-static-field-inline-arrays.gc",
                          {"\"second\" \"first\"\n"