deftype and defmethod syntax major changes (#3094)

Major change to how `deftype` shows up in our code:
- the decompiler will no longer emit the `offset-assert`,
`method-count-assert`, `size-assert` and `flag-assert` parameters. There
are extremely few cases where having this in the decompiled code is
helpful, as the types there come from `all-types` which already has
those parameters. This also doesn't break type consistency because:
  - the asserts aren't compared.
- the first step of the test uses `all-types`, which has the asserts,
which will throw an error if they're bad.
- the decompiler won't emit the `heap-base` parameter unless necessary
now.
- the decompiler will try its hardest to turn a fixed-offset field into
an `overlay-at` field. It falls back to the old offset if all else
fails.
- `overlay-at` now supports field "dereferencing" to specify the offset
that's within a field that's a structure, e.g.:
```lisp
(deftype foobar (structure)
  ((vec    vector  :inline)
   (flags  int32   :overlay-at (-> vec w))
   )
  )
```
in this structure, the offset of `flags` will be 12 because that is the
final offset of `vec`'s `w` field within this structure.
- **removed ID from all method declarations.** IDs are only ever
automatically assigned now. Fixes #3068.
- added an `:overlay` parameter to method declarations, in order to
declare a new method that goes on top of a previously-defined method.
Syntax is `:overlay <method-name>`. Please do not ever use this.
- added `state-methods` list parameter. This lets you quickly specify a
list of states to be put in the method table. Same syntax as the
`states` list parameter. The decompiler will try to put as many states
in this as it can without messing with the method ID order.

Also changes `defmethod` to make the first type definition (before the
arguments) optional. The type can now be inferred from the first
argument. Fixes #3093.

---------

Co-authored-by: Hat Kid <6624576+Hat-Kid@users.noreply.github.com>

common/goos/PrettyPrinter2.cpp

@@ -288,7 +288,11 @@ void break_list(Node* node) {
       node->sub_elt_indent += name.size();
     } else if (name == "defmethod") {
       // things with 4 things in the top line: (defmethod <method> <type> <args>
-      node->top_line_count = 4;
+      // or just 3 things in the top line: (defmethod <method> <args>
+      node->top_line_count = 3;
+      if (node->child_nodes.size() >= 4 && node->child_nodes[2].kind == Node::Kind::ATOM) {
+        node->top_line_count = 4;
+      }
     } else if (name == "until" || name == "while" || name == "dotimes" || name == "countdown" ||
                name == "when" || name == "behavior" || name == "lambda" || name == "defpart" ||
                name == "define") {

common/type_system/Type.cpp

@@ -421,10 +421,12 @@ int Type::get_num_methods() const {
  * Add a method defined specifically for this type.
  */
 const MethodInfo& Type::add_method(const MethodInfo& info) {
-  for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
-    if (!it->overrides_parent && !it->only_overrides_docstring) {
-      ASSERT(it->id + 1 == info.id);
-      break;
+  if (!info.overrides_parent) {
+    for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
+      if (!it->overrides_parent && !it->only_overrides_docstring) {
+        ASSERT(it->id + 1 == info.id);
+        break;
+      }
     }
   }
 

common/type_system/Type.h

@@ -32,6 +32,7 @@ struct MethodInfo {
   bool overrides_parent = false;
   bool only_overrides_docstring = false;
   std::optional<std::string> docstring;
+  std::optional<std::string> overlay_name;
 
   bool operator==(const MethodInfo& other) const;
   bool operator!=(const MethodInfo& other) const { return !((*this) == other); }

common/type_system/TypeSystem.cpp

@@ -516,14 +516,13 @@ int TypeSystem::get_load_size_allow_partial_def(const TypeSpec& ts) const {
 }
 
 MethodInfo TypeSystem::override_method(Type* type,
-                                       const std::string& /*type_name*/,
-                                       const int method_id,
+                                       const std::string& method_name,
                                        const std::optional<std::string>& docstring) {
   // Lookup the method from the parent type
   MethodInfo existing_info;
-  bool exists = try_lookup_method(type->get_parent(), method_id, &existing_info);
+  bool exists = try_lookup_method(type->get_parent(), method_name, &existing_info);
   if (!exists) {
-    throw_typesystem_error("Trying to use override a method that has no parent declaration");
+    throw_typesystem_error("Trying to override a method that has no parent declaration");
   }
   // use the existing ID.
   return type->add_method({existing_info.id, existing_info.name, existing_info.type,
@@ -554,8 +553,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
                                       const std::optional<std::string>& docstring,
                                       bool no_virtual,
                                       const TypeSpec& ts,
-                                      bool override_type,
-                                      int id) {
+                                      bool override_type) {
   if (method_name == "new") {
     if (override_type) {
       throw_typesystem_error("Cannot use :replace option with a new method.");
@@ -569,7 +567,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
 
   if (override_type) {
     if (!got_existing) {
-      if (id != -1 && try_lookup_method(type->get_parent(), id, &existing_info)) {
+      if (try_lookup_method(type->get_parent(), method_name, &existing_info)) {
       } else {
         throw_typesystem_error(
             "Cannot use :replace on method {} of {} because this method was not previously "
@@ -616,6 +614,34 @@ MethodInfo TypeSystem::declare_method(Type* type,
   }
 }
 
+/*!
+ * Adds a new method that is overlayed on top of a different, existing method.
+ * This should be used basically never (happens once in Jak 1).
+ */
+MethodInfo TypeSystem::overlay_method(Type* type,
+                                      const std::string& method_name,
+                                      const std::string& method_overlay_name,
+                                      const std::optional<std::string>& docstring,
+                                      const TypeSpec& ts) {
+  // look up the method
+  MethodInfo existing_info;
+  bool got_existing = try_lookup_method(type, method_overlay_name, &existing_info);
+
+  if (!got_existing) {
+    if (try_lookup_method(type->get_parent(), method_overlay_name, &existing_info)) {
+    } else {
+      throw_typesystem_error(
+          "Cannot use :overlay-at on method {} of {} because this method was not previously "
+          "declared in a parent.",
+          method_overlay_name, type->get_name());
+    }
+  }
+
+  // use the existing ID.
+  return type->add_method({existing_info.id, method_name, ts, type->get_name(), false, true, false,
+                           docstring, std::make_optional(method_overlay_name)});
+}
+
 MethodInfo TypeSystem::define_method(const std::string& type_name,
                                      const std::string& method_name,
                                      const TypeSpec& ts,
@@ -1885,30 +1911,36 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
     }
   }
 
-  if (type->heap_base()) {
+  if (type->heap_base() &&
+      type->heap_base() !=
+          ((type->get_size_in_memory() - get_type_of_type<BasicType>("process")->size() + 0xf) &
+           ~0xf)) {
+    // don't print if auto heap-base does the job
     result.append(fmt::format("  :heap-base #x{:x}\n", type->heap_base()));
   }
 
   auto method_count = get_next_method_id(type);
-  result.append(fmt::format("  :method-count-assert {}\n", get_next_method_id(type)));
-  result.append(fmt::format("  :size-assert         #x{:x}\n", type->get_size_in_memory()));
+  // result.append(fmt::format("  :method-count-assert {}\n", get_next_method_id(type)));
+  // result.append(fmt::format("  :size-assert         #x{:x}\n", type->get_size_in_memory()));
   TypeFlags flags;
   flags.heap_base = type->heap_base();
   flags.size = type->get_size_in_memory();
   flags.pad = 0;
   flags.methods = method_count;
 
-  result.append(fmt::format("  :flag-assert         #x{:x}\n  ", flags.flag));
+  // result.append(fmt::format("  :flag-assert         #x{:x}\n", flags.flag));
   if (!type->gen_inspect()) {
-    result.append(":no-inspect\n  ");
+    result.append("  :no-inspect\n  ");
   }
 
   std::string methods_string;
+  std::string state_methods_string;
+  std::string states_string;
 
   // New Method
   auto new_info = type->get_new_method_defined_for_type();
   if (new_info) {
-    methods_string.append("(new (");
+    methods_string.append("    (new (");
     for (size_t i = 0; i < new_info->type.arg_count() - 1; i++) {
       methods_string.append(new_info->type.get_arg(i).print());
       if (i != new_info->type.arg_count() - 2) {
@@ -1916,24 +1948,41 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
       }
     }
     methods_string.append(
-        fmt::format(") {} ", new_info->type.get_arg(new_info->type.arg_count() - 1).print(), 0));
+        fmt::format(") {}", new_info->type.get_arg(new_info->type.arg_count() - 1).print(), 0));
 
     auto behavior = new_info->type.try_get_tag("behavior");
     if (behavior) {
-      methods_string.append(fmt::format(":behavior {} ", *behavior));
+      methods_string.append(fmt::format(" :behavior {}", *behavior));
     }
 
-    methods_string.append("0)\n    ");
+    methods_string.append(")\n");
   }
 
   // Rest of methods
+  bool done_with_state_methods = false;  // TODO fix this... this depends on the order of m_methods
   for (auto& info : type->get_methods_defined_for_type()) {
+    if (!done_with_state_methods && info.type.base_type() == "state" && !info.overrides_parent) {
+      if (info.type.arg_count() > 1) {
+        state_methods_string.append(fmt::format("    ({}", info.name));
+        for (size_t i = 0; i < info.type.arg_count() - 1; ++i) {
+          state_methods_string.push_back(' ');
+          state_methods_string.append(info.type.get_arg(i).print());
+        }
+        state_methods_string.append(")\n");
+      } else {
+        state_methods_string.append(fmt::format("    {}\n", info.name));
+      }
+      continue;
+    } else {
+      done_with_state_methods = true;
+    }
+
     // check if we only override the docstring
     if (info.only_overrides_docstring) {
       continue;
     }
 
-    methods_string.append(fmt::format("({} (", info.name));
+    methods_string.append(fmt::format("    ({} (", info.name));
     for (size_t i = 0; i < info.type.arg_count() - 1; i++) {
       methods_string.append(info.type.get_arg(i).print());
       if (i != info.type.arg_count() - 2) {
@@ -1941,35 +1990,32 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
       }
     }
     methods_string.append(
-        fmt::format(") {} ", info.type.get_arg(info.type.arg_count() - 1).print()));
-
-    if (info.no_virtual) {
-      methods_string.append(":no-virtual ");
-    }
-
-    if (info.overrides_parent) {
-      methods_string.append(":replace ");
-    }
+        fmt::format(") {}", info.type.get_arg(info.type.arg_count() - 1).print()));
 
     auto behavior = info.type.try_get_tag("behavior");
     if (behavior) {
-      methods_string.append(fmt::format(":behavior {} ", *behavior));
+      methods_string.append(fmt::format(" :behavior {}", *behavior));
     }
 
     if (info.type.base_type() == "state") {
-      methods_string.append(":state ");
+      methods_string.append(" :state");
     }
 
-    methods_string.append(fmt::format("{})\n    ", info.id));
+    if (info.no_virtual) {
+      methods_string.append(" :no-virtual");
+    }
+
+    if (info.overrides_parent) {
+      if (info.overlay_name.has_value()) {
+        methods_string.append(fmt::format(" :overlay-at {}", *info.overlay_name));
+      } else {
+        methods_string.append(" :replace");
+      }
+    }
+
+    methods_string.append(fmt::format(")\n", info.id));
   }
 
-  if (!methods_string.empty()) {
-    result.append("(:methods\n    ");
-    result.append(methods_string);
-    result.append(")\n  ");
-  }
-
-  std::string states_string;
   for (auto& info : type->get_states_declared_for_type()) {
     if (info.second.arg_count() > 1) {
       states_string.append(fmt::format("    ({}", info.first));
@@ -1983,16 +2029,157 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
     }
   }
 
-  if (!states_string.empty()) {
-    result.append("(:states\n");
-    result.append(states_string);
-    result.append("    )\n  ");
+  if (!state_methods_string.empty()) {
+    result.append("  (:state-methods\n");
+    result.append(state_methods_string);
+    result.append("    )\n");
   }
 
-  result.append(")\n");
+  if (!methods_string.empty()) {
+    result.append("  (:methods\n");
+    result.append(methods_string);
+    result.append("    )\n");
+  }
+
+  if (!states_string.empty()) {
+    result.append("  (:states\n");
+    result.append(states_string);
+    result.append("    )\n");
+  }
+
+  result.append("  )\n");
   return result;
 }
 
+std::optional<std::string> find_best_field_in_structure(const TypeSystem& ts,
+                                                        const StructureType* st,
+                                                        int offset,
+                                                        const Field& requesting_field,
+                                                        bool want_fixed,
+                                                        int start_field,
+                                                        int end_field = -1) {
+  // performs best field lookup within a structure, at an offset.
+  const Field* best_val = nullptr;
+  const Field* best_exact = nullptr;
+  const Field* best_struct = nullptr;
+  std::pair<const Field*, int> best_val_arr = {nullptr, -1};
+  std::pair<const Field*, int> best_exact_arr = {nullptr, -1};
+  std::pair<const Field*, int> best_struct_arr = {nullptr, -1};
+  std::optional<std::string> best_struct_field_deref;
+  const Field* best = nullptr;
+  if (end_field == -1) {
+    end_field = st->fields().size();
+  }
+  for (size_t i = start_field; i < end_field; ++i) {
+    const auto& field = st->fields().at(i);
+    auto type = ts.lookup_type(field.type());
+    if (field.is_dynamic() || field.offset() > offset || field.user_placed() != want_fixed) {
+      continue;
+    }
+    if (!field.is_array()) {
+      if (!field.is_inline() && field.offset() + type->get_load_size() > offset) {
+        if (field.offset() == offset) {
+          // not array, not inline - can fit in register, only check exact offset.
+          if (!best_val ||
+              type->get_load_size() == ts.lookup_type(requesting_field.type())->get_load_size()) {
+            best_val = &field;
+          }
+        }
+      } else if (field.is_inline() && field.offset() + type->get_size_in_memory() > offset) {
+        if (field.type() == requesting_field.type() && field.offset() == offset) {
+          // not array, inlined and exact same as this field, just overlay directly on top
+          best_exact = &field;
+        } else {
+          auto f_type = dynamic_cast<StructureType*>(type);
+          if (f_type) {
+            // struct that encompasses this field
+            // simply search that structure for the field we want, offset by the field's offset
+            auto best_field_in_struct = find_best_field_in_structure(
+                ts, f_type, offset - field.offset(), requesting_field, want_fixed, 0);
+            if (best_field_in_struct) {
+              best_struct_field_deref = best_field_in_struct;
+              best_struct = &field;
+            }
+          }
+        }
+      }
+    } else {
+      int rel_offset = offset - field.offset();
+      // array case (and array encompasses what we want)
+      int array_idx = rel_offset / type->get_size_in_memory();
+      if (!field.is_inline() &&
+          field.offset() + field.array_size() * type->get_load_size() > offset) {
+        if (rel_offset % type->get_load_size() == 0) {
+          // found exact match for array index
+          if (!best_val_arr.first ||
+              type->get_load_size() == ts.lookup_type(requesting_field.type())->get_load_size()) {
+            best_val_arr.first = &field;
+            best_val_arr.second = rel_offset / type->get_load_size();
+          }
+        }
+      } else if (field.is_inline() &&
+                 field.offset() + field.array_size() * type->get_size_in_memory() > offset) {
+        if (field.type() == requesting_field.type() &&
+            rel_offset % type->get_size_in_memory() == 0) {
+          // same type
+          best_exact_arr.first = &field;
+          best_exact_arr.second = array_idx;
+        } else if (requesting_field.is_array() && rel_offset % type->get_size_in_memory() == 0 &&
+                   array_idx == 0) {
+          // starts at the same offset as another array. just use the field with nothing extra
+          best_exact = &field;
+        } else {
+          auto f_type = dynamic_cast<StructureType*>(type);
+          if (f_type && field.offset() + f_type->get_size_in_memory() > offset) {
+            // struct that encompasses this field
+            // simply search that structure for the field we want, offset by the field's offset
+            auto best_field_in_struct =
+                find_best_field_in_structure(ts, f_type, rel_offset % type->get_size_in_memory(),
+                                             requesting_field, want_fixed, 0);
+            if (best_field_in_struct) {
+              best_struct_field_deref = best_field_in_struct;
+              best_struct_arr.first = &field;
+              best_struct_arr.second = array_idx;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  int best_array_idx = -1;
+  if (best_exact) {
+    best = best_exact;
+  } else if (best_exact_arr.first) {
+    best = best_exact_arr.first;
+    best_array_idx = best_exact_arr.second;
+  } else if (best_val) {
+    best = best_val;
+  } else if (best_val_arr.first) {
+    best = best_val_arr.first;
+    best_array_idx = best_val_arr.second;
+  } else if (best_struct) {
+    best = best_struct;
+  } else if (best_struct_arr.first) {
+    best = best_struct_arr.first;
+    best_array_idx = best_struct_arr.second;
+  }
+  if (best) {
+    auto ret =
+        best_array_idx == -1 ? best->name() : fmt::format("{} {}", best->name(), best_array_idx);
+    if (best == best_struct || best == best_struct_arr.first) {
+      return ret + " " + *best_struct_field_deref;
+    } else {
+      return ret;
+    }
+  } else if (!want_fixed) {
+    // try again but with a user-placed offset
+    return find_best_field_in_structure(ts, st, offset, requesting_field, true, start_field,
+                                        end_field);
+  }
+  return {};
+}
+
 std::string TypeSystem::generate_deftype_for_structure(const StructureType* st) const {
   std::string result;
   result += fmt::format("(deftype {} ({})\n", st->get_name(), st->get_parent());
@@ -2004,10 +2191,10 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
   int longest_field_name = 0;
   int longest_type_name = 0;
   int longest_mods = 0;
+  int longest_mods_with_user_placed = 0;
 
   const std::string inline_string = ":inline";
   const std::string dynamic_string = ":dynamic";
-  bool has_offset_assert = false;
 
   // calculate longest strings needed, for basic linting
 
@@ -2021,33 +2208,33 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
   // normal fields
   for (size_t i = st->first_unique_field_idx(); i < st->fields().size(); i++) {
     const auto& field = st->fields().at(i);
-    longest_field_name = std::max(longest_field_name, int(field.name().size()));
-    longest_type_name = std::max(longest_type_name, int(field.type().print().size()));
 
     int mods = 0;
     // mods are array size, :inline, :dynamic
     if (field.is_array() && !field.is_dynamic()) {
+      mods++;
       mods += std::to_string(field.array_size()).size();
     }
 
     if (field.is_inline()) {
-      if (mods) {
-        mods++;  // space
-      }
+      mods++;  // space
       mods += inline_string.size();
     }
 
     if (field.is_dynamic()) {
-      if (mods) {
-        mods++;  // space
-      }
+      mods++;  // space
       mods += dynamic_string.size();
     }
 
-    if (!field.user_placed()) {
-      has_offset_assert = true;
+    longest_field_name = std::max(longest_field_name, int(field.name().size()));
+    if (mods > 0 || field.user_placed()) {
+      // this is only relevant for fields that have mods
+      longest_type_name = std::max(longest_type_name, int(field.type().print().size()));
     }
     longest_mods = std::max(longest_mods, mods);
+    if (field.user_placed()) {
+      longest_mods_with_user_placed = std::max(longest_mods_with_user_placed, mods);
+    }
   }
 
   // now actually write out the fields
@@ -2057,10 +2244,9 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
     const auto& field = st->fields().at(i);
     result += "(";
     result += field.name();
-    result.append(1 + (longest_field_name - int(field.name().size())), ' ');
+    result.append(2 + (longest_field_name - int(field.name().size())), ' ');
     result += field.type().print();
     result.append(1 + (longest_type_name - int(field.type().print().size())), ' ');
-    result.append(1 + longest_mods, ' ');
     result.append(":override)\n   ");
   }
 
@@ -2069,40 +2255,51 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
     const auto& field = st->fields().at(i);
     result += "(";
     result += field.name();
-    result.append(1 + (longest_field_name - int(field.name().size())), ' ');
+    result.append(2 + (longest_field_name - int(field.name().size())), ' ');
     result += field.type().print();
-    result.append(1 + (longest_type_name - int(field.type().print().size())), ' ');
 
     std::string mods;
     if (field.is_array() && !field.is_dynamic()) {
-      mods += std::to_string(field.array_size());
       mods += " ";
+      mods += std::to_string(field.array_size());
     }
 
     if (field.is_inline()) {
-      mods += inline_string;
       mods += " ";
+      mods += inline_string;
     }
 
     if (field.is_dynamic()) {
-      mods += dynamic_string;
       mods += " ";
+      mods += dynamic_string;
     }
 
+    if (!mods.empty()) {
+      result.append(1 + longest_type_name - int(field.type().print().size()), ' ');
+    }
     result.append(mods);
-    result.append(longest_mods - int(mods.size() - 1), ' ');
 
-    if (!field.user_placed()) {
-      result.append(":offset-assert ");
-    } else {
-      if (has_offset_assert) {
-        result.append(":offset        ");
+    if (field.user_placed()) {
+      result.append(longest_mods_with_user_placed - int(mods.size()), ' ');
+      if (mods.empty()) {
+        result.append(1 + longest_type_name - int(field.type().print().size()), ' ');
+      }
+      // find best field for :overlay-at
+      // we find the first field that does not come after the current one
+      // and either use it, or check if one of its fields (recursively) is appropriate
+      // we also check for array offsets. we ALSO do bounds-checking!
+      // non-fixed offset fields get priority! dynamic fields are IGNORED.
+      // if all else fails, print as fixed offset.
+      auto best_match = find_best_field_in_structure(*this, st, field.offset(), field, false, 0, i);
+      if (!best_match) {
+        result.append(fmt::format(" :offset {:3d}", field.offset()));
+      } else if (best_match->find(' ') == std::string::npos) {
+        result.append(fmt::format(" :overlay-at {}", *best_match));
       } else {
-        result.append(":offset ");
+        result.append(fmt::format(" :overlay-at (-> {})", *best_match));
       }
     }
 
-    result.append(fmt::format("{:3d}", field.offset()));
     result.append(")\n   ");
   }
 

common/type_system/TypeSystem.h

@@ -161,8 +161,7 @@ class TypeSystem {
   int get_load_size_allow_partial_def(const TypeSpec& ts) const;
 
   MethodInfo override_method(Type* type,
-                             const std::string& type_name,
-                             const int method_id,
+                             const std::string& method_name,
                              const std::optional<std::string>& docstring);
   MethodInfo declare_method(const std::string& type_name,
                             const std::string& method_name,
@@ -175,8 +174,12 @@ class TypeSystem {
                             const std::optional<std::string>& docstring,
                             bool no_virtual,
                             const TypeSpec& ts,
-                            bool override_type,
-                            int id = -1);
+                            bool override_type);
+  MethodInfo overlay_method(Type* type,
+                            const std::string& method_name,
+                            const std::string& method_overlay_name,
+                            const std::optional<std::string>& docstring,
+                            const TypeSpec& ts);
   MethodInfo define_method(const std::string& type_name,
                            const std::string& method_name,
                            const TypeSpec& ts,

common/type_system/deftype.cpp

@@ -136,14 +136,65 @@ void add_field(
           throw std::runtime_error(fmt::format("Field {} not found to override", name));
         }
       } else if (opt_name == ":overlay-at") {
-        auto field_name = symbol_string(car(rest));
-        Field overlay_field;
-        if (!structure->lookup_field(field_name, &overlay_field)) {
-          throw std::runtime_error(
-              fmt::format("Field {} not found to overlay for {}", field_name, name));
-        }
-        offset_override = overlay_field.offset();
+        const auto& param = car(rest);
         rest = cdr(rest);
+        Field overlay_field;
+        if (param.is_symbol()) {
+          auto field_name = symbol_string(param);
+          if (!structure->lookup_field(field_name, &overlay_field)) {
+            throw std::runtime_error(
+                fmt::format("Field {} not found to overlay for {}", field_name, name));
+          }
+          offset_override = overlay_field.offset();
+        } else if (param.is_pair() && car(&param).is_symbol("->")) {
+          auto name_it = cdr(&param);
+          if (name_it->is_empty_list()) {
+            throw std::runtime_error(
+                fmt::format("Field list for overlay-at in {} was empty", name));
+          }
+          auto type_to_use = structure;
+          offset_override = 0;
+          while (!name_it->is_empty_list()) {
+            const auto& deref_field = car(name_it);
+            if (deref_field.is_int()) {
+              auto ref_array_field = !type_to_use && !overlay_field.is_inline()
+                                         ? ts->lookup_type_allow_partial_def(overlay_field.type())
+                                         : nullptr;
+              if (ref_array_field) {
+                // we can have an array of references (non-inline) to a forward-declared type
+                offset_override += ref_array_field->get_load_size() * deref_field.as_int();
+              } else {
+                auto type_to_deref = type_to_use && overlay_field.is_inline()
+                                         ? TypeSpec("inline-array")
+                                         : TypeSpec("pointer");
+                type_to_deref.add_arg(overlay_field.type());
+                auto deref_info = ts->get_deref_info(type_to_deref);
+                if (!deref_info.can_deref) {
+                  throw std::runtime_error(
+                      fmt::format("Array could not be dereferenced for overlay-at in {}", name));
+                }
+                // overlay_field.type() = deref_info.result_type;
+                offset_override += deref_info.stride * deref_field.as_int();
+              }
+            } else {
+              if (!type_to_use) {
+                throw std::runtime_error(
+                    fmt::format("Field {} not inside a structure for overlay-at in {}",
+                                overlay_field.name(), name));
+              }
+              auto field_name = symbol_string(car(name_it));
+              if (!type_to_use->lookup_field(field_name, &overlay_field)) {
+                throw std::runtime_error(
+                    fmt::format("Field {} not found to overlay for {}", field_name, name));
+              }
+              type_to_use = dynamic_cast<StructureType*>(ts->lookup_type(overlay_field.type()));
+              offset_override += overlay_field.offset();
+            }
+            name_it = cdr(name_it);
+          }
+        } else {
+          throw std::runtime_error(fmt::format("Unknown parameter for overlay-at in {}", name));
+        }
       } else if (opt_name == ":score") {
         score = get_float(car(rest));
         rest = cdr(rest);
@@ -276,14 +327,20 @@ void declare_method(Type* type,
     // - this effectively does a :replace without having to re-define the name and signature and
     // keep that in-sync
     std::string method_name;
+    std::string method_overlay_name;
     TypeSpec function_typespec("function");
     std::optional<std::string> docstring;
     goos::Object args;
     goos::Object return_type;
     bool no_virtual = false;
     bool replace_method = false;
+    bool overlay_method = false;
     bool overriding_doc = false;
 
+    // name
+    method_name = symbol_string(car(obj));
+    obj = cdr(obj);
+
     if (!obj->is_empty_list() && car(obj).is_symbol(":override-doc")) {
       obj = cdr(obj);
       if (car(obj).is_string()) {
@@ -296,10 +353,6 @@ void declare_method(Type* type,
     }
 
     if (!overriding_doc) {
-      // name
-      method_name = symbol_string(car(obj));
-      obj = cdr(obj);
-
       // docstring
       if (obj->is_pair() && car(obj).is_string()) {
         docstring = str_util::trim_newline_indents(car(obj).as_string()->data);
@@ -325,13 +378,9 @@ void declare_method(Type* type,
         } else if (keyword == ":replace") {
           replace_method = true;
         } else if (keyword == ":state") {
-          auto behavior_tag = function_typespec.try_get_tag("behavior");
           function_typespec = TypeSpec("state");
-          if (behavior_tag) {
-            function_typespec.add_new_tag("behavior", behavior_tag.value());
-          }
           // parse state docstrings if available
-          if (car(cdr(obj)).is_list()) {
+          if (!cdr(obj)->is_empty_list() && car(cdr(obj)).is_list()) {
             obj = cdr(obj);
             auto docstring_list = &car(obj);
             auto elem = docstring_list;
@@ -362,6 +411,13 @@ void declare_method(Type* type,
             throw std::runtime_error("Bad usage of :behavior in a method declaration");
           }
           function_typespec.add_new_tag("behavior", symbol_string(obj->as_pair()->car));
+        } else if (keyword == ":overlay-at") {
+          obj = cdr(obj);
+          if (!car(obj).is_symbol()) {
+            throw std::runtime_error("Invalid parameter to method overlay-at");
+          }
+          method_overlay_name = symbol_string(car(obj));
+          overlay_method = true;
         }
         obj = cdr(obj);
       }
@@ -373,36 +429,62 @@ void declare_method(Type* type,
       function_typespec.add_arg(parse_typespec(type_system, return_type));
     }
 
-    // determine the method id, it should be the last in the list
-    int id = -1;
-    if (!obj->is_empty_list() && car(obj).is_int()) {
-      auto& id_obj = car(obj);
-      id = get_int(id_obj);
-      obj = cdr(obj);
+    if (!obj->is_empty_list()) {
+      throw std::runtime_error(fmt::format("found unknown data in a method declaration:\n{}\n\n{}",
+                                           obj->print(), _obj.print()));
     }
 
-    if (!obj->is_empty_list()) {
-      throw std::runtime_error("found symbols after the `id` in a method defintion: " +
-                               def.print());
+    if (overlay_method && (no_virtual || replace_method)) {
+      throw std::runtime_error(
+          fmt::format("method {} in type {} has invalid combination of keywords", method_name,
+                      type->get_name()));
     }
 
     MethodInfo info;
     if (overriding_doc) {
-      info = type_system->override_method(type, method_name, id, docstring);
+      info = type_system->override_method(type, method_name, docstring);
+    } else if (overlay_method) {
+      info = type_system->overlay_method(type, method_name, method_overlay_name, docstring,
+                                         function_typespec);
     } else {
       info = type_system->declare_method(type, method_name, docstring, no_virtual,
-                                         function_typespec, replace_method, id);
+                                         function_typespec, replace_method);
     }
+  });
+}
 
-    // check the method assert
-    if (id != -1) {
-      // method id assert!
-      if (id != info.id) {
-        lg::print("WARNING - ID assert failed on method {} of type {} (wanted {} got {})\n",
-                  method_name.c_str(), type->get_name().c_str(), id, info.id);
-        throw std::runtime_error("Method ID assert failed");
+void declare_state_methods(Type* type,
+                           TypeSystem* type_system,
+                           const goos::Object& def,
+                           StructureDefResult& struct_def) {
+  for_each_in_list(def, [&](const goos::Object& _obj) {
+    auto obj = &_obj;
+    // either state-name or (state-name args...) or (state-name "docstring" args...)
+    std::string method_name;
+    TypeSpec function_typespec("state");
+    std::optional<std::string> docstring;
+
+    if (obj->is_symbol()) {
+      method_name = obj->as_symbol().name_ptr;
+    } else if (obj->is_list()) {
+      if (!car(obj).is_symbol()) {
+        throw std::runtime_error(
+            fmt::format("{} is not a valid name for a state-method", obj->print()));
       }
+      method_name = car(obj).as_symbol().name_ptr;
+      auto& args = *cdr(obj);
+      if (car(obj).is_string()) {
+        // docstring first
+        docstring = car(obj).as_string()->data;
+        obj = cdr(obj);
+      }
+      for_each_in_list(args, [&](const goos::Object& o) {
+        function_typespec.add_arg(parse_typespec(type_system, o));
+      });
     }
+    function_typespec.add_arg(TypeSpec("_type_"));
+
+    type_system->declare_method(type, method_name, docstring, false, function_typespec, false);
   });
 }
 
@@ -492,6 +574,8 @@ StructureDefResult parse_structure_def(
         declare_method(type, ts, *opt_list, result);
       } else if (list_name == ":states") {
         declare_state(type, ts, *opt_list, result);
+      } else if (list_name == ":state-methods") {
+        declare_state_methods(type, ts, *opt_list, result);
       } else {
         throw std::runtime_error("Invalid option list in field specification: " +
                                  car(rest).print());