shrub: start implementing extract_shrub

common/custom_data/Tfrag3Data.h

@@ -114,6 +114,17 @@ struct TfragTree {
   void serialize(Serializer& ser);
 };
 
+// A shrub model
+struct ShrubTree {
+  BVH bvh;
+  std::vector<StripDraw> static_draws;    // the actual topology and settings
+  std::vector<PreloadedVertex> vertices;  // mesh vertices
+  std::vector<TimeOfDayColor> colors;     // vertex colors (pre-interpolation)
+
+  // TODO - wind stuff
+  void serialize(Serializer& ser);
+};
+
 // A tie model
 struct TieTree {
   BVH bvh;
@@ -131,9 +142,10 @@ struct Level {
   std::string level_name;
   std::vector<Texture> textures;
   std::vector<TfragTree> tfrag_trees;
+  std::vector<ShrubTree> shrub_trees;
   std::vector<TieTree> tie_trees;
   u16 version2 = TFRAG3_VERSION;
   void serialize(Serializer& ser);
 };
 
-}  // namespace tfrag3
+}  // namespace tfrag3

decompiler/CMakeLists.txt

@@ -76,7 +76,7 @@ add_library(
         VuDisasm/VuDisassembler.cpp
         VuDisasm/VuInstruction.cpp
 
-        config.cpp)
+        config.cpp "level_extractor/extract_shrub.cpp" "level_extractor/extract_shrub.h" "level_extractor/extract_common.h")
 
 if (UNIX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2")
@@ -91,7 +91,7 @@ target_link_libraries(decomp
         )
 
 add_executable(decompiler
-        main.cpp)
+        main.cpp "level_extractor/extract_shrub.cpp" "level_extractor/extract_shrub.h" "level_extractor/extract_common.h")
 
 target_link_libraries(decompiler
         decomp

decompiler/config/all-types.gc

@@ -9504,7 +9504,7 @@
 (deftype prototype-bucket-shrub (prototype-bucket)
   ((mod-count     uint16  4       :offset-assert 88)
    (last          dma-packet 4       :offset-assert 96)
-   (something     uint128         :offset 80)
+   (count-clear   uint128         :offset 80)
    (last-clear    uint128         :offset 96)
    )
   :method-count-assert 9

decompiler/level_extractor/extract_common.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include "decompiler/level_extractor/BspHeader.h"
+#include "common/math/Vector.h"
+#include "common/custom_data/Tfrag3Data.h"
+#include "decompiler/data/TextureDB.h"
+
+namespace decompiler {
+
+/// <summary>
+/// Get the index of the first draw node in an array. Works for node or tfrag.
+/// </summary>
+/// <param name="array"></param>
+/// <returns></returns>
+u16 get_first_idx(const level_tools::DrawableInlineArray* array) {
+  auto as_tie_instances = dynamic_cast<const level_tools::DrawableInlineArrayInstanceTie*>(array);
+  auto as_nodes = dynamic_cast<const level_tools::DrawableInlineArrayNode*>(array);
+  if (as_tie_instances) {
+    return as_tie_instances->instances.at(0).id;
+  } else if (as_nodes) {
+    return as_nodes->draw_nodes.at(0).id;
+  } else {
+    assert(false);
+  }
+}
+
+/// <summary>
+/// Verify node indices follow the patterns we expect. Takes start as the expected first, writes the
+/// end.
+/// </summary>
+/// <param name="array"></param>
+/// <param name="start"></param>
+/// <param name="end"></param>
+/// <returns></returns>
+bool verify_node_indices_from_array(const level_tools::DrawableInlineArray* array,
+                                    u16 start,
+                                    u16* end) {
+  auto as_tie_instances = dynamic_cast<const level_tools::DrawableInlineArrayInstanceTie*>(array);
+  auto as_nodes = dynamic_cast<const level_tools::DrawableInlineArrayNode*>(array);
+
+  if (as_tie_instances) {
+    for (auto& elt : as_tie_instances->instances) {
+      if (elt.id != start) {
+        fmt::print("bad inst: exp {} got {}\n", start, elt.id);
+        return false;
+      }
+      start++;
+    }
+    *end = start;
+    return true;
+  } else if (as_nodes) {
+    for (auto& elt : as_nodes->draw_nodes) {
+      if (elt.id != start) {
+        fmt::print("bad node: exp {} got {}\n", start, elt.id);
+        return false;
+      }
+      start++;
+    }
+    *end = start;
+    return true;
+  } else {
+    fmt::print("bad node array type: {}\n", array->my_type());
+    return false;
+  }
+}
+}  // namespace decompiler

decompiler/level_extractor/extract_shrub.cpp

@@ -0,0 +1,180 @@
+#include <array>
+
+#include "extract_common.h"
+#include "extract_shrub.h"
+
+#include "decompiler/ObjectFile/LinkedObjectFile.h"
+
+#include "common/util/FileUtil.h"
+
+namespace decompiler {
+
+/// <summary>
+/// Verify all node indices in a tree.
+/// </summary>
+/// <param name="tree"></param>
+/// <returns>If valid or not</returns>
+bool verify_node_indices(const shrub_types::DrawableTreeInstanceShrub* tree) {
+  u16 start = get_first_idx(tree->arrays.at(0).get());
+  for (auto& array : tree->arrays) {
+    if (!verify_node_indices_from_array(array.get(), start, &start)) {
+      return false;
+    }
+    start = (start + 31) & ~(31);  // TODO - is this wrong for shrub (just copy pasted from tie for now)
+  }
+  return true;
+}
+
+/// <summary>
+/// Extract the visibility tree. This does not insert nodes for the bottom level.
+/// </summary>
+/// <param name="tree"></param>
+/// <param name="first_child"></param>
+/// <param name="out"></param>
+void extract_vis_data(const shrub_types::DrawableTreeInstanceShrub* tree,
+                      u16 first_child,
+                      tfrag3::ShrubTree& out) {
+  fmt::print("shrub::extract_vis_data not yet implemeneted!");
+}
+
+std::vector<shrub_types::PrototypeArrayShrubInfo> collect_shrub_info(
+    const shrub_types::DrawableInlineArrayInstanceShrub* instances,
+    const std::vector<shrub_types::PrototypeBucketShrub>* protos) {
+  std::vector<shrub_types::PrototypeArrayShrubInfo> result;
+  for (auto& instance : instances->instances) {
+    // todo left off
+    shrub_types::Shrub info;
+
+    info.prototype_idx = instance.bucket_index;
+    info.vis_id = instance.id;
+    for (int i = 0; i < 4; i++) {
+      info.bsphere[i] = instance.bsphere.data[i];
+    }
+    info.mat = extract_tie_matrix(instance.origin.data);
+    info.mat[3][0] += info.bsphere[0];
+    info.mat[3][1] += info.bsphere[1];
+    info.mat[3][2] += info.bsphere[2];
+    info.wind_index = instance.wind_index;
+    // there's a value stashed here that we can get rid of
+    // it is related to wind.
+    info.unknown_wind_related_value = info.mat[0][3];
+    info.mat[0][3] = 0.f;
+
+    // each fragment has its own color data (3 dmatags)
+
+    // the number of colors (qwc) is stored in the prototype, in the color-index-qwc array of bytes.
+    // at an offset of index-start[geom] + frag_idx.
+
+    // the actual data is located at the instance's color-indices + (proto.base-qw[geom] * 16)
+
+    // and this is only the indices.... there's yet another lookup on the VU
+    auto& proto = protos->at(info.prototype_idx);
+    u32 offset_bytes = proto.base_qw[GEOM_IDX] * 16;
+    for (int frag_idx = 0; frag_idx < proto.frag_count[GEOM_IDX]; frag_idx++) {
+      TieInstanceFragInfo frag_info;
+      u32 num_color_qwc = proto.color_index_qwc.at(proto.index_start[GEOM_IDX] + frag_idx);
+      for (u32 i = 0; i < num_color_qwc * 4; i++) {
+        for (u32 j = 0; j < 4; j++) {
+          frag_info.color_indices.push_back(
+              instance.color_indices.data->words_by_seg.at(instance.color_indices.seg)
+                  .at(((offset_bytes + instance.color_indices.byte_offset) / 4) + i)
+                  .get_byte(j));
+        }
+      }
+      info.frags.push_back(std::move(frag_info));
+      assert(info.frags.back().color_indices.size() > 0);
+      offset_bytes += num_color_qwc * 16;
+    }
+
+    if (result.size() <= info.prototype_idx) {
+      result.resize(info.prototype_idx + 1);
+    }
+    result[info.prototype_idx].instances.push_back(info);
+  }
+
+  return result;
+}
+
+void extract_shrub(const shrub_types::DrawableTreeInstanceShrub* tree,
+                   const std::string& debug_name,
+                   const std::vector<level_tools::TextureRemap>& map,
+                   const TextureDB& tex_db,
+                   const std::vector<std::pair<int, int>>& expected_missing_textures,
+                   tfrag3::Level& out,
+                   bool dump_level) {
+  tfrag3::ShrubTree this_tree;
+
+  // sanity check the vis tree (not a perfect check, but this is used in game and should be right)
+  assert(tree->length == (int)tree->arrays.size());
+  assert(tree->length > 0);
+  auto last_array = tree->arrays.at(6).get();  // shrub is the second last tree in `background-work`
+  auto as_instance_array = dynamic_cast<level_tools::DrawableInlineArrayInstanceTie*>(last_array);
+  assert(as_instance_array);
+  assert(as_instance_array->length == (int)as_instance_array->instances.size());
+  assert(as_instance_array->length > 0);
+  u16 idx = as_instance_array->instances.front().id;
+  for (auto& elt : as_instance_array->instances) {
+    assert(elt.id == idx);
+    idx++;
+  }
+  bool ok = verify_node_indices(tree);
+  assert(ok);
+  fmt::print("    tree has {} arrays and {} instances\n", tree->length, as_instance_array->length);
+
+  // extract the vis tree. Note that this extracts the tree only down to the last draw node, a
+  // parent of between 1 and 8 instances.
+  extract_vis_data(tree, as_instance_array->instances.front().id, this_tree);
+
+  // map of instance ID to its parent. We'll need this later.
+  std::unordered_map<int, int> instance_parents;
+  for (size_t node_idx = 0; node_idx < this_tree.bvh.vis_nodes.size(); node_idx++) {
+    const auto& node = this_tree.bvh.vis_nodes[node_idx];
+    if (node.flags == 0) {
+      for (int i = 0; i < node.num_kids; i++) {
+        instance_parents[node.child_id + i] = node_idx;
+      }
+    }
+  }
+
+  auto info = collect_instance_info(as_instance_array, &tree->prototypes.prototype_array_tie.data);
+  update_proto_info(&info, tex_map, tex_db, tree->prototypes.prototype_array_tie.data);
+
+  // AHHHH - likely stuck
+
+  // debug_print_info(info);
+  emulate_tie_prototype_program(info);
+  emulate_tie_instance_program(info);
+  emulate_kicks(info);
+
+  if (dump_level) {
+    auto dir = file_util::get_file_path({fmt::format("debug_out/tie-{}/", debug_name)});
+    file_util::create_dir_if_needed(dir);
+    for (auto& proto : info) {
+      auto data = debug_dump_proto_to_obj(proto);
+      file_util::write_text_file(fmt::format("{}/{}.obj", dir, proto.name), data);
+      // file_util::create_dir_if_needed()
+    }
+
+    auto full = dump_full_to_obj(info);
+    file_util::write_text_file(fmt::format("{}/ALL.obj", dir), full);
+  }
+
+  auto full_palette = make_big_palette(info);
+  add_vertices_and_static_draw(this_tree, out, tex_db, info);
+
+  for (auto& draw : this_tree.static_draws) {
+    for (auto& str : draw.vis_groups) {
+      auto it = instance_parents.find(str.vis_idx);
+      if (it == instance_parents.end()) {
+        str.vis_idx = UINT32_MAX;
+      } else {
+        str.vis_idx = it->second;
+      }
+    }
+  }
+
+  this_tree.colors = full_palette.colors;
+  fmt::print("TIE tree has {} draws\n", this_tree.static_draws.size());
+  out.tie_trees.push_back(std::move(this_tree));
+}
+}  // namespace decompiler

decompiler/level_extractor/extract_shrub.h

@@ -0,0 +1,171 @@
+#pragma once
+
+#include "decompiler/level_extractor/BspHeader.h"
+#include "common/math/Vector.h"
+#include "common/custom_data/Tfrag3Data.h"
+#include "decompiler/data/TextureDB.h"
+
+namespace decompiler {
+
+namespace shrub_types {
+
+struct Shrubbery : public level_tools::Drawable {
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats) override;
+  std::string print(const level_tools::PrintSettings& settings, int indent) const override;
+  std::string my_type() const override { return "shrubbery"; }
+
+  // 4 - textures
+  u16 gif_count;
+  std::vector<u8> gif_data;
+
+  u32 header_unknown;  // 8
+  u8 obj_qwc;          // 12
+  u8 vtx_qwc;          // 13
+  u8 col_qwc;          // 14
+  u8 stq_qwc;          // 15
+
+  u32 obj;   // 16
+  u32 vtx;   // 20
+  u32 col;   // 24
+  u32 steq;  // 28
+};
+
+struct PrototypeShrubbery : public level_tools::DrawableInlineArray {
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats) override;
+  std::string print(const level_tools::PrintSettings& settings, int indent) const override;
+  std::string my_type() const override;
+  s16 id;      // 0
+  s16 length;  // 4
+
+  level_tools::Vector bsphere;    // 16
+  std::vector<Shrubbery> shrubs;  // 32
+};
+
+struct PrototypeBucketShrub {
+  std::string name;  // 4
+  u32 flags;         // 8
+  u16 in_level;      // 12
+  u16 utextures;     // 14
+
+  PrototypeShrubbery geometry[4];  // 16
+
+  level_tools::Vector dists;  // 32
+  // - near-plane
+  // - near-stiff
+  // - mid-plane
+  // - far-plane
+  level_tools::Vector rdists;  // 48
+  // - rlength-near
+  // - rlength-stiff
+  // - rlength-mid
+  // - stiffness
+
+  u32 next[4];   // 64
+  u16 count[4];  // 80 | NOTE - overlayed in places as a 128bit value
+
+  u16 mod_count[4];  // 88
+  Ref last[4];       // 96 | NOTE - overlayed to clear with a single uint128
+
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats);
+  std::string print(const level_tools::PrintSettings& settings, int indent) const;
+};
+
+struct PrototypeArrayShrub {
+  s16 id;                                  // 4
+  s16 length;                              // 6
+  level_tools::Vector bsphere;             // 16
+  std::vector<PrototypeBucketShrub> data;  // 32
+
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats);
+  std::string print(const level_tools::PrintSettings& settings, int indent) const;
+};
+
+struct PrototypeArrayShrubInfo {
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats);
+  std::string print(const level_tools::PrintSettings& settings, int indent) const;
+
+  PrototypeArrayShrub prototype_array_shrub;
+  // todo wind vectors.
+};
+
+struct InstanceShrubbery : public level_tools::Drawable {
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats) override;
+  std::string print(const level_tools::PrintSettings& settings, int indent) const override;
+  std::string my_type() const override { return "instance-shrubbery"; }
+
+  s16 id;                        // 4
+  u16 bucket_index;              // 6
+  level_tools::Vector bsphere;   // 16
+  level_tools::Matrix4h origin;  // 32
+  u16 flags;                     // ??
+  u16 wind_index;                // 62
+
+  // --- instance-shrubbery ---
+  u32 color;                        // 8
+  level_tools::Vector flat_normal;  // 64
+  level_tools::Vector flat_hwidth;  // 76
+};
+
+struct DrawableInlineArrayInstanceShrub : public level_tools::DrawableInlineArray {
+  s16 id;
+  s16 length;
+  level_tools::Vector bsphere;
+
+  std::vector<InstanceShrubbery> instances;
+
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats) override;
+  std::string print(const level_tools::PrintSettings& settings, int indent) const override;
+  std::string my_type() const override;
+};
+
+struct DrawableTreeInstanceShrub : public level_tools::DrawableTree {
+  void read_from_file(TypedRef ref,
+                      const decompiler::DecompilerTypeSystem& dts,
+                      level_tools::DrawStats* stats) override;
+  std::string print(const level_tools::PrintSettings& settings, int indent) const override;
+  std::string my_type() const override;
+
+  s16 id;                        // 0
+  s16 length;                    // 4
+  PrototypeArrayShrubInfo info;  // 8
+  u32 colors_added;              // 12
+  level_tools::Vector bsphere;   // 16
+
+  std::vector<std::unique_ptr<level_tools::DrawableInlineArray>> arrays;
+};
+
+}  // namespace shrub_types
+
+/// <summary>
+/// Extract shrubs from the level
+/// </summary>
+/// <param name="tree"></param>
+/// <param name="debug_name"></param>
+/// <param name="map"></param>
+/// <param name="tex_db"></param>
+/// <param name="expected_missing_textures"></param>
+/// <param name="out"></param>
+/// <param name="dump_level"></param>
+void extract_shrub(const shrub_types::DrawableTreeInstanceShrub* tree,
+                   const std::string& debug_name,
+                   const std::vector<level_tools::TextureRemap>& map,
+                   const TextureDB& tex_db,
+                   const std::vector<std::pair<int, int>>& expected_missing_textures,
+                   tfrag3::Level& out,
+                   bool dump_level);
+
+}  // namespace decompiler

decompiler/level_extractor/extract_tie.cpp

@@ -1,5 +1,6 @@
 #include <array>
 
+#include "extract_common.h"
 #include "extract_tie.h"
 
 #include "decompiler/ObjectFile/LinkedObjectFile.h"
@@ -8,57 +9,6 @@
 
 namespace decompiler {
 
-/*!
- * Get the index of the first draw node in an array. Works for node or tfrag.
- */
-u16 get_first_idx(const level_tools::DrawableInlineArray* array) {
-  auto as_tie_instances = dynamic_cast<const level_tools::DrawableInlineArrayInstanceTie*>(array);
-  auto as_nodes = dynamic_cast<const level_tools::DrawableInlineArrayNode*>(array);
-  if (as_tie_instances) {
-    return as_tie_instances->instances.at(0).id;
-  } else if (as_nodes) {
-    return as_nodes->draw_nodes.at(0).id;
-  } else {
-    assert(false);
-  }
-}
-
-/*!
- * Verify node indices follow the patterns we expect. Takes start as the expected first,
- * writes the end.
- */
-bool verify_node_indices_from_array(const level_tools::DrawableInlineArray* array,
-                                    u16 start,
-                                    u16* end) {
-  auto as_tie_instances = dynamic_cast<const level_tools::DrawableInlineArrayInstanceTie*>(array);
-  auto as_nodes = dynamic_cast<const level_tools::DrawableInlineArrayNode*>(array);
-
-  if (as_tie_instances) {
-    for (auto& elt : as_tie_instances->instances) {
-      if (elt.id != start) {
-        fmt::print("bad inst: exp {} got {}\n", start, elt.id);
-        return false;
-      }
-      start++;
-    }
-    *end = start;
-    return true;
-  } else if (as_nodes) {
-    for (auto& elt : as_nodes->draw_nodes) {
-      if (elt.id != start) {
-        fmt::print("bad node: exp {} got {}\n", start, elt.id);
-        return false;
-      }
-      start++;
-    }
-    *end = start;
-    return true;
-  } else {
-    fmt::print("bad node array type: {}\n", array->my_type());
-    return false;
-  }
-}
-
 /*!
  * Verify all node indices in a tree.
  */