diff --git a/common/custom_data/Tfrag3Data.h b/common/custom_data/Tfrag3Data.h index ceca48701b..00654e46ab 100644 --- a/common/custom_data/Tfrag3Data.h +++ b/common/custom_data/Tfrag3Data.h @@ -114,6 +114,17 @@ struct TfragTree { void serialize(Serializer& ser); }; +// A shrub model +struct ShrubTree { + BVH bvh; + std::vector static_draws; // the actual topology and settings + std::vector vertices; // mesh vertices + std::vector colors; // vertex colors (pre-interpolation) + + // TODO - wind stuff + void serialize(Serializer& ser); +}; + // A tie model struct TieTree { BVH bvh; @@ -131,9 +142,10 @@ struct Level { std::string level_name; std::vector textures; std::vector tfrag_trees; + std::vector shrub_trees; std::vector tie_trees; u16 version2 = TFRAG3_VERSION; void serialize(Serializer& ser); }; -} // namespace tfrag3 \ No newline at end of file +} // namespace tfrag3 diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 1e81e8d938..50c67bcd27 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -76,7 +76,7 @@ add_library( VuDisasm/VuDisassembler.cpp VuDisasm/VuInstruction.cpp - config.cpp) + config.cpp "level_extractor/extract_shrub.cpp" "level_extractor/extract_shrub.h" "level_extractor/extract_common.h") if (UNIX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2") @@ -91,7 +91,7 @@ target_link_libraries(decomp ) add_executable(decompiler - main.cpp) + main.cpp "level_extractor/extract_shrub.cpp" "level_extractor/extract_shrub.h" "level_extractor/extract_common.h") target_link_libraries(decompiler decomp diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index ecc3f2dc5c..38817a43f1 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -9504,7 +9504,7 @@ (deftype prototype-bucket-shrub (prototype-bucket) ((mod-count uint16 4 :offset-assert 88) (last dma-packet 4 :offset-assert 96) - (something uint128 :offset 80) + (count-clear uint128 :offset 80) (last-clear uint128 :offset 96) ) :method-count-assert 9 diff --git a/decompiler/level_extractor/extract_common.h b/decompiler/level_extractor/extract_common.h new file mode 100644 index 0000000000..3c8932185c --- /dev/null +++ b/decompiler/level_extractor/extract_common.h @@ -0,0 +1,66 @@ +#pragma once + +#include "decompiler/level_extractor/BspHeader.h" +#include "common/math/Vector.h" +#include "common/custom_data/Tfrag3Data.h" +#include "decompiler/data/TextureDB.h" + +namespace decompiler { + +/// +/// Get the index of the first draw node in an array. Works for node or tfrag. +/// +/// +/// +u16 get_first_idx(const level_tools::DrawableInlineArray* array) { + auto as_tie_instances = dynamic_cast(array); + auto as_nodes = dynamic_cast(array); + if (as_tie_instances) { + return as_tie_instances->instances.at(0).id; + } else if (as_nodes) { + return as_nodes->draw_nodes.at(0).id; + } else { + assert(false); + } +} + +/// +/// Verify node indices follow the patterns we expect. Takes start as the expected first, writes the +/// end. +/// +/// +/// +/// +/// +bool verify_node_indices_from_array(const level_tools::DrawableInlineArray* array, + u16 start, + u16* end) { + auto as_tie_instances = dynamic_cast(array); + auto as_nodes = dynamic_cast(array); + + if (as_tie_instances) { + for (auto& elt : as_tie_instances->instances) { + if (elt.id != start) { + fmt::print("bad inst: exp {} got {}\n", start, elt.id); + return false; + } + start++; + } + *end = start; + return true; + } else if (as_nodes) { + for (auto& elt : as_nodes->draw_nodes) { + if (elt.id != start) { + fmt::print("bad node: exp {} got {}\n", start, elt.id); + return false; + } + start++; + } + *end = start; + return true; + } else { + fmt::print("bad node array type: {}\n", array->my_type()); + return false; + } +} +} // namespace decompiler diff --git a/decompiler/level_extractor/extract_shrub.cpp b/decompiler/level_extractor/extract_shrub.cpp new file mode 100644 index 0000000000..e434869b1a --- /dev/null +++ b/decompiler/level_extractor/extract_shrub.cpp @@ -0,0 +1,180 @@ +#include + +#include "extract_common.h" +#include "extract_shrub.h" + +#include "decompiler/ObjectFile/LinkedObjectFile.h" + +#include "common/util/FileUtil.h" + +namespace decompiler { + +/// +/// Verify all node indices in a tree. +/// +/// +/// If valid or not +bool verify_node_indices(const shrub_types::DrawableTreeInstanceShrub* tree) { + u16 start = get_first_idx(tree->arrays.at(0).get()); + for (auto& array : tree->arrays) { + if (!verify_node_indices_from_array(array.get(), start, &start)) { + return false; + } + start = (start + 31) & ~(31); // TODO - is this wrong for shrub (just copy pasted from tie for now) + } + return true; +} + +/// +/// Extract the visibility tree. This does not insert nodes for the bottom level. +/// +/// +/// +/// +void extract_vis_data(const shrub_types::DrawableTreeInstanceShrub* tree, + u16 first_child, + tfrag3::ShrubTree& out) { + fmt::print("shrub::extract_vis_data not yet implemeneted!"); +} + +std::vector collect_shrub_info( + const shrub_types::DrawableInlineArrayInstanceShrub* instances, + const std::vector* protos) { + std::vector result; + for (auto& instance : instances->instances) { + // todo left off + shrub_types::Shrub info; + + info.prototype_idx = instance.bucket_index; + info.vis_id = instance.id; + for (int i = 0; i < 4; i++) { + info.bsphere[i] = instance.bsphere.data[i]; + } + info.mat = extract_tie_matrix(instance.origin.data); + info.mat[3][0] += info.bsphere[0]; + info.mat[3][1] += info.bsphere[1]; + info.mat[3][2] += info.bsphere[2]; + info.wind_index = instance.wind_index; + // there's a value stashed here that we can get rid of + // it is related to wind. + info.unknown_wind_related_value = info.mat[0][3]; + info.mat[0][3] = 0.f; + + // each fragment has its own color data (3 dmatags) + + // the number of colors (qwc) is stored in the prototype, in the color-index-qwc array of bytes. + // at an offset of index-start[geom] + frag_idx. + + // the actual data is located at the instance's color-indices + (proto.base-qw[geom] * 16) + + // and this is only the indices.... there's yet another lookup on the VU + auto& proto = protos->at(info.prototype_idx); + u32 offset_bytes = proto.base_qw[GEOM_IDX] * 16; + for (int frag_idx = 0; frag_idx < proto.frag_count[GEOM_IDX]; frag_idx++) { + TieInstanceFragInfo frag_info; + u32 num_color_qwc = proto.color_index_qwc.at(proto.index_start[GEOM_IDX] + frag_idx); + for (u32 i = 0; i < num_color_qwc * 4; i++) { + for (u32 j = 0; j < 4; j++) { + frag_info.color_indices.push_back( + instance.color_indices.data->words_by_seg.at(instance.color_indices.seg) + .at(((offset_bytes + instance.color_indices.byte_offset) / 4) + i) + .get_byte(j)); + } + } + info.frags.push_back(std::move(frag_info)); + assert(info.frags.back().color_indices.size() > 0); + offset_bytes += num_color_qwc * 16; + } + + if (result.size() <= info.prototype_idx) { + result.resize(info.prototype_idx + 1); + } + result[info.prototype_idx].instances.push_back(info); + } + + return result; +} + +void extract_shrub(const shrub_types::DrawableTreeInstanceShrub* tree, + const std::string& debug_name, + const std::vector& map, + const TextureDB& tex_db, + const std::vector>& expected_missing_textures, + tfrag3::Level& out, + bool dump_level) { + tfrag3::ShrubTree this_tree; + + // sanity check the vis tree (not a perfect check, but this is used in game and should be right) + assert(tree->length == (int)tree->arrays.size()); + assert(tree->length > 0); + auto last_array = tree->arrays.at(6).get(); // shrub is the second last tree in `background-work` + auto as_instance_array = dynamic_cast(last_array); + assert(as_instance_array); + assert(as_instance_array->length == (int)as_instance_array->instances.size()); + assert(as_instance_array->length > 0); + u16 idx = as_instance_array->instances.front().id; + for (auto& elt : as_instance_array->instances) { + assert(elt.id == idx); + idx++; + } + bool ok = verify_node_indices(tree); + assert(ok); + fmt::print(" tree has {} arrays and {} instances\n", tree->length, as_instance_array->length); + + // extract the vis tree. Note that this extracts the tree only down to the last draw node, a + // parent of between 1 and 8 instances. + extract_vis_data(tree, as_instance_array->instances.front().id, this_tree); + + // map of instance ID to its parent. We'll need this later. + std::unordered_map instance_parents; + for (size_t node_idx = 0; node_idx < this_tree.bvh.vis_nodes.size(); node_idx++) { + const auto& node = this_tree.bvh.vis_nodes[node_idx]; + if (node.flags == 0) { + for (int i = 0; i < node.num_kids; i++) { + instance_parents[node.child_id + i] = node_idx; + } + } + } + + auto info = collect_instance_info(as_instance_array, &tree->prototypes.prototype_array_tie.data); + update_proto_info(&info, tex_map, tex_db, tree->prototypes.prototype_array_tie.data); + + // AHHHH - likely stuck + + // debug_print_info(info); + emulate_tie_prototype_program(info); + emulate_tie_instance_program(info); + emulate_kicks(info); + + if (dump_level) { + auto dir = file_util::get_file_path({fmt::format("debug_out/tie-{}/", debug_name)}); + file_util::create_dir_if_needed(dir); + for (auto& proto : info) { + auto data = debug_dump_proto_to_obj(proto); + file_util::write_text_file(fmt::format("{}/{}.obj", dir, proto.name), data); + // file_util::create_dir_if_needed() + } + + auto full = dump_full_to_obj(info); + file_util::write_text_file(fmt::format("{}/ALL.obj", dir), full); + } + + auto full_palette = make_big_palette(info); + add_vertices_and_static_draw(this_tree, out, tex_db, info); + + for (auto& draw : this_tree.static_draws) { + for (auto& str : draw.vis_groups) { + auto it = instance_parents.find(str.vis_idx); + if (it == instance_parents.end()) { + str.vis_idx = UINT32_MAX; + } else { + str.vis_idx = it->second; + } + } + } + + this_tree.colors = full_palette.colors; + fmt::print("TIE tree has {} draws\n", this_tree.static_draws.size()); + out.tie_trees.push_back(std::move(this_tree)); +} +} // namespace decompiler diff --git a/decompiler/level_extractor/extract_shrub.h b/decompiler/level_extractor/extract_shrub.h new file mode 100644 index 0000000000..d812677857 --- /dev/null +++ b/decompiler/level_extractor/extract_shrub.h @@ -0,0 +1,171 @@ +#pragma once + +#include "decompiler/level_extractor/BspHeader.h" +#include "common/math/Vector.h" +#include "common/custom_data/Tfrag3Data.h" +#include "decompiler/data/TextureDB.h" + +namespace decompiler { + +namespace shrub_types { + +struct Shrubbery : public level_tools::Drawable { + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats) override; + std::string print(const level_tools::PrintSettings& settings, int indent) const override; + std::string my_type() const override { return "shrubbery"; } + + // 4 - textures + u16 gif_count; + std::vector gif_data; + + u32 header_unknown; // 8 + u8 obj_qwc; // 12 + u8 vtx_qwc; // 13 + u8 col_qwc; // 14 + u8 stq_qwc; // 15 + + u32 obj; // 16 + u32 vtx; // 20 + u32 col; // 24 + u32 steq; // 28 +}; + +struct PrototypeShrubbery : public level_tools::DrawableInlineArray { + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats) override; + std::string print(const level_tools::PrintSettings& settings, int indent) const override; + std::string my_type() const override; + s16 id; // 0 + s16 length; // 4 + + level_tools::Vector bsphere; // 16 + std::vector shrubs; // 32 +}; + +struct PrototypeBucketShrub { + std::string name; // 4 + u32 flags; // 8 + u16 in_level; // 12 + u16 utextures; // 14 + + PrototypeShrubbery geometry[4]; // 16 + + level_tools::Vector dists; // 32 + // - near-plane + // - near-stiff + // - mid-plane + // - far-plane + level_tools::Vector rdists; // 48 + // - rlength-near + // - rlength-stiff + // - rlength-mid + // - stiffness + + u32 next[4]; // 64 + u16 count[4]; // 80 | NOTE - overlayed in places as a 128bit value + + u16 mod_count[4]; // 88 + Ref last[4]; // 96 | NOTE - overlayed to clear with a single uint128 + + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats); + std::string print(const level_tools::PrintSettings& settings, int indent) const; +}; + +struct PrototypeArrayShrub { + s16 id; // 4 + s16 length; // 6 + level_tools::Vector bsphere; // 16 + std::vector data; // 32 + + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats); + std::string print(const level_tools::PrintSettings& settings, int indent) const; +}; + +struct PrototypeArrayShrubInfo { + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats); + std::string print(const level_tools::PrintSettings& settings, int indent) const; + + PrototypeArrayShrub prototype_array_shrub; + // todo wind vectors. +}; + +struct InstanceShrubbery : public level_tools::Drawable { + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats) override; + std::string print(const level_tools::PrintSettings& settings, int indent) const override; + std::string my_type() const override { return "instance-shrubbery"; } + + s16 id; // 4 + u16 bucket_index; // 6 + level_tools::Vector bsphere; // 16 + level_tools::Matrix4h origin; // 32 + u16 flags; // ?? + u16 wind_index; // 62 + + // --- instance-shrubbery --- + u32 color; // 8 + level_tools::Vector flat_normal; // 64 + level_tools::Vector flat_hwidth; // 76 +}; + +struct DrawableInlineArrayInstanceShrub : public level_tools::DrawableInlineArray { + s16 id; + s16 length; + level_tools::Vector bsphere; + + std::vector instances; + + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats) override; + std::string print(const level_tools::PrintSettings& settings, int indent) const override; + std::string my_type() const override; +}; + +struct DrawableTreeInstanceShrub : public level_tools::DrawableTree { + void read_from_file(TypedRef ref, + const decompiler::DecompilerTypeSystem& dts, + level_tools::DrawStats* stats) override; + std::string print(const level_tools::PrintSettings& settings, int indent) const override; + std::string my_type() const override; + + s16 id; // 0 + s16 length; // 4 + PrototypeArrayShrubInfo info; // 8 + u32 colors_added; // 12 + level_tools::Vector bsphere; // 16 + + std::vector> arrays; +}; + +} // namespace shrub_types + +/// +/// Extract shrubs from the level +/// +/// +/// +/// +/// +/// +/// +/// +void extract_shrub(const shrub_types::DrawableTreeInstanceShrub* tree, + const std::string& debug_name, + const std::vector& map, + const TextureDB& tex_db, + const std::vector>& expected_missing_textures, + tfrag3::Level& out, + bool dump_level); + +} // namespace decompiler diff --git a/decompiler/level_extractor/extract_tie.cpp b/decompiler/level_extractor/extract_tie.cpp index 104173cbbb..547c3a75b4 100644 --- a/decompiler/level_extractor/extract_tie.cpp +++ b/decompiler/level_extractor/extract_tie.cpp @@ -1,5 +1,6 @@ #include +#include "extract_common.h" #include "extract_tie.h" #include "decompiler/ObjectFile/LinkedObjectFile.h" @@ -8,57 +9,6 @@ namespace decompiler { -/*! - * Get the index of the first draw node in an array. Works for node or tfrag. - */ -u16 get_first_idx(const level_tools::DrawableInlineArray* array) { - auto as_tie_instances = dynamic_cast(array); - auto as_nodes = dynamic_cast(array); - if (as_tie_instances) { - return as_tie_instances->instances.at(0).id; - } else if (as_nodes) { - return as_nodes->draw_nodes.at(0).id; - } else { - assert(false); - } -} - -/*! - * Verify node indices follow the patterns we expect. Takes start as the expected first, - * writes the end. - */ -bool verify_node_indices_from_array(const level_tools::DrawableInlineArray* array, - u16 start, - u16* end) { - auto as_tie_instances = dynamic_cast(array); - auto as_nodes = dynamic_cast(array); - - if (as_tie_instances) { - for (auto& elt : as_tie_instances->instances) { - if (elt.id != start) { - fmt::print("bad inst: exp {} got {}\n", start, elt.id); - return false; - } - start++; - } - *end = start; - return true; - } else if (as_nodes) { - for (auto& elt : as_nodes->draw_nodes) { - if (elt.id != start) { - fmt::print("bad node: exp {} got {}\n", start, elt.id); - return false; - } - start++; - } - *end = start; - return true; - } else { - fmt::print("bad node array type: {}\n", array->my_type()); - return false; - } -} - /*! * Verify all node indices in a tree. */