decompiler: export blerc data to glb (#4316)

This adds the blerc data from merc models to the GLB export as shape
keys, also including the data from animations. Blender is unfortunately
a bit weird about this and only really lets you change what animated
weights are used via the NLA track editor by selecting both the
corresponding armature animation and then the one for the shape keys.
Additionally, due to the way the Blender GLB import works when you have
animated weights on a model + animations as actions for an armature, any
models that have blerc data get an extra useless empty, but this seems
to be harmless.

This PR also fixes materials that use texture animations so they now
have their base texture in the export, rather than using the default
material. Materials now also get named after their base texture for
easier recognition.

common/custom_data/Tfrag3Data.h

@@ -578,6 +578,7 @@ struct MercModifiableDrawGroup {
   std::vector<u8> fragment_mask;
   Blerc blerc;
   u32 expect_vidx_end = 0;
+  std::vector<u32> mod_to_global_vertex_idx;  // not serialized, used for glb export
 
   void serialize(Serializer& ser);
   void memory_usage(MemoryUsageTracker* tracker) const;

decompiler/level_extractor/common_formats.h

@@ -27,6 +27,7 @@ struct UncompressedJointAnim {
   std::vector<UncompressedSingleJointAnim> joints;
   float framerate = 30;
   int frames = 0;
+  std::vector<u8> blend_shape_data;
 };
 
 struct CompressedMatrixMetadata {
@@ -144,6 +145,7 @@ struct ArtJointAnim {
   float artist_step;
   s16 length;
   JointAnimCompressedControl frames;
+  std::vector<u8> blend_shape_data;
 };
 
 /*!

decompiler/level_extractor/extract_anim.cpp

@@ -3,6 +3,7 @@
 #include "common_formats.h"
 
 #include "decompiler/ObjectFile/LinkedObjectFile.h"
+#include "decompiler/ObjectFile/LinkedWord.h"
 #include "decompiler/util/goal_data_reader.h"
 
 #include "third-party/lzokay/lzokay.hpp"
@@ -257,6 +258,13 @@ void extract_animations(const ObjectFileData& ag_data,
         frames_ref.byte_offset += 4;
       }
     }
+    // extract blerc data if present
+    const char* blerc_field = version == GameVersion::Jak1 ? "blerc-data" : "blend-shape-anim";
+    if (get_word_kind_for_field(ref, blerc_field, dts) == LinkedWord::PTR) {
+      Ref blerc_data_ref = deref_label(get_field_ref(ref, blerc_field, dts));
+      ja.blend_shape_data = get_plain_data_bytes_up_to_label(blerc_data_ref);
+    }
+
     // some master art groups and model names do not match, this remaps the model name based on the
     // animation name prefix for these rare exceptions
     PerGameVersion<std::vector<std::string>> mdl_name_remap{

decompiler/level_extractor/extract_level.cpp

@@ -346,7 +346,8 @@ void extract_common(const ObjectFileDB& db,
   if (config.rip_levels) {
     auto file_path = file_util::get_jak_project_dir() / "decompiler_out" /
                      game_version_names[config.game_version] / "levels" / "common";
-    save_level_foreground_as_gltf(tfrag_level, art_group_data, file_path);
+    save_level_foreground_as_gltf(tfrag_level, art_group_data, file_path,
+                                  tex_db.animated_tex_output_to_anim_slot);
   }
 }
 
@@ -394,7 +395,8 @@ void extract_from_level(const ObjectFileDB& db,
     auto fore_file_path = file_util::get_jak_project_dir() / "decompiler_out" /
                           game_version_names[config.game_version] / "levels" /
                           level_data.level_name;
-    save_level_foreground_as_gltf(level_data, art_group_data, fore_file_path);
+    save_level_foreground_as_gltf(level_data, art_group_data, fore_file_path,
+                                  tex_db.animated_tex_output_to_anim_slot);
   }
   file_util::write_text_file(entities_folder / fmt::format("{}-actors.json", level_data.level_name),
                              extract_actors_to_json(bsp_header.actors));

decompiler/level_extractor/extract_merc.cpp

@@ -1529,6 +1529,7 @@ void create_modifiable_vertex_data(
               vtx_to_mod_vtx[vidx] = idx;
               // add the vertex
               effect.mod.vertices.push_back(out.vertices.at(vidx));
+              effect.mod.mod_to_global_vertex_idx.push_back(vidx);
               // look up where this one came from in the EE memory layout
               auto src = vtx_srcs.at(vidx - first_out_vertex);
               ASSERT(src.combined_lump4 < UINT16_MAX);

decompiler/level_extractor/fr3_to_gltf.cpp

@@ -1,7 +1,6 @@
 #include "fr3_to_gltf.h"
 
 #include <algorithm>
-#include <unordered_map>
 
 #include "common/custom_data/Tfrag3Data.h"
 #include "common/math/Vector.h"
@@ -569,13 +568,13 @@ int add_material_for_tex(const tfrag3::Level& level,
                          std::unordered_map<int, int>& tex_image_map,
                          const DrawMode& draw_mode) {
   if (tex_idx < 0) {
-    // anim textures, just use default material
     return 0;
   }
   int mat_idx = (int)model.materials.size();
   auto& mat = model.materials.emplace_back();
   auto& tex = level.textures.at(tex_idx);
 
+  mat.name = tex.debug_name;
   mat.doubleSided = true;
   // the 2.0 here compensates for the ps2's weird blending where 0.5 behaves like 1.0
   mat.pbrMetallicRoughness.baseColorFactor = {2.0, 2.0, 2.0, 2.0};
@@ -1029,6 +1028,7 @@ level_tools::UncompressedJointAnim decompress_anim(const level_tools::ArtJointAn
     }
   }
 
+  out.blend_shape_data = art_anim.blend_shape_data;
   return out;
 }
 
@@ -1111,7 +1111,9 @@ int make_anim_vec4_accessor(const std::vector<math::Vector4f>& values, tinygltf:
 
 void add_animation_to_gltf(const level_tools::UncompressedJointAnim& anim,
                            const tinygltf::Skin& skin,
-                           tinygltf::Model& model) {
+                           tinygltf::Model& model,
+                           int mesh_node_idx,
+                           int num_targets) {
   if (anim.frames == 0 || anim.joints.size() <= 2)
     return;
 
@@ -1148,13 +1150,145 @@ void add_animation_to_gltf(const level_tools::UncompressedJointAnim& anim,
     if ((int)joint.scale_frames.size() == anim.frames)
       add_channel(make_anim_vec3_accessor(joint.scale_frames, model), "scale");
   }
+
+  if (num_targets > 0 && !anim.blend_shape_data.empty() &&
+      (int)anim.blend_shape_data.size() >= anim.frames * num_targets) {
+    std::vector<float> weights(anim.frames * num_targets);
+    for (int fi = 0; fi < anim.frames; fi++) {
+      for (int ti = 0; ti < num_targets; ti++) {
+        u8 raw = anim.blend_shape_data[fi * num_targets + ti];
+        weights[fi * num_targets + ti] = (raw - 64.f) / 128.f;
+      }
+    }
+    int si = (int)gltf_anim.samplers.size();
+    auto& sampler = gltf_anim.samplers.emplace_back();
+    sampler.input = time_acc;
+    sampler.output = make_anim_float_accessor(weights, model);
+    sampler.interpolation = "LINEAR";
+    auto& channel = gltf_anim.channels.emplace_back();
+    channel.sampler = si;
+    channel.target_node = mesh_node_idx;
+    channel.target_path = "weights";
+  }
+}
+
+int make_vec3_float_accessor(const std::vector<float>& data, tinygltf::Model& model) {
+  int buffer_idx = (int)model.buffers.size();
+  auto& buffer = model.buffers.emplace_back();
+  buffer.data.resize(data.size() * sizeof(float));
+  memcpy(buffer.data.data(), data.data(), buffer.data.size());
+
+  int buffer_view_idx = (int)model.bufferViews.size();
+  auto& bv = model.bufferViews.emplace_back();
+  bv.buffer = buffer_idx;
+  bv.byteOffset = 0;
+  bv.byteLength = buffer.data.size();
+
+  int accessor_idx = (int)model.accessors.size();
+  auto& accessor = model.accessors.emplace_back();
+  accessor.bufferView = buffer_view_idx;
+  accessor.byteOffset = 0;
+  accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
+  accessor.count = data.size() / 3;
+  accessor.type = TINYGLTF_TYPE_VEC3;
+  return accessor_idx;
+}
+
+void add_blerc_targets(const tfrag3::Level& level,
+                       const tfrag3::MercModel& mmodel,
+                       tinygltf::Mesh& mesh,
+                       tinygltf::Model& model) {
+  // find max target index across all effects to know how many morph targets we need
+  u32 num_targets = 0;
+  for (const auto& effect : mmodel.effects) {
+    const auto& blerc = effect.mod.blerc;
+    if (blerc.int_data.empty()) {
+      continue;
+    }
+    bool skip_next = false;
+    for (u32 v : blerc.int_data) {
+      if (skip_next) {
+        skip_next = false;
+        continue;
+      }
+      if (v == tfrag3::Blerc::kTargetIdxTerminator) {
+        skip_next = true;
+      } else {
+        num_targets = std::max(num_targets, v + 1);
+      }
+    }
+  }
+  if (num_targets == 0) {
+    return;
+  }
+
+  const auto num_vtx = (u32)level.merc_data.vertices.size();
+  std::vector pos_deltas(num_targets, std::vector(num_vtx * 3, 0.f));
+  std::vector nrm_deltas(num_targets, std::vector(num_vtx * 3, 0.f));
+
+  for (const auto& effect : mmodel.effects) {
+    const auto& blerc = effect.mod.blerc;
+    if (blerc.int_data.empty() || effect.mod.mod_to_global_vertex_idx.empty()) {
+      continue;
+    }
+
+    size_t float_idx = 0;
+    size_t int_idx = 0;
+    while (int_idx < blerc.int_data.size()) {
+      float_idx++;  // skip base pos/nrm entry
+
+      // collect (target_idx, float_data_index) pairs for this vertex
+      struct TargetEntry {
+        u32 tgt_idx;
+        size_t float_offset;
+      };
+      std::vector<TargetEntry> vertex_targets;
+      while (blerc.int_data[int_idx] != tfrag3::Blerc::kTargetIdxTerminator) {
+        vertex_targets.push_back({blerc.int_data[int_idx++], float_idx++});
+      }
+      int_idx++;
+      u32 dest = blerc.int_data[int_idx++];
+
+      if (dest >= effect.mod.mod_to_global_vertex_idx.size()) {
+        continue;
+      }
+      u32 global_idx = effect.mod.mod_to_global_vertex_idx[dest];
+
+      for (const auto& te : vertex_targets) {
+        if (te.tgt_idx >= num_targets || te.float_offset >= blerc.float_data.size()) {
+          continue;
+        }
+        const auto& fd = blerc.float_data[te.float_offset];
+        pos_deltas[te.tgt_idx][global_idx * 3 + 0] = fd.v[0] * (8192.f / 4096.f);
+        pos_deltas[te.tgt_idx][global_idx * 3 + 1] = fd.v[1] * (8192.f / 4096.f);
+        pos_deltas[te.tgt_idx][global_idx * 3 + 2] = fd.v[2] * (8192.f / 4096.f);
+        nrm_deltas[te.tgt_idx][global_idx * 3 + 0] = fd.v[4] * 8192.f;
+        nrm_deltas[te.tgt_idx][global_idx * 3 + 1] = fd.v[5] * 8192.f;
+        nrm_deltas[te.tgt_idx][global_idx * 3 + 2] = fd.v[6] * 8192.f;
+      }
+    }
+  }
+
+  // build one accessor pair per morph target and attach to all primitives
+  std::vector<std::map<std::string, int>> morph_targets;
+  for (u32 t = 0; t < num_targets; t++) {
+    auto& target = morph_targets.emplace_back();
+    target["POSITION"] = make_vec3_float_accessor(pos_deltas[t], model);
+    target["NORMAL"] = make_vec3_float_accessor(nrm_deltas[t], model);
+  }
+
+  for (auto& prim : mesh.primitives) {
+    prim.targets = morph_targets;
+  }
+  mesh.weights.assign(num_targets, 0.0);
 }
 
 void add_merc(const tfrag3::Level& level,
               const std::map<std::string, level_tools::ArtData>& art_data,
               const tfrag3::MercModel& mmodel,
               tinygltf::Model& model,
-              std::unordered_map<int, int>& tex_image_map) {
+              std::unordered_map<int, int>& tex_image_map,
+              const std::unordered_map<int, int>& anim_slot_to_base_tex) {
   const auto& mverts = level.merc_data.vertices;
 
   // create position and uv buffers
@@ -1225,6 +1359,34 @@ void add_merc(const tfrag3::Level& level,
     }
     ASSERT(skin.skeleton + n_bones == (int)model.nodes.size());
     skin.inverseBindMatrices = make_inv_matrix_bind_poses(game_bones, model);
+
+    for (int i = 0; i < n_bones; i++) {
+      if (game_bones[i].parent_idx < 0) {
+        model.scenes.at(0).nodes.push_back(skin.skeleton + i);
+      }
+    }
+  }
+
+  u32 num_blend_targets = 0;
+  for (const auto& effect : mmodel.effects) {
+    bool skip_next = false;
+    for (u32 v : effect.mod.blerc.int_data) {
+      if (skip_next) {
+        skip_next = false;
+        continue;
+      }
+      if (v == tfrag3::Blerc::kTargetIdxTerminator) {
+        skip_next = true;
+      } else {
+        num_blend_targets = std::max(num_blend_targets, v + 1);
+      }
+    }
+  }
+
+  // when we have animated blend targets, blender adds an extra empty during import,
+  // rename it to not conflict with the actual model
+  if (num_blend_targets > 0) {
+    model.nodes[node_idx].name = mmodel.name + "_blerc";
   }
 
   if (art != art_data.end() && !art->second.anims.empty() && node.skin >= 0 &&
@@ -1232,7 +1394,7 @@ void add_merc(const tfrag3::Level& level,
     const auto& skin = model.skins[node.skin];
     for (const auto& ja : art->second.anims) {
       auto uncompressed = decompress_anim(ja);
-      add_animation_to_gltf(uncompressed, skin, model);
+      add_animation_to_gltf(uncompressed, skin, model, node_idx, (int)num_blend_targets);
     }
   }
 
@@ -1241,8 +1403,13 @@ void add_merc(const tfrag3::Level& level,
     for (size_t draw_idx = 0; draw_idx < effect.all_draws.size(); draw_idx++) {
       const auto& draw = effect.all_draws[draw_idx];
       auto& prim = mesh.primitives.emplace_back();
-      prim.material =
-          add_material_for_tex(level, model, draw.tree_tex_id, tex_image_map, draw.mode);
+      // resolve texture animation draws to their base texture
+      int tex_id = draw.tree_tex_id;
+      if (tex_id < 0) {
+        const auto it = anim_slot_to_base_tex.find(-(tex_id + 1));
+        tex_id = it != anim_slot_to_base_tex.end() ? it->second : draw.tree_tex_id;
+      }
+      prim.material = add_material_for_tex(level, model, tex_id, tex_image_map, draw.mode);
       prim.indices =
           make_index_buffer_accessor(model, draw_to_start[effect_idx][draw_idx],
                                      draw_to_count[effect_idx][draw_idx], index_buffer_view);
@@ -1254,6 +1421,8 @@ void add_merc(const tfrag3::Level& level,
       prim.mode = TINYGLTF_MODE_TRIANGLES;
     }
   }
+
+  add_blerc_targets(level, mmodel, mesh, model);
 }
 }  // namespace
 
@@ -1299,9 +1468,20 @@ void save_level_background_as_gltf(const tfrag3::Level& level, const fs::path& g
                             true);  // write binary
 }
 
-void save_level_foreground_as_gltf(const tfrag3::Level& level,
-                                   const std::map<std::string, level_tools::ArtData>& art_data,
-                                   const fs::path& glb_path) {
+void save_level_foreground_as_gltf(
+    const tfrag3::Level& level,
+    const std::map<std::string, level_tools::ArtData>& art_data,
+    const fs::path& glb_path,
+    const std::unordered_map<std::string, u32>& animated_tex_output_to_anim_slot) {
+  // map animated texture slots back to the base texture
+  std::unordered_map<int, int> anim_slot_to_base_tex;
+  for (int i = 0; i < (int)level.textures.size(); i++) {
+    const auto it = animated_tex_output_to_anim_slot.find(level.textures[i].debug_name);
+    if (it != animated_tex_output_to_anim_slot.end()) {
+      anim_slot_to_base_tex[(int)it->second] = i;
+    }
+  }
+
   for (size_t model_idx = 0; model_idx < level.merc_data.models.size(); model_idx++) {
     const auto& mmodel = level.merc_data.models[model_idx];
 
@@ -1321,7 +1501,7 @@ void save_level_foreground_as_gltf(const tfrag3::Level& level,
 
     std::unordered_map<int, int> tex_image_map;
 
-    add_merc(level, art_data, mmodel, model, tex_image_map);
+    add_merc(level, art_data, mmodel, model, tex_image_map, anim_slot_to_base_tex);
 
     model.asset.generator = "opengoal";
 

decompiler/level_extractor/fr3_to_gltf.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <map>
+#include <unordered_map>
 
 #include "common/custom_data/Tfrag3Data.h"
 #include "common/util/FileUtil.h"
@@ -11,6 +12,8 @@
  * Export the background geometry (tie, tfrag, shrub) to a GLTF binary format (.glb) file.
  */
 void save_level_background_as_gltf(const tfrag3::Level& level, const fs::path& glb_file);
-void save_level_foreground_as_gltf(const tfrag3::Level& level,
-                                   const std::map<std::string, level_tools::ArtData>& art_data,
-                                   const fs::path& glb_path);
+void save_level_foreground_as_gltf(
+    const tfrag3::Level& level,
+    const std::map<std::string, level_tools::ArtData>& art_data,
+    const fs::path& glb_path,
+    const std::unordered_map<std::string, u32>& animated_tex_output_to_anim_slot);