[merc2] support vertex updates, use this for blerc in jak 1 and jak 2 (#2179)

This PR adds a feature to merc2 to update vertices. This will be needed
to efficient do effects like blerc/ripple/texture scroll. It's enabled
for blerc in jak 1 and jak 2, but with a few disclaimers:
- currently we still use the mips2c blerc implementation, which is slow
and has some "jittering" because of integer precision. When porting to
PC, there was an additional synchronization problem because blerc
overwrites the merc data as its being read by the renderers. I _think_
this wasn't an issue on PS2 because the blerc dma is higher priority
than the VIF1 DMA, but I'm not certain. Either way, I had to add a mutex
for this on PC to avoid very slight flickering/gaps. This isn't ideal
for performance, but still beats generic by a significant amount in
every place I tested. If you see merc taking 2ms to draw, it is likely
because it is stuck waiting on blerc to finish. This will go away once
blerc itself is ported to C++.
- in jak 1, we end up using generic in some cases where we could use
merc. In particular maia in village3 hut. This will be fixed later once
we can use merc in more places. I don't want to mess with the
merc/generic selection logic when we're hopefully going to get rid of it
soon.
- There is no support for ripple or texture scroll. These use generic on
jak 1, and remain broken on jak 2.
- Like with `emerc`, jak 1 has a toggle to go back to the old behavior
`*blerc-hack*`.
- In most cases, toggling this causes no visual differences. One
exception is Gol's teeth. I believe this is caused by texture coordinate
rounding issues, where generic has an additional float -> int -> float
compared to PC merc. It is very hard to notice so I'm not going to worry
about it.

decompiler/level_extractor/extract_merc.cpp

@@ -94,6 +94,12 @@ struct MercUnpackedVtx {
 
   u16 dst0;
   u16 dst1;
+
+  bool can_be_modified = false;
+  int idx_in_combined_lump4 = -1;  // divided by 3
+
+  int flump4 = -1;
+  int frag = -1;
 };
 
 /*!
@@ -107,6 +113,7 @@ struct ConvertedMercEffect {
   // draws from all fragments.
   std::vector<MercDraw> draws;
   std::vector<MercUnpackedVtx> vertices;
+  std::vector<u32> verts_per_frag;
   bool has_envmap = false;
   DrawMode envmap_mode;
   u32 envmap_texture;
@@ -216,7 +223,9 @@ void update_mode_from_alpha1(GsAlpha reg, DrawMode& mode) {
 /*!
  * Convert merc shader to PC draw mode
  */
-DrawMode process_draw_mode(const MercShader& info, bool enable_alpha_test) {
+DrawMode process_draw_mode(const MercShader& info,
+                           bool enable_alpha_test,
+                           bool enable_alpha_blend) {
   DrawMode mode;
   /*
    *       (new 'static 'gs-test
@@ -237,7 +246,7 @@ DrawMode process_draw_mode(const MercShader& info, bool enable_alpha_test) {
   mode.set_depth_test(GsTest::ZTest::GEQUAL);
 
   // check these
-  mode.disable_ab();
+  mode.set_ab(enable_alpha_blend);
   mode.set_alpha_blend(DrawMode::AlphaBlend::SRC_DST_SRC_DST);
   mode.set_tcc(info.tex0.tcc());
   mode.set_decal(info.tex0.tfx() == GsTex0::TextureFunction::DECAL);
@@ -306,7 +315,10 @@ void handle_frag(const std::string& debug_name,
                  const MercFragmentControl& frag_ctrl,
                  const MercState& state,
                  std::vector<MercUnpackedVtx>& effect_vertices,
-                 MercMemory& memory) {
+                 MercMemory& memory,
+                 bool can_be_modified,
+                 int base_lump4,
+                 int frag_idx) {
   (void)frag_ctrl;
   (void)debug_name;
   // lg::print("handling frag: {}\n", debug_name);
@@ -330,6 +342,10 @@ void handle_frag(const std::string& debug_name,
   for (size_t i = 0; i < mat123_cnt; i++) {
     u32 current_vtx_idx = effect_vertices.size();  // idx in effect vertex list.
     auto& vtx = effect_vertices.emplace_back();
+    vtx.can_be_modified = can_be_modified;
+    vtx.idx_in_combined_lump4 = lump_ptr / 3 + base_lump4;
+    vtx.frag = frag_idx;
+    vtx.flump4 = lump_ptr / 3;
 
     if (i < mat1_cnt) {
       vtx.kind = 1;  // 1 matrix
@@ -712,7 +728,7 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
   result.effect_idx = effect_idx;
   if (input_effect.extra_info.shader) {
     result.has_envmap = true;
-    result.envmap_mode = process_draw_mode(*input_effect.extra_info.shader, false);
+    result.envmap_mode = process_draw_mode(*input_effect.extra_info.shader, false, false);
     result.envmap_mode.set_ab(true);
     u32 new_tex = remap_texture(input_effect.extra_info.shader->original_tex, map);
     ASSERT(result.envmap_mode.get_tcc_enable());
@@ -763,6 +779,12 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
     result.envmap_mode = mode;
     result.envmap_mode.set_ab(true);
   }
+
+  bool use_alpha_blend = false;
+  if (version == GameVersion::Jak2) {
+    use_alpha_blend = input_effect.texture_index == 4;  // water
+  }
+
   // full reset of state per effect.
   // we have no idea what the previous effect draw will be - it might be given to
   // mercneric.
@@ -771,6 +793,8 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
   MercMemory merc_memories[2];   // double buffered output
   int memory_buffer_toggle = 0;  // which output we're in
 
+  int combined_lump4_addr = 0;
+
   for (size_t fi = 0; fi < input_effect.frag_ctrl.size(); fi++) {
     const auto& frag = input_effect.frag_geo[fi];
     const auto& frag_ctrl = input_effect.frag_ctrl[fi];
@@ -792,9 +816,20 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
     // run the frag.
     // this will add vertices to the per-effect vertex lists and also update the merc memory
     // to point to these.
+    bool can_be_modified = false;
+    if (fi < input_effect.blend_ctrl.size()) {
+      can_be_modified = input_effect.blend_ctrl.at(fi).blend_vtx_count > 0;
+    }
+
+    if (input_effect.effect_bits & kRippleEffectBit) {
+      can_be_modified = true;
+    }
 
     handle_frag(debug_name, ctrl_header, frag, frag_ctrl, merc_state, result.vertices,
-                merc_memories[memory_buffer_toggle]);
+                merc_memories[memory_buffer_toggle], can_be_modified, combined_lump4_addr, fi);
+    u32 vert_count = frag.lump4_unpacked.size() / 3;
+    combined_lump4_addr += vert_count;
+    result.verts_per_frag.push_back(vert_count);
 
     // we'll add draws after this draw, but wait to actually populate the index lists until
     // we've processed all the vertices.
@@ -819,7 +854,8 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
     for (size_t i = 0; i < frag.fp_header.shader_cnt; i++) {
       const auto& shader = frag.shaders.at(i);
       // update merc state from shader (will hold over to next fragment, if needed)
-      merc_state.merc_draw_mode.mode = process_draw_mode(shader, result.has_envmap);
+      merc_state.merc_draw_mode.mode =
+          process_draw_mode(shader, result.has_envmap, use_alpha_blend);
       if (!merc_state.merc_draw_mode.mode.get_tcc_enable()) {
         ASSERT(false);
       }
@@ -956,6 +992,193 @@ tfrag3::MercVertex convert_vertex(const MercUnpackedVtx& vtx, float xyz_scale) {
   return out;
 }
 
+struct VertexSourceInfo {
+  int combined_lump4;
+  int frag;
+  int flump4;
+};
+
+void create_modifiable_vertex_data(
+    const std::vector<bool>& vtx_mod_flag,
+    const std::vector<VertexSourceInfo>& vtx_srcs,
+    tfrag3::MercModelGroup& out,
+    size_t first_out_vertex,
+    size_t first_out_model,
+    const std::vector<std::vector<ConvertedMercEffect>>& all_effects) {
+  ASSERT(vtx_mod_flag.size() + first_out_vertex == out.vertices.size());
+
+  // we need to be able to modify some vertices at runtime.
+  // this can be detected vertex-by-vertex
+  // the plan is to find MercEffects that contain modifiable vertices, and provide an alternate way
+  // to draw them. In the case where no vertices should be modified, we can fall back to the normal
+  // merc drawing path.
+
+  // In this modifiable draw path, there will be a list of "fixed draws", which draw vertices that
+  // cannot be modified. This set is known at build-time.
+  // The "mod draws" will draw the modifiable vertices. These use the normal index buffer, but
+  // index into a per-effect modifiable vertex buffer.
+
+  //  std::vector<tfrag3::MercDraw> fixed_draws, mod_draws;
+
+  // some stats
+  int num_tris = 0;  // all triangles
+  int mod_tris = 0;  // triangles in mod draws
+
+  // loop over models added from this art-group
+  for (size_t mi = first_out_model; mi < out.models.size(); mi++) {
+    auto& model = out.models.at(mi);
+    // loop over "effects" within this model. the pc format merges all fragments in an effect
+    // together.
+
+    for (size_t ei = 0; ei < model.effects.size(); ei++) {
+      auto& effect = model.effects[ei];
+
+      std::vector<std::vector<u32>> inds_per_mod_draw;
+
+      for (const auto& draw : effect.all_draws) {
+        num_tris += draw.num_triangles;
+
+        // first check to see what's in this draw
+        bool found_mod = false;
+        bool found_fixed = false;
+        for (int i = 0; i < (int)draw.index_count; i++) {
+          u32 idx = out.indices.at(draw.first_index + i);
+          if (idx == UINT32_MAX) {
+            continue;
+          }
+          ASSERT(idx >= first_out_vertex);
+          if (vtx_mod_flag.at(idx - first_out_vertex)) {
+            found_mod = true;
+          } else {
+            found_fixed = true;
+          }
+        }
+
+        if (!found_fixed && !found_mod) {
+          // nothing found at all, bad
+          ASSERT_NOT_REACHED();
+        } else if (found_fixed && !found_mod) {
+          // only fixed. can just copy the fixed draw
+          effect.mod.fix_draw.push_back(draw);
+        } else if (found_mod && !found_fixed) {
+          // only mod
+          effect.mod.mod_draw.push_back(draw);
+          auto& inds_out = inds_per_mod_draw.emplace_back();
+          for (u32 i = 0; i < draw.index_count; i++) {
+            inds_out.push_back(out.indices.at(draw.first_index + i));
+          }
+          mod_tris += draw.num_triangles;
+        } else {
+          // it's a mix...
+          std::vector<std::vector<u32>> strips;
+          strips.emplace_back();
+          for (u32 i = 0; i < draw.index_count; i++) {
+            u32 val = out.indices.at(draw.first_index + i);
+            if (val == UINT32_MAX) {
+              if (!strips.back().empty()) {
+                strips.emplace_back();
+              }
+            } else {
+              strips.back().push_back(val);
+            }
+          }
+
+          tfrag3::MercDraw mod = draw;
+          tfrag3::MercDraw fix = draw;
+          std::vector<u32> mod_ind, fix_ind;
+          for (auto& strip : strips) {
+            bool strip_has_mod = false;
+            for (auto ind : strip) {
+              if (vtx_mod_flag.at(ind - first_out_vertex)) {
+                strip_has_mod = true;
+                break;
+              }
+            }
+            if (strip_has_mod) {
+              mod_ind.insert(mod_ind.end(), strip.begin(), strip.end());
+              mod_ind.push_back(UINT32_MAX);
+            } else {
+              fix_ind.insert(fix_ind.end(), strip.begin(), strip.end());
+              fix_ind.push_back(UINT32_MAX);
+            }
+          }
+
+          mod.index_count = mod_ind.size();
+          inds_per_mod_draw.push_back(mod_ind);
+          fix.first_index = out.indices.size();
+          fix.index_count = fix_ind.size();
+          out.indices.insert(out.indices.end(), fix_ind.begin(), fix_ind.end());
+
+          effect.mod.mod_draw.push_back(mod);
+          effect.mod.fix_draw.push_back(fix);
+        }
+      }  // for draw
+
+      // if there are no modifiable draws, we can't possible modify anything, so not worth
+      // storing the fixed draws
+      if (effect.mod.mod_draw.empty()) {
+        effect.mod.fix_draw.clear();
+      } else {
+        effect.has_mod_draw = true;
+        // need to set up the vertex buffer for the modifiable draws
+        // map of original vertex indices to mod buffer index
+        std::unordered_map<u32, u32> vtx_to_mod_vtx;
+        for (size_t mdi = 0; mdi < effect.mod.mod_draw.size(); mdi++) {
+          auto& draw = effect.mod.mod_draw[mdi];
+          auto& orig_inds = inds_per_mod_draw.at(mdi);
+          u32 new_first_index = out.indices.size();
+          for (auto vidx : orig_inds) {
+            if (vidx == UINT32_MAX) {
+              out.indices.push_back(UINT32_MAX);
+              continue;  // strip restart
+            }
+            const auto& existing = vtx_to_mod_vtx.find(vidx);
+            if (existing == vtx_to_mod_vtx.end()) {
+              // add vertex to mod buffer
+              auto idx = effect.mod.vertices.size();
+              vtx_to_mod_vtx[vidx] = idx;
+              effect.mod.vertices.push_back(out.vertices.at(vidx));
+              auto src = vtx_srcs.at(vidx - first_out_vertex);
+              ASSERT(src.combined_lump4 < UINT16_MAX);
+              effect.mod.vertex_lump4_addr.push_back(src.combined_lump4);
+              u32 frag_idx = src.frag;
+              if (frag_idx >= effect.mod.fragment_mask.size()) {
+                effect.mod.fragment_mask.resize(frag_idx + 1);
+              }
+              effect.mod.fragment_mask[frag_idx] = true;
+              out.indices.push_back(idx);
+            } else {
+              out.indices.push_back(existing->second);
+            }
+          }
+          draw.first_index = new_first_index;
+        }
+
+        // splice out masked fragments, the renderer won't index them
+        const auto& frag_counts = all_effects.at(mi - first_out_model).at(ei).verts_per_frag;
+        std::unordered_map<u32, u32> old_to_new;
+        u32 old_idx = 0;
+        u32 new_idx = 0;
+        for (size_t fi = 0; fi < effect.mod.fragment_mask.size(); fi++) {
+          if (effect.mod.fragment_mask[fi]) {
+            for (u32 vi = 0; vi < frag_counts.at(fi); vi++) {
+              old_to_new[old_idx] = new_idx;
+              old_idx++;
+              new_idx++;
+            }
+          } else {
+            old_idx += frag_counts.at(fi);
+          }
+        }
+        effect.mod.expect_vidx_end = new_idx;
+        for (auto& v : effect.mod.vertex_lump4_addr) {
+          v = old_to_new.at(v);
+        }
+      }
+    }
+  }
+}
+
 /*!
  * Top-level merc extraction
  */
@@ -990,9 +1213,12 @@ void extract_merc(const ObjectFileData& ag_data,
     }
   }
 
+  size_t first_out_vertex = out.merc_data.vertices.size();
   // convert to PC format
   // first pass, before merging indices
   u32 first_model = out.merc_data.models.size();
+  std::vector<bool> vertex_modify_flags;
+  std::vector<VertexSourceInfo> vertex_srcs;
   std::vector<std::vector<std::vector<std::vector<u32>>>> indices_temp;  // ctrl, effect, draw, vtx
   for (size_t ci = 0; ci < ctrls.size(); ci++) {
     indices_temp.emplace_back();
@@ -1002,6 +1228,8 @@ void extract_merc(const ObjectFileData& ag_data,
     pc_ctrl.name = ctrl.name;
     pc_ctrl.max_draws = 0;
     pc_ctrl.max_bones = 0;
+    pc_ctrl.st_vif_add = ctrl.header.st_vif_add;
+    pc_ctrl.xyz_scale = ctrl.header.xyz_scale;
 
     for (size_t ei = 0; ei < ctrls[ci].effects.size(); ei++) {
       indices_temp[ci].emplace_back();
@@ -1013,6 +1241,8 @@ void extract_merc(const ObjectFileData& ag_data,
       u32 first_vertex = out.merc_data.vertices.size();
       for (auto& vtx : effect.vertices) {
         auto cvtx = convert_vertex(vtx, ctrl.header.xyz_scale);
+        vertex_modify_flags.push_back(vtx.can_be_modified);
+        vertex_srcs.push_back({vtx.idx_in_combined_lump4, vtx.frag, vtx.flump4});
         out.merc_data.vertices.push_back(cvtx);
         for (int i = 0; i < 3; i++) {
           pc_ctrl.max_bones = std::max(pc_ctrl.max_bones, (u32)cvtx.mats[i]);
@@ -1023,22 +1253,21 @@ void extract_merc(const ObjectFileData& ag_data,
       std::map<u64, u64> draw_mode_dedup;
 
       for (auto& draw : effect.draws) {
-        pc_ctrl.max_draws++;
         indices_temp[ci][ei].emplace_back();
         // find draw to add to, or create a new one
         const auto& existing = draw_mode_dedup.find(draw.state.merc_draw_mode.as_u64());
         tfrag3::MercDraw* pc_draw = nullptr;
         u64 pc_draw_idx = -1;
         if (existing == draw_mode_dedup.end()) {
-          pc_draw_idx = pc_effect.draws.size();
+          pc_draw_idx = pc_effect.all_draws.size();
           draw_mode_dedup[draw.state.merc_draw_mode.as_u64()] = pc_draw_idx;
-          pc_draw = &pc_effect.draws.emplace_back();
+          pc_draw = &pc_effect.all_draws.emplace_back();
           pc_draw->mode = draw.state.merc_draw_mode.mode;
           pc_draw->tree_tex_id = find_or_add_texture_to_level(
               out, tex_db, ctrl.name, draw.state.merc_draw_mode.pc_combo_tex_id);
         } else {
           pc_draw_idx = existing->second;
-          pc_draw = &pc_effect.draws.at(pc_draw_idx);
+          pc_draw = &pc_effect.all_draws.at(pc_draw_idx);
         }
 
         for (auto idx : draw.indices) {
@@ -1057,8 +1286,8 @@ void extract_merc(const ObjectFileData& ag_data,
     auto& pc_ctrl = out.merc_data.models.at(ci + first_model);
     for (size_t ei = 0; ei < ctrls[ci].effects.size(); ei++) {
       auto& pc_effect = pc_ctrl.effects.at(ei);
-      for (size_t di = 0; di < pc_effect.draws.size(); di++) {
-        auto& pc_draw = pc_effect.draws.at(di);
+      for (size_t di = 0; di < pc_effect.all_draws.size(); di++) {
+        auto& pc_draw = pc_effect.all_draws.at(di);
         auto& inds = indices_temp[ci][ei][di];
         pc_draw.num_triangles = clean_up_vertex_indices(inds);
         pc_draw.first_index = out.merc_data.indices.size();
@@ -1067,5 +1296,20 @@ void extract_merc(const ObjectFileData& ag_data,
       }
     }
   }
+
+  create_modifiable_vertex_data(vertex_modify_flags, vertex_srcs, out.merc_data, first_out_vertex,
+                                first_model, all_effects);
+
+  // compute max draws
+  for (u32 mi = first_model; mi < out.merc_data.models.size(); mi++) {
+    auto& model = out.merc_data.models[mi];
+    model.max_draws = 0;
+    for (auto& e : model.effects) {
+      model.max_draws += e.all_draws.size();
+      if (e.has_mod_draw) {
+        model.max_draws += e.mod.mod_draw.size() + e.mod.fix_draw.size();
+      }
+    }
+  }
 }
 }  // namespace decompiler