custom levels: add support for packing models and textures into the FR3 file (#2936)

custom_levels/test-zone/test-zone.jsonc

@@ -34,7 +34,24 @@
    // The base actor id for your custom level. If you have multiple levels this should be unique!
    "base_id": 100,
 
-  "ambients": [],
+  // All art groups you want to use in your custom level. Will add their models and corresponding textures to the FR3 file.
+  // Note: You will still have to add them to your level's .gd file.
+  "art_groups": ["plat-ag"],
+
+  // Ambients you want to use in your custom level. Ambients can be used for various things like changing the music variation (flava),
+  // adding ambient sounds, level name hints, etc.
+  "ambients": [
+    {
+      "trans": [-21.6238, 20.0496, 17.1191, 10.0],
+      "bsphere": [-21.6238, 20.0496, 17.1191, 15.0],
+      "lump": {
+        "name": "test-ambient",
+        "type": "'hint",
+        "text-id": ["int32", 557], // text id for the "POWER CELL" text
+        "play-mode": "'notice"
+      }
+    }
+  ],
 
   "actors" : [
     {
@@ -70,6 +87,16 @@
       "lump": {
         "name":"test-eco"
       }
+    },
+    {
+      "trans": [-7.41, 3.5, 28.42], // translation
+      "etype": "plat",  // actor type
+      "game_task": 0, // associated game task (for powercells, etc)
+      "quat": [0, 0, 0, 1], // quaternion
+      "bsphere": [-7.41, 3.5, 28.42, 10], // bounding sphere
+      "lump": {
+        "name": "test-plat"
+      }
     }
   ]
 }

custom_levels/test-zone/testzone.gd

@@ -5,4 +5,5 @@
 ("TSZ.DGO"
  ("static-screen.o"
   "test-zone.go"
+  "plat-ag.go"
  ))

decompiler/level_extractor/extract_level.cpp

@@ -131,6 +131,27 @@ void extract_art_groups_from_level(const ObjectFileDB& db,
   }
 }
 
+std::vector<level_tools::TextureRemap> extract_tex_remap(const ObjectFileDB& db,
+                                                         const std::string& dgo_name) {
+  auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name), dgo_name);
+  if (!bsp_rec) {
+    lg::warn("Skipping extract for {} because the BSP file was not found", dgo_name);
+    return {};
+  }
+  std::string level_name = bsp_rec->name.substr(0, bsp_rec->name.length() - 4);
+
+  lg::info("Processing level {} ({})", dgo_name, level_name);
+  const auto& bsp_file = db.lookup_record(*bsp_rec);
+  bool ok = is_valid_bsp(bsp_file.linked_data);
+  ASSERT(ok);
+
+  level_tools::DrawStats draw_stats;
+  level_tools::BspHeader bsp_header;
+  bsp_header.read_from_file(bsp_file.linked_data, db.dts, &draw_stats, db.version());
+
+  return bsp_header.texture_remap_table;
+}
+
 std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB& db,
                                                               const TextureDB& tex_db,
                                                               const std::string& dgo_name,

decompiler/level_extractor/extract_level.h

@@ -2,6 +2,8 @@
 
 #include <vector>
 
+#include "common_formats.h"
+
 #include "common/math/Vector.h"
 
 #include "decompiler/ObjectFile/ObjectFileDB.h"
@@ -17,4 +19,13 @@ void extract_all_levels(const ObjectFileDB& db,
                         bool debug_dump_level,
                         bool extract_collision,
                         const fs::path& path);
+void add_all_textures_from_level(tfrag3::Level& lev,
+                                 const std::string& level_name,
+                                 const TextureDB& tex_db);
+tfrag3::Texture make_texture(u32 id,
+                             const TextureDB::TextureData& tex,
+                             const std::string& tpage_name,
+                             bool pool_load);
+std::vector<level_tools::TextureRemap> extract_tex_remap(const ObjectFileDB& db,
+                                                         const std::string& dgo_name);
 }  // namespace decompiler

goalc/CMakeLists.txt

@@ -56,7 +56,7 @@ add_library(compiler
         regalloc/Allocator_v2.cpp
         compiler/docs/DocTypes.cpp)
 
-target_link_libraries(compiler common Zydis tiny_gltf)
+target_link_libraries(compiler common Zydis tiny_gltf decomp)
 
 if (WIN32)
     target_link_libraries(compiler mman)

goalc/build_level/build_level.cpp

@@ -1,3 +1,5 @@
+#include "build_level.h"
+
 #include "common/custom_data/Tfrag3Data.h"
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
@@ -5,6 +7,7 @@
 #include "common/util/json_util.h"
 #include "common/util/string_util.h"
 
+#include "decompiler/level_extractor/extract_merc.h"
 #include "goalc/build_level/Entity.h"
 #include "goalc/build_level/FileInfo.h"
 #include "goalc/build_level/LevelFile.h"
@@ -36,6 +39,27 @@ std::vector<std::string> get_build_level_deps(const std::string& input_file) {
   return {level_json.at("gltf_file").get<std::string>()};
 }
 
+// Find all art groups the custom level needs in a list of object files,
+// skipping any that we already found in other dgos before
+std::vector<decompiler::ObjectFileRecord> find_art_groups(
+    std::vector<std::string>& processed_ags,
+    const std::vector<std::string>& custom_level_ag,
+    const std::vector<decompiler::ObjectFileRecord>& dgo_files) {
+  std::vector<decompiler::ObjectFileRecord> art_groups;
+  for (const auto& file : dgo_files) {
+    // skip any art groups we already added from other dgos
+    if (std::find(processed_ags.begin(), processed_ags.end(), file.name) != processed_ags.end()) {
+      continue;
+    }
+    if (std::find(custom_level_ag.begin(), custom_level_ag.end(), file.name) !=
+        custom_level_ag.end()) {
+      art_groups.push_back(file);
+      processed_ags.push_back(file.name);
+    }
+  }
+  return art_groups;
+}
+
 bool run_build_level(const std::string& input_file,
                      const std::string& bsp_output_file,
                      const std::string& output_prefix) {
@@ -117,6 +141,74 @@ bool run_build_level(const std::string& input_file,
   lg::print("Saving to {}\n", save_path.string());
   file_util::write_binary_file(save_path, result.data(), result.size());
 
+  // Add textures and models
+  // TODO remove hardcoded config settings
+  if (level_json.contains("art_groups") && !level_json.at("art_groups").empty()) {
+    decompiler::Config config;
+    try {
+      config = decompiler::read_config_file(
+          file_util::get_jak_project_dir() / "decompiler/config/jak1/jak1_config.jsonc", "ntsc_v1",
+          R"({"decompile_code": false, "find_functions": false, "levels_extract": true, "allowed_objects": []})");
+    } catch (const std::exception& e) {
+      lg::error("Failed to parse config: {}", e.what());
+      return false;
+    }
+
+    fs::path in_folder;
+    lg::info("Looking for ISO path...");
+    for (const auto& entry :
+         fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) {
+      if (entry.is_directory() &&
+          entry.path().filename().string().find("jak1") != std::string::npos) {
+        lg::info("Found ISO path: {}", entry.path().string());
+        in_folder = entry.path();
+      }
+    }
+    if (!fs::exists(in_folder)) {
+      lg::error("Could not find ISO path!");
+      return false;
+    }
+    std::vector<fs::path> dgos, objs;
+    for (const auto& dgo_name : config.dgo_names) {
+      dgos.push_back(in_folder / dgo_name);
+    }
+
+    for (const auto& obj_name : config.object_file_names) {
+      objs.push_back(in_folder / obj_name);
+    }
+
+    decompiler::ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, {},
+                                config);
+
+    // need to process link data for tpages
+    db.process_link_data(config);
+
+    decompiler::TextureDB tex_db;
+    auto textures_out = file_util::get_jak_project_dir() / "decompiler_out/jak1/textures";
+    file_util::create_dir_if_needed(textures_out);
+    db.process_tpages(tex_db, textures_out, config);
+
+    std::vector<std::string> processed_art_groups;
+
+    // find all art groups used by the custom level in other dgos
+    for (auto& dgo : config.dgo_names) {
+      // remove "DGO/" prefix
+      const auto& dgo_name = dgo.substr(4);
+      const auto& files = db.obj_files_by_dgo.at(dgo_name);
+      auto art_groups = find_art_groups(
+          processed_art_groups, level_json.at("art_groups").get<std::vector<std::string>>(), files);
+      auto tex_remap = decompiler::extract_tex_remap(db, dgo_name);
+      for (const auto& ag : art_groups) {
+        if (ag.name.length() > 3 && !ag.name.compare(ag.name.length() - 3, 3, "-ag")) {
+          const auto& ag_file = db.lookup_record(ag);
+          lg::print("custom level: extracting art group {}\n", ag_file.name_in_dgo);
+          decompiler::extract_merc(ag_file, tex_db, db.dts, tex_remap, pc_level, false,
+                                   db.version());
+        }
+      }
+    }
+  }
+
   // Save the PC level
   save_pc_data(file.nickname, pc_level,
                file_util::get_jak_project_dir() / "out" / output_prefix / "fr3");

goalc/build_level/build_level.h

@@ -3,6 +3,8 @@
 #include <string>
 #include <vector>
 
+#include "decompiler/level_extractor/extract_level.h"
+
 bool run_build_level(const std::string& input_file,
                      const std::string& bsp_output_file,
                      const std::string& output_prefix);