From 8344ac696304ee5f61e5c9fb6caa85ce3c18cc6d Mon Sep 17 00:00:00 2001 From: ManDude <7569514+ManDude@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:12:57 +0100 Subject: [PATCH] use only the real level name in the Loader (#3495) This fixes issues with certain Jak 3 levels not rendering because there is a mismatch between the DGO name, nickname and real level name (bsp name). FR3s use a different filename, so you can delete the ones you have after this is merged. This affects custom levels, but I don't have that toolchain set up so someone else will have to test that. --- .vs/launch.vs.json | 14 ++++++++++++++ common/custom_data/TFrag3Data.cpp | 5 +++-- decompiler/config/jak1/ntsc_v1/hacks.jsonc | 2 +- decompiler/config/jak2/ntsc_v1/hacks.jsonc | 2 +- decompiler/level_extractor/BspHeader.cpp | 1 + decompiler/level_extractor/BspHeader.h | 1 + decompiler/level_extractor/extract_level.cpp | 16 +++++++--------- .../background/background_common.h | 6 +++--- game/graphics/opengl_renderer/loader/Loader.cpp | 3 +-- goal_src/jak1/engine/gfx/tfrag/tfrag.gc | 8 +++++--- goal_src/jak1/engine/level/level.gc | 4 ++-- .../jak2/engine/gfx/background/background.gc | 10 +++++----- goal_src/jak2/engine/level/level.gc | 4 ++-- .../jak3/engine/gfx/background/background.gc | 10 +++++----- goal_src/jak3/engine/level/level.gc | 5 ++--- goalc/build_level/jak1/build_level.cpp | 2 +- goalc/build_level/jak2/build_level.cpp | 2 +- 17 files changed, 55 insertions(+), 40 deletions(-) diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json index eec32c8868..e884e07347 100644 --- a/.vs/launch.vs.json +++ b/.vs/launch.vs.json @@ -243,6 +243,20 @@ "ntsc_v1" ] }, + { + "type": "default", + "project": "CMakeLists.txt", + "projectTarget": "decompiler.exe (bin\\decompiler.exe)", + "name": "Decompiler - Jak 3 - Extract", + "args": [ + "${workspaceRoot}/decompiler/config/jak3/jak3_config.jsonc", + "${workspaceRoot}/iso_data", + "${workspaceRoot}/decompiler_out", + "--version", + "ntsc_v1", + "--config-override \"{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, \\\"allowed_objects\\\": []}\"" + ] + }, { "type": "default", "project": "CMakeLists.txt", diff --git a/common/custom_data/TFrag3Data.cpp b/common/custom_data/TFrag3Data.cpp index 5b66d841fb..f66b821c3b 100644 --- a/common/custom_data/TFrag3Data.cpp +++ b/common/custom_data/TFrag3Data.cpp @@ -9,6 +9,7 @@ #include "third-party/sse2neon/sse2neon.h" #endif +#include "common/log/log.h" #include "common/util/Assert.h" namespace tfrag3 { @@ -795,8 +796,8 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) { for (const auto& x : known_categories) { if (x.second) { - fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024, - 100.f * (float)x.second / uncompressed_data_size); + lg::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024, + 100.f * (float)x.second / uncompressed_data_size); } } } diff --git a/decompiler/config/jak1/ntsc_v1/hacks.jsonc b/decompiler/config/jak1/ntsc_v1/hacks.jsonc index c9b83b339d..7717155354 100644 --- a/decompiler/config/jak1/ntsc_v1/hacks.jsonc +++ b/decompiler/config/jak1/ntsc_v1/hacks.jsonc @@ -634,7 +634,7 @@ // there are some missing textures. I don't know what the game actually does here. // the format for entries is [level, tpage, index] - "missing_textures": [["finalboss-vis", 1419, 3]], + "missing_textures": [["finalboss", 1419, 3]], // some object files have garbage pad data at the end which makes the decompiler // assume they must be different files, such as the art group for orb-cache-top. diff --git a/decompiler/config/jak2/ntsc_v1/hacks.jsonc b/decompiler/config/jak2/ntsc_v1/hacks.jsonc index b28403c107..cd609bbfcd 100644 --- a/decompiler/config/jak2/ntsc_v1/hacks.jsonc +++ b/decompiler/config/jak2/ntsc_v1/hacks.jsonc @@ -769,7 +769,7 @@ // there are some missing textures. I don't know what the game actually does here. // the format for entries is [level, tpage, index] "missing_textures": [ - ["vinroom-vis", 0, 0], + ["vinroom", 0, 0], ["ctyfence", 0, 0] ], diff --git a/decompiler/level_extractor/BspHeader.cpp b/decompiler/level_extractor/BspHeader.cpp index d195a4c7ce..fe44d73060 100644 --- a/decompiler/level_extractor/BspHeader.cpp +++ b/decompiler/level_extractor/BspHeader.cpp @@ -2045,6 +2045,7 @@ void BspHeader::read_from_file(const decompiler::LinkedObjectFile& file, file_info.read_from_file(get_and_check_ref_to_basic(ref, "info", "file-info", dts), dts); bsphere.read_from_file(get_field_ref(ref, "bsphere", dts)); + name = read_symbol_field(ref, "name", dts); texture_remap_table.clear(); s32 tex_remap_len = read_plain_data_field(ref, "texture-remap-table-len", dts); diff --git a/decompiler/level_extractor/BspHeader.h b/decompiler/level_extractor/BspHeader.h index 13d24bffed..26e497ea70 100644 --- a/decompiler/level_extractor/BspHeader.h +++ b/decompiler/level_extractor/BspHeader.h @@ -885,6 +885,7 @@ struct BspHeader { // (unk-zero-0 basic :offset-assert 68) // // (name symbol :offset-assert 72) + std::string name; // (nickname symbol :offset-assert 76) // (vis-info level-vis-info 8 :offset-assert 80) // (actors drawable-inline-array-actor :offset-assert 112) diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index f9a50ea247..0572f03220 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -168,9 +168,8 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, lg::warn("Skipping extract for {} because the BSP file was not found", dgo_name); return {}; } - std::string level_name = bsp_rec->name; - lg::info("Processing level {} ({})", dgo_name, level_name); + lg::info("Processing {}...", dgo_name); const auto& bsp_file = db.lookup_record(*bsp_rec); bool ok = is_valid_bsp(bsp_file.linked_data); ASSERT(ok); @@ -207,7 +206,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, auto as_tfrag_tree = dynamic_cast(draw_tree.get()); ASSERT(as_tfrag_tree); std::vector> expected_missing_textures; - auto it = hacks.missing_textures_by_level.find(level_name); + auto it = hacks.missing_textures_by_level.find(bsp_header.name); if (it != hacks.missing_textures_by_level.end()) { expected_missing_textures = it->second; } @@ -219,7 +218,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, } extract_tfrag(as_tfrag_tree, fmt::format("{}-{}", dgo_name, i++), bsp_header.texture_remap_table, tex_db, expected_missing_textures, level_data, - false, level_name, atest_disable_flag); + false, bsp_header.name, atest_disable_flag); } else if (draw_tree->my_type() == "drawable-tree-instance-tie") { auto as_tie_tree = dynamic_cast(draw_tree.get()); ASSERT(as_tie_tree); @@ -250,7 +249,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, extract_collide_frags(bsp_header.collide_hash, all_ties, config, fmt::format("{}-{}-collide", dgo_name, i++), db.dts, level_data); } - level_data.level_name = level_name; + level_data.level_name = bsp_header.name; return bsp_header; } @@ -358,13 +357,12 @@ void extract_from_level(const ObjectFileDB& db, level_data.serialize(ser); auto compressed = compression::compress_zstd(ser.get_save_result().first, ser.get_save_result().second); - lg::info("stats for {}", dgo_name); + lg::info("stats for {}", level_data.level_name); print_memory_usage(level_data, ser.get_save_result().second); lg::info("compressed: {} -> {} ({:.2f}%)", ser.get_save_result().second, compressed.size(), 100.f * compressed.size() / ser.get_save_result().second); - file_util::write_binary_file( - output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)), - compressed.data(), compressed.size()); + file_util::write_binary_file(output_folder / fmt::format("{}.fr3", level_data.level_name), + compressed.data(), compressed.size()); if (config.rip_levels) { auto back_file_path = file_util::get_jak_project_dir() / "glb_out" / diff --git a/game/graphics/opengl_renderer/background/background_common.h b/game/graphics/opengl_renderer/background/background_common.h index 2fbb1ba69f..0701853bbc 100644 --- a/game/graphics/opengl_renderer/background/background_common.h +++ b/game/graphics/opengl_renderer/background/background_common.h @@ -19,9 +19,9 @@ struct GoalBackgroundCameraData { // the GOAL code assumes this memory layout. struct TfragPcPortData { GoalBackgroundCameraData camera; - char level_name[16]; + char level_name[32]; }; -static_assert(sizeof(TfragPcPortData) == 16 * 24); +static_assert(sizeof(TfragPcPortData) == 16 * 25); // inputs to background renderers. struct TfragRenderSettings { @@ -119,4 +119,4 @@ u32 make_index_list_from_vis_and_proto_string(std::pair* group_out, const std::vector& vis_data, const std::vector& proto_vis_data, const u32* idx_in, - u32* num_tris_out); \ No newline at end of file + u32* num_tris_out); diff --git a/game/graphics/opengl_renderer/loader/Loader.cpp b/game/graphics/opengl_renderer/loader/Loader.cpp index 8ea45231bf..524c225336 100644 --- a/game/graphics/opengl_renderer/loader/Loader.cpp +++ b/game/graphics/opengl_renderer/loader/Loader.cpp @@ -188,8 +188,7 @@ void Loader::loader_thread() { // load the fr3 file prof().begin_event("read-file"); Timer disk_timer; - auto data = - file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", uppercase_string(lev))); + auto data = file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", lev)); double disk_load_time = disk_timer.getSeconds(); prof().end_event(); diff --git a/goal_src/jak1/engine/gfx/tfrag/tfrag.gc b/goal_src/jak1/engine/gfx/tfrag/tfrag.gc index 494d5f2d57..37df66d092 100644 --- a/goal_src/jak1/engine/gfx/tfrag/tfrag.gc +++ b/goal_src/jak1/engine/gfx/tfrag/tfrag.gc @@ -343,7 +343,7 @@ (defun add-pc-tfrag3-data ((dma-buf dma-buffer) (lev level)) "Add PC-port specific tfrag data" (let ((packet (the-as dma-packet (-> dma-buf base)))) - (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 24)) + (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 25)) (set! (-> packet vif0) (new 'static 'vif-tag)) (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port))) (set! (-> dma-buf base) (the pointer (&+ packet 16))) @@ -379,9 +379,11 @@ (set! (-> data-ptr 20) (-> *math-camera* perspective vector 1 quad)) (set! (-> data-ptr 21) (-> *math-camera* perspective vector 2 quad)) (set! (-> data-ptr 22) (-> *math-camera* perspective vector 3 quad)) - (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (-> lev nickname))) + (if (<= 32 (length (symbol->string (bsp-name lev)))) + (format #t "level name ~a is too long!! must be shorter than 32 characters~%" (bsp-name lev))) + (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (bsp-name lev))) ) - (&+! (-> dma-buf base) (* 16 24)) + (&+! (-> dma-buf base) (* 16 25)) ) ;;;;;;;;;;;;;;;;;;;;; diff --git a/goal_src/jak1/engine/level/level.gc b/goal_src/jak1/engine/level/level.gc index 50b0a10493..a255e234a2 100644 --- a/goal_src/jak1/engine/level/level.gc +++ b/goal_src/jak1/engine/level/level.gc @@ -2152,8 +2152,8 @@ ;; tell PC port about our levels (__pc-set-levels - (if (= (-> this level0 status) 'inactive) "none" (symbol->string (-> this level0 nickname))) - (if (= (-> this level1 status) 'inactive) "none" (symbol->string (-> this level1 nickname))) + (if (symbol-member? (-> this level0 status) '(active alive loaded)) (symbol->string (bsp-name (-> this level0))) "none") + (if (symbol-member? (-> this level1 status) '(active alive loaded)) (symbol->string (bsp-name (-> this level1))) "none") ) 0 diff --git a/goal_src/jak2/engine/gfx/background/background.gc b/goal_src/jak2/engine/gfx/background/background.gc index 1a4c304d75..f1a5b0b9c9 100644 --- a/goal_src/jak2/engine/gfx/background/background.gc +++ b/goal_src/jak2/engine/gfx/background/background.gc @@ -233,7 +233,7 @@ (defun add-pc-tfrag3-data ((dma-buf dma-buffer) (lev level)) "Add PC-port specific tfrag data" (let ((packet (the-as dma-packet (-> dma-buf base)))) - (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 24)) + (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 25)) (set! (-> packet vif0) (new 'static 'vif-tag)) (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port))) (set! (-> dma-buf base) (the pointer (&+ packet 16))) @@ -309,16 +309,16 @@ ) ) - (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (-> lev nickname))) + (copyn-charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (bsp-name lev)) 32) ) - (&+! (-> dma-buf base) (* 16 24)) + (&+! (-> dma-buf base) (* 16 25)) ) (defun add-pc-camera-data ((dma-buf dma-buffer)) "Add PC-port specific camera data. used as fallback for collide mesh renderer. Same as add-pc-trag3-data but level-specific data is left undefined." (let ((packet (the-as dma-packet (-> dma-buf base)))) - (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 24)) + (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 25)) (set! (-> packet vif0) (new 'static 'vif-tag)) (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port))) (set! (-> dma-buf base) (the pointer (&+ packet 16))) @@ -360,7 +360,7 @@ (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string #f)) ) - (&+! (-> dma-buf base) (* 16 24)) + (&+! (-> dma-buf base) (* 16 25)) ) diff --git a/goal_src/jak2/engine/level/level.gc b/goal_src/jak2/engine/level/level.gc index f2dea4faa9..ea71d6be0f 100644 --- a/goal_src/jak2/engine/level/level.gc +++ b/goal_src/jak2/engine/level/level.gc @@ -2967,9 +2967,9 @@ into 7 sections, which might explain the weird sizes in the center. ((or (= (-> this level i status) 'active) (= (-> this level i status) 'alive) (= (-> this level i status) 'loaded)) - (set! (-> lev-names i) (symbol->string (-> this level i nickname))) + (set! (-> lev-names i) (symbol->string (bsp-name (-> this level i)))) (if (-> this level i display?) - (set! (-> active-lev-names i) (symbol->string (-> this level i nickname)))) + (set! (-> active-lev-names i) (-> lev-names i))) ) ) ) diff --git a/goal_src/jak3/engine/gfx/background/background.gc b/goal_src/jak3/engine/gfx/background/background.gc index 0172bb5671..406418cc18 100644 --- a/goal_src/jak3/engine/gfx/background/background.gc +++ b/goal_src/jak3/engine/gfx/background/background.gc @@ -8,7 +8,7 @@ (defun add-pc-tfrag3-data ((dma-buf dma-buffer) (lev level)) "Add PC-port specific tfrag data" (let ((packet (the-as dma-packet (-> dma-buf base)))) - (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 24)) + (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 25)) (set! (-> packet vif0) (new 'static 'vif-tag)) (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port))) (set! (-> dma-buf base) (the pointer (&+ packet 16))) @@ -84,9 +84,9 @@ ) ) - (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (-> lev nickname))) + (copyn-charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string (bsp-name lev)) 32) ) - (&+! (-> dma-buf base) (* 16 24)) + (&+! (-> dma-buf base) (* 16 25)) ) ;; DECOMP BEGINS @@ -226,7 +226,7 @@ "Add PC-port specific camera data. used as fallback for collide mesh renderer. Same as add-pc-trag3-data but level-specific data is left undefined." (let ((packet (the-as dma-packet (-> dma-buf base)))) - (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 24)) + (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 25)) (set! (-> packet vif0) (new 'static 'vif-tag)) (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port))) (set! (-> dma-buf base) (the pointer (&+ packet 16))) @@ -268,7 +268,7 @@ (charp<-string (the (pointer uint8) (&-> data-ptr 23)) (symbol->string #f)) ) - (&+! (-> dma-buf base) (* 16 24)) + (&+! (-> dma-buf base) (* 16 25)) ) (defun add-pc-port-background-data ((dma-buf dma-buffer)) diff --git a/goal_src/jak3/engine/level/level.gc b/goal_src/jak3/engine/level/level.gc index 6502592979..d69d81ed12 100644 --- a/goal_src/jak3/engine/level/level.gc +++ b/goal_src/jak3/engine/level/level.gc @@ -3862,10 +3862,9 @@ ((or (= (-> this level i status) 'active) (= (-> this level i status) 'alive) (= (-> this level i status) 'loaded)) - ;; using info nickname since desert's bsp nickname is desert-vis, not dst. - (set! (-> lev-names i) (symbol->string (-> this level i info nickname))) + (set! (-> lev-names i) (symbol->string (bsp-name (-> this level i)))) (if (-> this level i display?) - (set! (-> active-lev-names i) (symbol->string (-> this level i info nickname)))) + (set! (-> active-lev-names i) (-> lev-names i))) ) ) ) diff --git a/goalc/build_level/jak1/build_level.cpp b/goalc/build_level/jak1/build_level.cpp index eb0477c05e..d09b914c6a 100644 --- a/goalc/build_level/jak1/build_level.cpp +++ b/goalc/build_level/jak1/build_level.cpp @@ -203,7 +203,7 @@ bool run_build_level(const std::string& input_file, } // Save the PC level - save_pc_data(file.nickname, pc_level, + save_pc_data(file.name, pc_level, file_util::get_jak_project_dir() / "out" / output_prefix / "fr3"); return true; diff --git a/goalc/build_level/jak2/build_level.cpp b/goalc/build_level/jak2/build_level.cpp index c92b2f4957..f7d3726ab8 100644 --- a/goalc/build_level/jak2/build_level.cpp +++ b/goalc/build_level/jak2/build_level.cpp @@ -189,7 +189,7 @@ bool run_build_level(const std::string& input_file, } // Save the PC level - save_pc_data(file.nickname, pc_level, + save_pc_data(file.name, pc_level, file_util::get_jak_project_dir() / "out" / output_prefix / "fr3"); return true;