From 9988c5305b8587b31e921be58a74e29d729d1020 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Tue, 14 Jul 2026 20:34:31 -0400 Subject: [PATCH] decomp: implement simple LRU cache for texture replacements --- decompiler/config.h | 2 +- decompiler/data/TextureDB.cpp | 19 ++++++--- decompiler/data/TextureDB.h | 44 +++++++++++++++++++- decompiler/decompilation_process.cpp | 6 +++ decompiler/level_extractor/extract_level.cpp | 13 +++--- decompiler/level_extractor/extract_level.h | 6 +-- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/decompiler/config.h b/decompiler/config.h index 1d08603bbf..02eced4c17 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -197,7 +197,7 @@ struct Config { // Returns whether or not with the current configuration, all we are doing is extracting the // levels this is so the decompiler can try to skip steps that are unnecessary for that. bool only_extracting_levels() const { - if (levels_extract && !disassemble_code && !!dump_function_metadata && !decompile_code && + if (levels_extract && !disassemble_code && !dump_function_metadata && !decompile_code && !write_scripts && !disassemble_data) { return true; } diff --git a/decompiler/data/TextureDB.cpp b/decompiler/data/TextureDB.cpp index eb6bc4320f..a27b7aefac 100644 --- a/decompiler/data/TextureDB.cpp +++ b/decompiler/data/TextureDB.cpp @@ -9,7 +9,7 @@ namespace decompiler { -TextureDB::TextureDB() { +TextureDB::TextureDB() : replacement_cache(TextureLRUCache(25)) { std::vector data(16 * 16, 0xffffffff); add_texture(kPlaceholderWhiteTexturePage, kPlaceholderWhiteTextureId, data, 16, 16, "placeholder-white", "placeholder", {}, 1, 0); @@ -148,11 +148,16 @@ void TextureDB::merge_texture(u32 id, std::vector& rgba) const { stbi_image_free(merge_data); } -std::optional TextureDB::replace_texture(u32 id) const { +std::optional TextureDB::replace_texture(u32 id) { if (!replace_texture_dir) { return std::nullopt; } + // First check the cache to see if it's there + if (auto replacement_texture = replacement_cache.get(id); replacement_texture != nullptr) { + return *replacement_texture; + } + const auto& tex = textures.at(id); const auto& tpage_name = tpage_names.at(tex.page); @@ -180,15 +185,16 @@ std::optional TextureDB::replace_texture(u32 id) const { result.w = static_cast(w); result.h = static_cast(h); result.rgba.resize(w * h); - memcpy(result.rgba.data(), data, w * h * 4); - stbi_image_free(data); + // Cache it + replacement_cache.put(id, result); + return result; } -ResolvedTextureData TextureDB::resolve_texture(u32 id) const { +ResolvedTextureData TextureDB::resolve_texture(u32 id) { const auto& tex = textures.at(id); ResolvedTextureData result{ @@ -197,6 +203,9 @@ ResolvedTextureData TextureDB::resolve_texture(u32 id) const { .rgba = tex.rgba_bytes, }; + // not bothering to LRU cache this, very niche feature intended for + // adding new font symbols and such in a way we can distribute + // in other words, these will always be small files. merge_texture(id, result.rgba); if (auto replacement = replace_texture(id)) { diff --git a/decompiler/data/TextureDB.h b/decompiler/data/TextureDB.h index ffe3c0cd89..f27b47638a 100644 --- a/decompiler/data/TextureDB.h +++ b/decompiler/data/TextureDB.h @@ -18,6 +18,45 @@ struct ResolvedTextureData { std::vector rgba; }; +struct TextureLRUCache { + using Entry = std::pair; + using List = std::list; + using Iterator = List::iterator; + + size_t capacity; + List entries; // front = most recently used + std::unordered_map lookup; + + TextureLRUCache(size_t capacity) : capacity(capacity) {} + + ResolvedTextureData* get(u32 id) { + auto it = lookup.find(id); + if (it == lookup.end()) { + return nullptr; + } + entries.splice(entries.begin(), entries, it->second); + return &it->second->second; + } + + void put(u32 id, ResolvedTextureData data) { + auto it = lookup.find(id); + if (it != lookup.end()) { + it->second->second = std::move(data); + entries.splice(entries.begin(), entries, it->second); + return; + } + + entries.emplace_front(id, std::move(data)); + lookup[id] = entries.begin(); + + if (lookup.size() > capacity) { + auto last = std::prev(entries.end()); + lookup.erase(last->first); + entries.pop_back(); + } + } +}; + struct TextureDB { TextureDB(); struct TextureData { @@ -34,13 +73,14 @@ struct TextureDB { std::unordered_map> texture_ids_per_level; std::optional merge_texture_dir; std::optional replace_texture_dir; + TextureLRUCache replacement_cache; // special textures for animation. std::map index_textures_by_combo_id; std::unordered_map animated_tex_output_to_anim_slot; - ResolvedTextureData resolve_texture(u32 id) const; + ResolvedTextureData resolve_texture(u32 id); static constexpr int kPlaceholderWhiteTexturePage = INT16_MAX; static constexpr int kPlaceholderWhiteTextureId = 0; @@ -69,7 +109,7 @@ struct TextureDB { void merge_textures(const fs::path& base_path); void replace_textures(const fs::path& path); void merge_texture(u32 id, std::vector& rgba) const; - std::optional replace_texture(u32 id) const; + std::optional replace_texture(u32 id); std::string generate_texture_dest_adjustment_table() const; }; diff --git a/decompiler/decompilation_process.cpp b/decompiler/decompilation_process.cpp index 58cc6b1d99..90e2353e35 100644 --- a/decompiler/decompilation_process.cpp +++ b/decompiler/decompilation_process.cpp @@ -311,6 +311,12 @@ void write_text_assets(Config config, lg::info("[Mem] After spool handling: {} MB", get_peak_rss() / (1024 * 1024)); } +// NOTE: Could save some memory usage here by making all the textures lazily loaded +// however it's quite the invasive refactor and the default textures from the game +// are not massive, so it's debatable if this is worth the effort +// +// Instead it might be better to try to reduce the need to process the DGOs / code so much +// as that is the vast majority of the default memory usage. std::unique_ptr handle_textures(Config config, const fs::path& in_folder, const fs::path& out_folder, diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index 8db3a8b895..d768a22c27 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -21,7 +21,6 @@ #include "decompiler/level_extractor/extract_tfrag.h" #include "decompiler/level_extractor/extract_tie.h" #include "decompiler/level_extractor/fr3_to_gltf.h" -#include "goalc/build_actor/jak1/build_actor.h" namespace decompiler { @@ -61,7 +60,7 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) { return true; } -tfrag3::Texture make_texture(u32 id, const TextureDB& tex_db, bool pool_load) { +tfrag3::Texture make_texture(u32 id, TextureDB& tex_db, bool pool_load) { const auto& tex = tex_db.textures.at(id); auto resolved = tex_db.resolve_texture(id); @@ -78,7 +77,7 @@ tfrag3::Texture make_texture(u32 id, const TextureDB& tex_db, bool pool_load) { void add_all_textures_from_level(tfrag3::Level& lev, const std::string& level_name, - const TextureDB& tex_db) { + TextureDB& tex_db) { auto level_it = tex_db.texture_ids_per_level.find(level_name); if (level_it == tex_db.texture_ids_per_level.end()) { return; @@ -269,7 +268,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, * but the bsp stuff is just empty. It will contain only textures/art groups. */ void extract_common(const ObjectFileDB& db, - const TextureDB& tex_db, + TextureDB& tex_db, const std::string& dgo_name, const fs::path& output_folder, const Config& config) { @@ -352,7 +351,7 @@ void extract_common(const ObjectFileDB& db, } void extract_from_level(const ObjectFileDB& db, - const TextureDB& tex_db, + TextureDB& tex_db, const std::string& dgo_name, const Config& config, const fs::path& output_folder, @@ -407,7 +406,7 @@ void extract_from_level(const ObjectFileDB& db, } void extract_all_levels(const ObjectFileDB& db, - const TextureDB& tex_db, + TextureDB& tex_db, const std::vector& dgo_names, const std::string& common_name, const Config& config, @@ -417,7 +416,7 @@ void extract_all_levels(const ObjectFileDB& db, game_version_names[config.game_version] / "entities"; file_util::create_dir_if_needed(entities_dir); - int num_workers = dgo_names.size(); + int num_workers = 1; if (tex_db.replace_texture_dir) { num_workers = 1; } diff --git a/decompiler/level_extractor/extract_level.h b/decompiler/level_extractor/extract_level.h index 9f6c8b983e..a242cd0dc3 100644 --- a/decompiler/level_extractor/extract_level.h +++ b/decompiler/level_extractor/extract_level.h @@ -53,15 +53,15 @@ struct MercSwapInfo { // extract everything void extract_all_levels(const ObjectFileDB& db, - const TextureDB& tex_db, + TextureDB& tex_db, const std::vector& dgo_names, const std::string& common_name, const Config& config, const fs::path& path); void add_all_textures_from_level(tfrag3::Level& lev, const std::string& level_name, - const TextureDB& tex_db); -tfrag3::Texture make_texture(u32 id, const TextureDB& tex_db, bool pool_load); + TextureDB& tex_db); +tfrag3::Texture make_texture(u32 id, TextureDB& tex_db, bool pool_load); std::vector extract_tex_remap(const ObjectFileDB& db, const std::string& dgo_name); std::optional get_bsp_file(const std::vector& records,