From b6597c2c80ed094461d373c3d05942804b1c44c3 Mon Sep 17 00:00:00 2001 From: Shokunin <30949000+tetrisgm@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:37:53 -0700 Subject: [PATCH 1/4] HD textures: decode the whole pack into RAM before the game runs Replacement images were decoded on demand by a single low-priority worker, so a texture first seen mid-race appeared late and could be evicted again. When replacements are on, configuring a pack now decodes every unambiguous image (DuckStation and Beetle formats) across all cores and pins it in RAM before the game draws; pinned images are never evicted by the decode budget. Logs the count, size and time. Co-Authored-By: Claude Opus 5.5 --- runtime/include/duckstation_texture_pack.h | 4 ++ runtime/include/hd_texture_pack.h | 3 + runtime/src/duckstation_texture_pack.cpp | 54 ++++++++++++++- runtime/src/gpu_hd_textures.cpp | 16 +++++ runtime/src/hd_texture_pack.cpp | 65 ++++++++++++++++++- .../tests/test_duckstation_texture_pack.cpp | 22 ++++++- 6 files changed, 160 insertions(+), 4 deletions(-) diff --git a/runtime/include/duckstation_texture_pack.h b/runtime/include/duckstation_texture_pack.h index eb0cab4e..b552e914 100644 --- a/runtime/include/duckstation_texture_pack.h +++ b/runtime/include/duckstation_texture_pack.h @@ -170,6 +170,10 @@ int duck_texture_pack_match_parts(DuckTexturePack* pack, * Release the lease even after pack destruction. */ void duck_texture_pack_set_decode_budget(DuckTexturePack* pack, size_t bytes); int duck_texture_pack_request_decode(DuckTexturePack* pack, uint64_t entry_id); +/* Decode every unambiguous replacement now (all cores) and pin it in RAM, so + * no draw waits for or evicts an image. Returns the count loaded, -1 on error; + * bytes_out gets the decoded RGBA total. */ +int duck_texture_pack_preload(DuckTexturePack* pack, size_t* bytes_out); int duck_texture_pack_acquire_decoded(DuckTexturePack* pack, uint64_t entry_id, DuckTexturePixels* out_pixels); void duck_texture_pixels_release(DuckTexturePixels* pixels); diff --git a/runtime/include/hd_texture_pack.h b/runtime/include/hd_texture_pack.h index af389ffe..aebd2c11 100644 --- a/runtime/include/hd_texture_pack.h +++ b/runtime/include/hd_texture_pack.h @@ -180,6 +180,9 @@ typedef struct HdTexturePixels { void hd_texture_pack_set_decode_budget(HdTexturePack* pack, size_t budget_bytes); +/* Decode every unambiguous replacement now (all cores) and pin it in RAM. + * Returns the count loaded, -1 on error; bytes_out gets the RGBA total. */ +int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out); int hd_texture_pack_request_decode(HdTexturePack* pack, uint32_t texture_hash, uint32_t palette_hash); diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index e577638f..2c0fdd3c 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -183,6 +183,7 @@ struct Decode { std::string path; std::shared_ptr image; uint64_t use = 0; + bool pinned = false; /* preloaded: never evicted */ }; struct Dump { fs::path path; std::shared_ptr image; }; struct DumpHistory { std::unordered_set names; std::deque order; }; @@ -243,7 +244,7 @@ struct Worker { while (used + incoming > budget) { auto candidate = decoded.end(); for (auto it = decoded.begin(); it != decoded.end(); ++it) - if (it->second.status == DecodeStatus::Ready && + if (it->second.status == DecodeStatus::Ready && !it->second.pinned && (candidate == decoded.end() || it->second.use < candidate->second.use)) candidate = it; if (candidate == decoded.end()) break; used -= candidate->second.image->rgba.size(); @@ -1122,7 +1123,7 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { auto victim = w.decoded.end(); for (auto item = w.decoded.begin(); item != w.decoded.end(); ++item) if ((item->second.status == DecodeStatus::Ready || item->second.status == DecodeStatus::Failed) && - (victim == w.decoded.end() || item->second.use < victim->second.use)) victim = item; + !item->second.pinned && (victim == w.decoded.end() || item->second.use < victim->second.use)) victim = item; if (victim == w.decoded.end()) return HD_TEXTURE_LOOKUP_NONE; if (victim->second.image) w.used -= victim->second.image->rgba.size(); w.decoded.erase(victim); @@ -1132,6 +1133,55 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { w.wake.notify_one(); return HD_TEXTURE_LOOKUP_NONE; } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } } +int duck_texture_pack_preload(DuckTexturePack* p, size_t* bytes_out) { + if (bytes_out) *bytes_out = 0; + if (!p) return -1; + try { + /* Decode every replacement before the game runs, across all cores, + * so no draw ever waits for (or misses) an image mid-game. */ + std::vector ids; + for (size_t i = 0; i < p->entries.size(); ++i) + if (!p->entries[i].ambiguous) ids.push_back(i + 1); + std::vector> images(ids.size()); + std::atomic next{0}; + auto work = [&] { + for (size_t k; (k = next++) < ids.size();) { + try { + const std::string& path = p->entries[ids[k] - 1].path; + std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); + const auto length = input ? input.tellg() : std::streampos(-1); + if (length <= 0 || static_cast(length) > kMaxEncoded) continue; + std::vector encoded(static_cast(length)); + input.seekg(0); + DecodedTextureImage decoded; std::string error; + if (!input.read(reinterpret_cast(encoded.data()), length) || + !texture_image_decode(encoded.data(), encoded.size(), SIZE_MAX, decoded, error)) continue; + auto image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; + image->rgba = std::move(decoded.rgba); images[k] = std::move(image); + } catch (...) {} + } + }; + unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); + std::vector pool; + for (unsigned t = 1; t < threads && t < ids.size(); ++t) pool.emplace_back(work); + work(); + for (auto& t : pool) t.join(); + Worker& w = p->worker; std::lock_guard lock(w.mutex); + int loaded = 0; size_t bytes = 0; + for (size_t k = 0; k < ids.size(); ++k) { + Decode& d = w.decoded[ids[k]]; + d.path = p->entries[ids[k] - 1].path; + if (!images[k]) { d.status = DecodeStatus::Failed; continue; } + if (d.image) w.used -= d.image->rgba.size(); + bytes += images[k]->rgba.size(); w.used += images[k]->rgba.size(); + d.image = std::move(images[k]); d.status = DecodeStatus::Ready; d.pinned = true; ++loaded; + } + if (w.budget < w.used) w.budget = w.used; + w.decoded_images += loaded; + if (bytes_out) *bytes_out = bytes; + return loaded; + } catch (...) { return -1; } +} int duck_texture_pack_acquire_decoded(DuckTexturePack* p, uint64_t id, DuckTexturePixels* out) { if (out) *out = {}; if (!p || !out) return HD_TEXTURE_LOOKUP_ERROR; diff --git a/runtime/src/gpu_hd_textures.cpp b/runtime/src/gpu_hd_textures.cpp index 62811dd9..63286fb8 100644 --- a/runtime/src/gpu_hd_textures.cpp +++ b/runtime/src/gpu_hd_textures.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -289,6 +290,21 @@ extern "C" int gpu_hd_textures_configure(const char* root, int replacements, std::filesystem::create_directories(directory / "dumps"); } if (!duck_texture_pack_create(root, &next->duck, error, capacity)) return 0; + if (next->replacements) { + /* Every replacement decoded and resident before the game draws. */ + const auto t0 = std::chrono::steady_clock::now(); + size_t bytes = 0, beetle_bytes = 0; + int loaded = duck_texture_pack_preload(next->duck, &bytes); + if (next->beetle) { + const int beetle_loaded = hd_texture_pack_preload(next->beetle, &beetle_bytes); + if (beetle_loaded > 0) { loaded = (loaded > 0 ? loaded : 0) + beetle_loaded; bytes += beetle_bytes; } + } + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + if (loaded > 0) + std::fprintf(stdout, "psxrecomp: texture pack: %d images preloaded (%.1f MiB) in %lld ms\n", + loaded, bytes / 1048576.0, static_cast(ms)); + } /* Quiesce renderer users (GL render thread: dump_draw appends to the * old session's sources) before flushing its dumps or reading its * residency; the cache is rebuilt from the new session. */ diff --git a/runtime/src/hd_texture_pack.cpp b/runtime/src/hd_texture_pack.cpp index 720b6794..f21551a1 100644 --- a/runtime/src/hd_texture_pack.cpp +++ b/runtime/src/hd_texture_pack.cpp @@ -5,6 +5,7 @@ #endif #include +#include #include #include #include @@ -448,6 +449,7 @@ struct DecodeItem { std::shared_ptr image; size_t bytes = 0; uint64_t last_used = 0; + bool pinned = false; /* preloaded: never evicted */ }; struct DecodeCache { @@ -480,7 +482,7 @@ void evict_decode_cache(DecodeCache& cache, size_t incoming = 0) { while (cache.used + incoming > cache.budget) { auto victim = cache.items.end(); for (auto it = cache.items.begin(); it != cache.items.end(); ++it) { - if (it->second.state != DecodeState::Ready) continue; + if (it->second.state != DecodeState::Ready || it->second.pinned) continue; if (victim == cache.items.end() || it->second.last_used < victim->second.last_used) victim = it; @@ -1282,6 +1284,67 @@ void hd_texture_pack_set_decode_budget(HdTexturePack* pack, evict_decode_cache(pack->decode); } +int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out) { + if (bytes_out) *bytes_out = 0; +#ifdef HD_TEXTURE_PACK_DISABLE_PNG_DECODE + (void)pack; + return -1; +#else + if (!pack) return -1; + try { + /* Decode every replacement before the game runs, across all cores. */ + std::vector> work_items; + for (const auto& entry : pack->entries) + if (!entry.second.ambiguous) + work_items.emplace_back(entry.first, entry.second.replacement_path); + std::vector> images(work_items.size()); + std::atomic next{0}; + auto work = [&] { + for (size_t k; (k = next++) < work_items.size();) { + std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); + const std::streamoff length = input ? static_cast(input.tellg()) : -1; + if (length <= 0 || static_cast(length) > kMaxPngFileBytes) continue; + std::vector encoded(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) continue; + int width = 0, height = 0, components = 0; + stbi_uc* loaded = stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), + &width, &height, &components, 4); + if (!loaded) continue; + if (width > 0 && height > 0) { + auto image = std::make_shared(); + image->width = static_cast(width); + image->height = static_cast(height); + image->rgba.assign(loaded, loaded + static_cast(width) * height * 4); + images[k] = std::move(image); + } + stbi_image_free(loaded); + } + }; + const unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); + std::vector pool; + for (unsigned t = 1; t < threads && t < work_items.size(); ++t) pool.emplace_back(work); + work(); + for (auto& t : pool) t.join(); + std::lock_guard lock(pack->decode.mutex); + int loaded = 0; size_t bytes = 0; + for (size_t k = 0; k < work_items.size(); ++k) { + DecodeItem& item = pack->decode.items[work_items[k].first]; + item.path = work_items[k].second; + if (!images[k]) { item.state = DecodeState::Failed; continue; } + if (item.state == DecodeState::Ready) pack->decode.used -= item.bytes; + item.bytes = images[k]->rgba.size(); + item.image = std::move(images[k]); + item.state = DecodeState::Ready; item.pinned = true; + pack->decode.used += item.bytes; bytes += item.bytes; ++loaded; + } + if (pack->decode.budget < pack->decode.used) pack->decode.budget = pack->decode.used; + if (bytes_out) *bytes_out = bytes; + return loaded; + } catch (...) { return -1; } +#endif +} + int hd_texture_pack_request_decode(HdTexturePack* pack, uint32_t texture_hash, uint32_t palette_hash) { diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index f7f57639..f13c07d2 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -109,6 +109,26 @@ bool decode_fails(DuckTexturePack* pack, uint64_t id) { return status == HD_TEXTURE_LOOKUP_ERROR; } +void preload(const fs::path& root) { + const char* name = "texpage-P4-3631D2A04EC85311-9CBCDEA9305D7528-64x256-4-2-8x2-P0-15.png"; + png(root/"replacements"/name,16,4); + std::vector vram(1024*512); + place(vram,65,258,2,words(4)); place(vram,512,400,16,words(16)); + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + size_t bytes = 0; + check(duck_texture_pack_preload(pack.get(),&bytes) == 1 && bytes == 16*4*4,"preload decodes the pack"); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"preloaded fixture matches"); + DuckTexturePixels pixels{}; + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND && + pixels.width == 16,"preloaded image is resident without a decode request"); + duck_texture_pixels_release(&pixels); + duck_texture_pack_set_decode_budget(pack.get(),1); + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, + "a preloaded image survives budget eviction"); + duck_texture_pixels_release(&pixels); +} + void fixed_hash_vectors() { /* Independently produced by Python xxhash 3.6.0's native wheel. These * literal expectations cover XXH3's short/medium/long-input boundaries. */ @@ -604,7 +624,7 @@ int main(int argc, char** argv) { palettes(root/"palettes"); st_alias_duplicates(root/"keys"); failed_payloads(root/"invalid"); dumping(root/"dump"); official_binary_oracle(root/"official-oracle"); defaults_and_options(root/"defaults"); immutable_palettes_and_pages(root/"snapshots"); split_coalesce_copy_parts(root/"lifecycle"); copied_pack_ownership(root/"ownership"); - resident_checkpoint(root/"resident"); + resident_checkpoint(root/"resident"); preload(root/"preload"); /* Every artifact is created beneath this fresh, explicitly named test * root. No user-provided paths participate in recursive cleanup. */ if (argc > 1 && !std::strcmp(argv[1],"--keep-artifacts")) std::printf("ARTIFACT_ROOT: %s\n",root.u8string().c_str()); From 50cbe5f653c319d0a28448e9d9c9a3abb44139cf Mon Sep 17 00:00:00 2001 From: Shokunin <30949000+tetrisgm@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:27:29 -0700 Subject: [PATCH 2/4] HD preload: bounded contract, safe threads, explicit partial result Both preload functions now take an aggregate decoded-byte cap and a thread cap (runtime: 512 MiB, 4 threads). Each image is probed and its RGBA reserved against the cap before decoding; per-image limits follow the pack's decode budget, which is no longer grown to usage (pinned images sit outside it, and outside the Duck 512-entry cap). The pool (texture_pack_ preload.hpp) joins every started thread on unwinding, survives thread creation failure, and catches worker exceptions. Images not preloaded are left to the asynchronous budgeted path instead of being marked failed, and HdTexturePreloadResult reports loaded / over-cap / failed / worker errors with a complete-or-partial status. The Beetle async worker no longer lets allocation failures escape its thread entry. Tests cover both formats: complete preload, over-cap skip with on-demand fallback, malformed images, injected worker and thread-creation failures, and budget reservation bounds. Co-Authored-By: Claude Opus 5.5 --- runtime/include/duckstation_texture_pack.h | 10 +- runtime/include/hd_texture_pack.h | 21 ++- runtime/include/texture_pack_preload.hpp | 96 ++++++++++ runtime/src/duckstation_texture_pack.cpp | 106 ++++++----- runtime/src/gpu_hd_textures.cpp | 32 +++- runtime/src/hd_texture_pack.cpp | 171 +++++++++++------- .../tests/test_duckstation_texture_pack.cpp | 86 +++++++-- runtime/tests/test_hd_texture_pack.cpp | 50 +++++ 8 files changed, 433 insertions(+), 139 deletions(-) create mode 100644 runtime/include/texture_pack_preload.hpp diff --git a/runtime/include/duckstation_texture_pack.h b/runtime/include/duckstation_texture_pack.h index b552e914..d0569139 100644 --- a/runtime/include/duckstation_texture_pack.h +++ b/runtime/include/duckstation_texture_pack.h @@ -170,10 +170,12 @@ int duck_texture_pack_match_parts(DuckTexturePack* pack, * Release the lease even after pack destruction. */ void duck_texture_pack_set_decode_budget(DuckTexturePack* pack, size_t bytes); int duck_texture_pack_request_decode(DuckTexturePack* pack, uint64_t entry_id); -/* Decode every unambiguous replacement now (all cores) and pin it in RAM, so - * no draw waits for or evicts an image. Returns the count loaded, -1 on error; - * bytes_out gets the decoded RGBA total. */ -int duck_texture_pack_preload(DuckTexturePack* pack, size_t* bytes_out); +/* Elective full preload with the hd_texture_pack_preload contract: bounded + * aggregate decoded bytes and concurrency, pinned images outside the decode + * budget, everything not preloaded left to the asynchronous decode path. + * Returns 1 complete, 0 partial (see out), -1 on error. */ +int duck_texture_pack_preload(DuckTexturePack* pack, size_t max_bytes, + unsigned max_threads, HdTexturePreloadResult* out); int duck_texture_pack_acquire_decoded(DuckTexturePack* pack, uint64_t entry_id, DuckTexturePixels* out_pixels); void duck_texture_pixels_release(DuckTexturePixels* pixels); diff --git a/runtime/include/hd_texture_pack.h b/runtime/include/hd_texture_pack.h index aebd2c11..aa0c2ff2 100644 --- a/runtime/include/hd_texture_pack.h +++ b/runtime/include/hd_texture_pack.h @@ -180,9 +180,24 @@ typedef struct HdTexturePixels { void hd_texture_pack_set_decode_budget(HdTexturePack* pack, size_t budget_bytes); -/* Decode every unambiguous replacement now (all cores) and pin it in RAM. - * Returns the count loaded, -1 on error; bytes_out gets the RGBA total. */ -int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out); +/* Elective full preload: decode unambiguous replacements up front and pin + * them, bounded by max_bytes of decoded RGBA in aggregate and by max_threads + * concurrent decodes (the caller's thread included). Pinned images are kept + * outside the on-demand decode budget, which stays as configured. Images not + * preloaded (over the cap, unreadable, malformed, a worker failure) are left + * to the normal asynchronous, budgeted decode path; nothing is marked failed + * by the preload. Returns 1 when every image was preloaded, 0 for a partial + * preload (see out), -1 on error. */ +typedef struct HdTexturePreloadResult { + int requested; /* unambiguous replacements considered */ + int loaded; /* decoded and pinned */ + int over_budget; /* skipped: the aggregate max_bytes was reached */ + int failed; /* unreadable, malformed, too large, or worker error */ + int worker_errors; /* of failed: a decode that threw (caught) */ + size_t bytes; /* decoded RGBA pinned */ +} HdTexturePreloadResult; +int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, + unsigned max_threads, HdTexturePreloadResult* out); int hd_texture_pack_request_decode(HdTexturePack* pack, uint32_t texture_hash, uint32_t palette_hash); diff --git a/runtime/include/texture_pack_preload.hpp b/runtime/include/texture_pack_preload.hpp new file mode 100644 index 00000000..6517802b --- /dev/null +++ b/runtime/include/texture_pack_preload.hpp @@ -0,0 +1,96 @@ +#ifndef PSX_TEXTURE_PACK_PRELOAD_HPP +#define PSX_TEXTURE_PACK_PRELOAD_HPP + +/* Shared machinery for the elective full texture-pack preload + * (hd_texture_pack_preload, duck_texture_pack_preload): an aggregate + * decoded-byte reservation and a bounded worker pool that can neither leak a + * joinable std::thread nor let an exception escape a thread entry. */ + +#include +#include +#include +#include +#include +#include +#include + +namespace psx_texpreload { + +/* Test-only fault injection. 1: the first item's decode throws + * std::bad_alloc. 2: creating the second pool thread throws std::system_error. */ +inline std::atomic& fault_mode() { + static std::atomic mode{0}; + return mode; +} + +/* Aggregate decoded bytes, reserved before an image is decoded. */ +class Budget { +public: + explicit Budget(size_t cap) : cap_(cap) {} + bool reserve(size_t bytes) { + size_t cur = used_.load(); + do { + if (bytes > cap_ || cur > cap_ - bytes) return false; + } while (!used_.compare_exchange_weak(cur, cur + bytes)); + return true; + } + void release(size_t bytes) { used_.fetch_sub(bytes); } +private: + size_t cap_; + std::atomic used_{0}; +}; + +/* Per-item outcome, written by exactly one worker. */ +enum ItemStatus : unsigned char { kPending = 0, kLoaded, kFailed, kOverBudget }; + +/* Joins every started thread when the pool goes out of scope, including while + * unwinding from a failed std::thread construction. */ +struct Joiner { + std::vector& threads; + ~Joiner() { + for (auto& t : threads) + if (t.joinable()) t.join(); + } +}; + +/* Runs item(k) for every k < n on at most max_threads threads, the caller + * being one of them. An exception from item(k) is caught and counted, never + * propagated out of a thread entry. A failure to start a thread only lowers + * concurrency. Returns the number of items whose item(k) threw. */ +template +size_t run(size_t n, unsigned max_threads, F&& item) { + std::atomic next{0}, errors{0}; + auto work = [&]() noexcept { + for (size_t k; (k = next.fetch_add(1)) < n;) { + try { + if (k == 0 && fault_mode().load() == 1) throw std::bad_alloc(); + item(k); + } catch (...) { + errors.fetch_add(1); + } + } + }; + unsigned hw = std::thread::hardware_concurrency(); + if (hw == 0) hw = 1; + const unsigned threads = std::max(1u, std::min(max_threads ? max_threads : 1u, hw)); + std::vector pool; + { + Joiner joiner{pool}; + for (unsigned t = 1; t < threads && t < n; ++t) { + try { + if (t == 1 && fault_mode().load() == 2) + throw std::system_error( + std::make_error_code(std::errc::resource_unavailable_try_again)); + pool.emplace_back(work); + } catch (...) { + break; + } + } + work(); + } + return errors.load(); +} + +} // namespace psx_texpreload + +#endif diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index 2c0fdd3c..73ee9319 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -1,5 +1,6 @@ #include "duckstation_texture_pack.h" #include "texture_image_decode.h" +#include "texture_pack_preload.hpp" #define XXH_INLINE_ALL #include "../third_party/xxhash.h" #include "png_write.h" @@ -232,6 +233,7 @@ struct Worker { size_t dump_bytes = 0; uint64_t tick = 0; uint64_t decoded_images = 0, decode_evictions = 0; + size_t pinned_count = 0; /* preloaded entries: outside the 512-entry cap */ bool stop = false; bool dump_busy = false; std::string dump_error; @@ -1119,7 +1121,7 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { } if (!w.budget) return HD_TEXTURE_LOOKUP_ERROR; if (w.decode_queue.size() >= kMaxDecodeQueue) return HD_TEXTURE_LOOKUP_NONE; - if (w.decoded.size() >= 512) { + if (w.decoded.size() - w.pinned_count >= 512) { auto victim = w.decoded.end(); for (auto item = w.decoded.begin(); item != w.decoded.end(); ++item) if ((item->second.status == DecodeStatus::Ready || item->second.status == DecodeStatus::Failed) && @@ -1133,53 +1135,73 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { w.wake.notify_one(); return HD_TEXTURE_LOOKUP_NONE; } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } } -int duck_texture_pack_preload(DuckTexturePack* p, size_t* bytes_out) { - if (bytes_out) *bytes_out = 0; +int duck_texture_pack_preload(DuckTexturePack* p, size_t max_bytes, unsigned max_threads, + HdTexturePreloadResult* out) { + HdTexturePreloadResult r{}; + if (out) *out = r; if (!p) return -1; try { - /* Decode every replacement before the game runs, across all cores, - * so no draw ever waits for (or misses) an image mid-game. */ + namespace tp = psx_texpreload; std::vector ids; for (size_t i = 0; i < p->entries.size(); ++i) if (!p->entries[i].ambiguous) ids.push_back(i + 1); - std::vector> images(ids.size()); - std::atomic next{0}; - auto work = [&] { - for (size_t k; (k = next++) < ids.size();) { - try { - const std::string& path = p->entries[ids[k] - 1].path; - std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); - const auto length = input ? input.tellg() : std::streampos(-1); - if (length <= 0 || static_cast(length) > kMaxEncoded) continue; - std::vector encoded(static_cast(length)); - input.seekg(0); - DecodedTextureImage decoded; std::string error; - if (!input.read(reinterpret_cast(encoded.data()), length) || - !texture_image_decode(encoded.data(), encoded.size(), SIZE_MAX, decoded, error)) continue; - auto image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; - image->rgba = std::move(decoded.rgba); images[k] = std::move(image); - } catch (...) {} - } - }; - unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); - std::vector pool; - for (unsigned t = 1; t < threads && t < ids.size(); ++t) pool.emplace_back(work); - work(); - for (auto& t : pool) t.join(); - Worker& w = p->worker; std::lock_guard lock(w.mutex); - int loaded = 0; size_t bytes = 0; - for (size_t k = 0; k < ids.size(); ++k) { - Decode& d = w.decoded[ids[k]]; - d.path = p->entries[ids[k] - 1].path; - if (!images[k]) { d.status = DecodeStatus::Failed; continue; } - if (d.image) w.used -= d.image->rgba.size(); - bytes += images[k]->rgba.size(); w.used += images[k]->rgba.size(); - d.image = std::move(images[k]); d.status = DecodeStatus::Ready; d.pinned = true; ++loaded; + size_t per_image = 0; + { + std::lock_guard lock(p->worker.mutex); + per_image = p->worker.budget; } - if (w.budget < w.used) w.budget = w.used; - w.decoded_images += loaded; - if (bytes_out) *bytes_out = bytes; - return loaded; + std::vector> images(ids.size()); + std::vector status(ids.size(), tp::kPending); + tp::Budget budget(max_bytes); + r.worker_errors = static_cast(tp::run(ids.size(), max_threads, [&](size_t k) { + status[k] = tp::kFailed; + const std::string& path = p->entries[ids[k] - 1].path; + std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); + const auto length = input ? input.tellg() : std::streampos(-1); + if (length <= 0 || static_cast(length) > kMaxEncoded) return; + std::vector encoded(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) return; + TextureImageProbe probe; std::string error; + if (!texture_image_probe(encoded.data(), encoded.size(), per_image, probe, error)) return; + const size_t need = static_cast(probe.width) * probe.height * 4u; + if (!budget.reserve(need)) { status[k] = tp::kOverBudget; return; } + struct Reservation { + tp::Budget& b; size_t n; bool keep = false; + ~Reservation() { if (!keep) b.release(n); } + } reservation{budget, need}; + DecodedTextureImage decoded; + if (!texture_image_decode(encoded.data(), encoded.size(), need, decoded, error) || + decoded.rgba.size() != need) return; + auto image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; + image->rgba = std::move(decoded.rgba); + images[k] = std::move(image); + reservation.keep = true; + status[k] = tp::kLoaded; + })); + r.requested = static_cast(ids.size()); + Worker& w = p->worker; std::lock_guard lock(w.mutex); + for (size_t k = 0; k < ids.size(); ++k) { + if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } + if (status[k] != tp::kLoaded) continue; + auto it = w.decoded.find(ids[k]); + if (it != w.decoded.end()) { + /* Already pinned, or an on-demand decode in flight: leave it. */ + if (it->second.pinned || it->second.status == DecodeStatus::Queued || + it->second.status == DecodeStatus::Loading) continue; + if (it->second.image) w.used -= it->second.image->rgba.size(); + } + Decode& entry = w.decoded[ids[k]]; + entry.path = p->entries[ids[k] - 1].path; + r.bytes += images[k]->rgba.size(); + entry.image = std::move(images[k]); entry.status = DecodeStatus::Ready; + entry.pinned = true; entry.use = ++w.tick; + ++w.pinned_count; ++r.loaded; + } + w.decoded_images += static_cast(r.loaded); + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; + return r.loaded == r.requested ? 1 : 0; } catch (...) { return -1; } } int duck_texture_pack_acquire_decoded(DuckTexturePack* p, uint64_t id, DuckTexturePixels* out) { diff --git a/runtime/src/gpu_hd_textures.cpp b/runtime/src/gpu_hd_textures.cpp index 63286fb8..a0fa917d 100644 --- a/runtime/src/gpu_hd_textures.cpp +++ b/runtime/src/gpu_hd_textures.cpp @@ -26,6 +26,11 @@ constexpr size_t kCompositionBudget = 64u * 1024u * 1024u; constexpr size_t kCompositionCount = 256; constexpr uint32_t kCompositionDimension = 8192; constexpr size_t kMaxParts = 64; +/* Elective full preload of an installed pack: at most this much decoded RGBA + * pinned across both formats, decoded by at most this many threads. Anything + * beyond decodes on demand under the normal budget. */ +constexpr size_t kPreloadMaxBytes = size_t{512} * 1024u * 1024u; +constexpr unsigned kPreloadThreads = 4; struct Composition { std::vector signature; std::vector rgba; @@ -291,19 +296,30 @@ extern "C" int gpu_hd_textures_configure(const char* root, int replacements, } if (!duck_texture_pack_create(root, &next->duck, error, capacity)) return 0; if (next->replacements) { - /* Every replacement decoded and resident before the game draws. */ + /* Replacements decoded and resident before the game draws, within + * kPreloadMaxBytes; the rest stays on the budgeted async path. */ const auto t0 = std::chrono::steady_clock::now(); - size_t bytes = 0, beetle_bytes = 0; - int loaded = duck_texture_pack_preload(next->duck, &bytes); + HdTexturePreloadResult dr{}, br{}; + const int dstatus = duck_texture_pack_preload(next->duck, kPreloadMaxBytes, + kPreloadThreads, &dr); + int bstatus = 1; if (next->beetle) { - const int beetle_loaded = hd_texture_pack_preload(next->beetle, &beetle_bytes); - if (beetle_loaded > 0) { loaded = (loaded > 0 ? loaded : 0) + beetle_loaded; bytes += beetle_bytes; } + const size_t left = dr.bytes < kPreloadMaxBytes ? kPreloadMaxBytes - dr.bytes : 0; + bstatus = hd_texture_pack_preload(next->beetle, left, kPreloadThreads, &br); } const auto ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0).count(); - if (loaded > 0) - std::fprintf(stdout, "psxrecomp: texture pack: %d images preloaded (%.1f MiB) in %lld ms\n", - loaded, bytes / 1048576.0, static_cast(ms)); + const int requested = dr.requested + br.requested; + if (requested > 0 || dstatus < 0 || bstatus < 0) + std::fprintf(stdout, "psxrecomp: texture pack: preloaded %d/%d images (%.1f MiB) in %lld ms" + "%s; %d over the %u MiB preload cap, %d failed (%d worker errors);" + " the rest decode on demand\n", + dr.loaded + br.loaded, requested, (dr.bytes + br.bytes) / 1048576.0, + static_cast(ms), + (dstatus < 0 || bstatus < 0) ? " (preload error)" : + (dstatus == 1 && bstatus == 1) ? " (complete)" : " (partial)", + dr.over_budget + br.over_budget, static_cast(kPreloadMaxBytes / (1024u * 1024u)), + dr.failed + br.failed, dr.worker_errors + br.worker_errors); } /* Quiesce renderer users (GL render thread: dump_draw appends to the * old session's sources) before flushing its dumps or reading its diff --git a/runtime/src/hd_texture_pack.cpp b/runtime/src/hd_texture_pack.cpp index f21551a1..ce3f0859 100644 --- a/runtime/src/hd_texture_pack.cpp +++ b/runtime/src/hd_texture_pack.cpp @@ -1,4 +1,5 @@ #include "hd_texture_pack.h" +#include "texture_pack_preload.hpp" #ifndef HD_TEXTURE_PACK_DISABLE_PNG_DECODE #include "../third_party/stb_image.h" /* declarations only; implementation is shared */ @@ -513,38 +514,46 @@ void decode_worker(DecodeCache* cache) { path = it->second.path; } - std::vector encoded; - { - std::ifstream input(path, std::ios::binary | std::ios::ate); - std::streamoff length = -1; - if (input) length = static_cast(input.tellg()); - if (length > 0 && static_cast(length) <= kMaxPngFileBytes && - static_cast(length) <= - static_cast(std::numeric_limits::max())) { - encoded.resize(static_cast(length)); - input.seekg(0); - if (!input.read(reinterpret_cast(encoded.data()), length)) - encoded.clear(); - } - } - int width = 0; - int height = 0; - int components = 0; - stbi_uc* loaded = encoded.empty() ? nullptr : stbi_load_from_memory( - encoded.data(), static_cast(encoded.size()), - &width, &height, &components, 4); std::shared_ptr decoded; size_t bytes = 0; - if (loaded && width > 0 && height > 0 && - static_cast(width) * static_cast(height) <= - std::numeric_limits::max() / 4) { - bytes = static_cast(width) * static_cast(height) * 4; - decoded = std::make_shared(); - decoded->width = static_cast(width); - decoded->height = static_cast(height); - decoded->rgba.assign(loaded, loaded + bytes); + /* Allocation failure (encoded buffer, RGBA copy) must not escape the + * thread entry: the item fails like an undecodable image. */ + try { + std::vector encoded; + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + std::streamoff length = -1; + if (input) length = static_cast(input.tellg()); + if (length > 0 && static_cast(length) <= kMaxPngFileBytes && + static_cast(length) <= + static_cast(std::numeric_limits::max())) { + encoded.resize(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) + encoded.clear(); + } + } + int width = 0; + int height = 0; + int components = 0; + std::unique_ptr loaded( + encoded.empty() ? nullptr : stbi_load_from_memory( + encoded.data(), static_cast(encoded.size()), + &width, &height, &components, 4), + stbi_image_free); + if (loaded && width > 0 && height > 0 && + static_cast(width) * static_cast(height) <= + std::numeric_limits::max() / 4) { + bytes = static_cast(width) * static_cast(height) * 4; + decoded = std::make_shared(); + decoded->width = static_cast(width); + decoded->height = static_cast(height); + decoded->rgba.assign(loaded.get(), loaded.get() + bytes); + } + } catch (...) { + decoded.reset(); + bytes = 0; } - if (loaded) stbi_image_free(loaded); std::lock_guard lock(cache->mutex); auto it = cache->items.find(key); @@ -1284,63 +1293,89 @@ void hd_texture_pack_set_decode_budget(HdTexturePack* pack, evict_decode_cache(pack->decode); } -int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out) { - if (bytes_out) *bytes_out = 0; +int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_threads, + HdTexturePreloadResult* out) { + HdTexturePreloadResult r{}; + if (out) *out = r; #ifdef HD_TEXTURE_PACK_DISABLE_PNG_DECODE - (void)pack; + (void)pack; (void)max_bytes; (void)max_threads; return -1; #else if (!pack) return -1; try { - /* Decode every replacement before the game runs, across all cores. */ + namespace tp = psx_texpreload; std::vector> work_items; for (const auto& entry : pack->entries) if (!entry.second.ambiguous) work_items.emplace_back(entry.first, entry.second.replacement_path); + size_t per_image = 0; + { + std::lock_guard lock(pack->decode.mutex); + per_image = pack->decode.budget; + } std::vector> images(work_items.size()); - std::atomic next{0}; - auto work = [&] { - for (size_t k; (k = next++) < work_items.size();) { - std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); - const std::streamoff length = input ? static_cast(input.tellg()) : -1; - if (length <= 0 || static_cast(length) > kMaxPngFileBytes) continue; - std::vector encoded(static_cast(length)); - input.seekg(0); - if (!input.read(reinterpret_cast(encoded.data()), length)) continue; - int width = 0, height = 0, components = 0; - stbi_uc* loaded = stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), - &width, &height, &components, 4); - if (!loaded) continue; - if (width > 0 && height > 0) { - auto image = std::make_shared(); - image->width = static_cast(width); - image->height = static_cast(height); - image->rgba.assign(loaded, loaded + static_cast(width) * height * 4); - images[k] = std::move(image); - } - stbi_image_free(loaded); - } - }; - const unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); - std::vector pool; - for (unsigned t = 1; t < threads && t < work_items.size(); ++t) pool.emplace_back(work); - work(); - for (auto& t : pool) t.join(); + std::vector status(work_items.size(), tp::kPending); + tp::Budget budget(max_bytes); + r.worker_errors = static_cast(tp::run(work_items.size(), max_threads, [&](size_t k) { + status[k] = tp::kFailed; + std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); + const std::streamoff length = input ? static_cast(input.tellg()) : -1; + if (length <= 0 || static_cast(length) > kMaxPngFileBytes || + static_cast(length) > static_cast(std::numeric_limits::max())) + return; + std::vector encoded(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) return; + int width = 0, height = 0, components = 0; + if (!stbi_info_from_memory(encoded.data(), static_cast(encoded.size()), + &width, &height, &components) || + width <= 0 || height <= 0 || + static_cast(width) * static_cast(height) > + static_cast(per_image) / 4) + return; + const size_t need = static_cast(width) * static_cast(height) * 4; + if (!budget.reserve(need)) { status[k] = tp::kOverBudget; return; } + struct Reservation { + tp::Budget& b; size_t n; bool keep = false; + ~Reservation() { if (!keep) b.release(n); } + } reservation{budget, need}; + int lw = 0, lh = 0, lc = 0; + std::unique_ptr loaded( + stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), + &lw, &lh, &lc, 4), + stbi_image_free); + if (!loaded || lw != width || lh != height) return; + auto image = std::make_shared(); + image->width = static_cast(width); + image->height = static_cast(height); + image->rgba.assign(loaded.get(), loaded.get() + need); + images[k] = std::move(image); + reservation.keep = true; + status[k] = tp::kLoaded; + })); + r.requested = static_cast(work_items.size()); std::lock_guard lock(pack->decode.mutex); - int loaded = 0; size_t bytes = 0; for (size_t k = 0; k < work_items.size(); ++k) { + if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } + if (status[k] != tp::kLoaded) continue; + auto it = pack->decode.items.find(work_items[k].first); + if (it != pack->decode.items.end()) { + /* Already pinned, or an on-demand decode in flight: leave it. */ + if (it->second.pinned || it->second.state == DecodeState::Queued || + it->second.state == DecodeState::Loading) continue; + if (it->second.state == DecodeState::Ready) pack->decode.used -= it->second.bytes; + } DecodeItem& item = pack->decode.items[work_items[k].first]; item.path = work_items[k].second; - if (!images[k]) { item.state = DecodeState::Failed; continue; } - if (item.state == DecodeState::Ready) pack->decode.used -= item.bytes; item.bytes = images[k]->rgba.size(); item.image = std::move(images[k]); item.state = DecodeState::Ready; item.pinned = true; - pack->decode.used += item.bytes; bytes += item.bytes; ++loaded; + item.last_used = ++pack->decode.tick; + r.bytes += item.bytes; ++r.loaded; } - if (pack->decode.budget < pack->decode.used) pack->decode.budget = pack->decode.used; - if (bytes_out) *bytes_out = bytes; - return loaded; + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; + return r.loaded == r.requested ? 1 : 0; } catch (...) { return -1; } #endif } diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index f13c07d2..5a808774 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -1,6 +1,9 @@ #define STB_IMAGE_IMPLEMENTATION #include "../third_party/stb_image.h" #include "duckstation_texture_pack.h" +#include "texture_pack_preload.hpp" +#include +#include #include "../src/png_write.h" #include @@ -110,23 +113,78 @@ bool decode_fails(DuckTexturePack* pack, uint64_t id) { } void preload(const fs::path& root) { + namespace tp = psx_texpreload; const char* name = "texpage-P4-3631D2A04EC85311-9CBCDEA9305D7528-64x256-4-2-8x2-P0-15.png"; png(root/"replacements"/name,16,4); std::vector vram(1024*512); place(vram,65,258,2,words(4)); place(vram,512,400,16,words(16)); - auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); - size_t bytes = 0; - check(duck_texture_pack_preload(pack.get(),&bytes) == 1 && bytes == 16*4*4,"preload decodes the pack"); - DuckTextureMatch match{}; - check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"preloaded fixture matches"); - DuckTexturePixels pixels{}; - check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND && - pixels.width == 16,"preloaded image is resident without a decode request"); - duck_texture_pixels_release(&pixels); - duck_texture_pack_set_decode_budget(pack.get(),1); - check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, - "a preloaded image survives budget eviction"); - duck_texture_pixels_release(&pixels); + { + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r) == 1 && r.loaded == 1 && + r.requested == 1 && r.bytes == 16*4*4 && r.failed == 0,"preload decodes the pack"); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"preloaded fixture matches"); + DuckTexturePixels pixels{}; + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND && + pixels.width == 16,"preloaded image is resident without a decode request"); + duck_texture_pixels_release(&pixels); + duck_texture_pack_set_decode_budget(pack.get(),1); + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, + "a preloaded image survives budget eviction"); + duck_texture_pixels_release(&pixels); + } + { /* Over the aggregate cap: partial, nothing pinned, async path still decodes. */ + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),16,4,&r) == 0 && r.loaded == 0 && r.over_budget == 1 && + r.bytes == 0,"oversized image is skipped by the preload cap"); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"capped fixture matches"); + DuckTexturePixels pixels = decoded(pack.get(),match.entry_id); + check(pixels.width == 16,"an image the preload skipped decodes on demand"); + duck_texture_pixels_release(&pixels); + } + { /* Injected worker failure: caught, reported, image left to the async path. */ + auto pack = create(root); + tp::fault_mode() = 1; + HdTexturePreloadResult r{}; + const int status = duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r); + tp::fault_mode() = 0; + check(status == 0 && r.worker_errors == 1 && r.failed == 1 && r.loaded == 0, + "a throwing decode is caught and reported as a partial preload"); + } + { /* Malformed image: failed, not pinned, async decode reports failure. */ + std::ofstream(root/"replacements"/name,std::ios::binary|std::ios::trunc) << "not a png"; + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r) == 0 && r.failed == 1 && + r.worker_errors == 0,"malformed image fails the preload item"); + DuckTextureMatch match{}; + if (duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND) + check(decode_fails(pack.get(),match.entry_id),"malformed image still fails on demand"); + } +} + +void preload_pool() { + namespace tp = psx_texpreload; + std::atomic ran{0}; + tp::fault_mode() = 2; /* the second thread cannot be created */ + size_t errors = tp::run(32,8,[&](size_t){ ++ran; }); + check(errors == 0 && ran == 32,"thread creation failure lowers concurrency, all items still run"); + tp::fault_mode() = 1; /* the first item throws bad_alloc */ + ran = 0; + errors = tp::run(32,8,[&](size_t){ ++ran; }); + tp::fault_mode() = 0; + check(errors == 1 && ran == 31,"a throwing item is caught and counted"); + ran = 0; + errors = tp::run(32,8,[&](size_t k){ if (k % 4 == 0) throw std::runtime_error("x"); ++ran; }); + check(errors == 8 && ran == 24,"item exceptions never escape a pool thread"); + tp::Budget budget(100); + check(budget.reserve(60) && !budget.reserve(41) && budget.reserve(40) && !budget.reserve(1), + "aggregate budget reservations are bounded"); + budget.release(50); + check(budget.reserve(50),"released bytes are reusable"); } void fixed_hash_vectors() { @@ -624,7 +682,7 @@ int main(int argc, char** argv) { palettes(root/"palettes"); st_alias_duplicates(root/"keys"); failed_payloads(root/"invalid"); dumping(root/"dump"); official_binary_oracle(root/"official-oracle"); defaults_and_options(root/"defaults"); immutable_palettes_and_pages(root/"snapshots"); split_coalesce_copy_parts(root/"lifecycle"); copied_pack_ownership(root/"ownership"); - resident_checkpoint(root/"resident"); preload(root/"preload"); + resident_checkpoint(root/"resident"); preload(root/"preload"); preload_pool(); /* Every artifact is created beneath this fresh, explicitly named test * root. No user-provided paths participate in recursive cleanup. */ if (argc > 1 && !std::strcmp(argv[1],"--keep-artifacts")) std::printf("ARTIFACT_ROOT: %s\n",root.u8string().c_str()); diff --git a/runtime/tests/test_hd_texture_pack.cpp b/runtime/tests/test_hd_texture_pack.cpp index 6d952041..3165dd9f 100644 --- a/runtime/tests/test_hd_texture_pack.cpp +++ b/runtime/tests/test_hd_texture_pack.cpp @@ -1,6 +1,7 @@ #define STB_IMAGE_IMPLEMENTATION #include "../third_party/stb_image.h" #include "hd_texture_pack.h" +#include "texture_pack_preload.hpp" #include #include @@ -512,6 +513,54 @@ void test_bounded_async_decode(Fixture& fixture) { hd_texture_pixels_release(&pixels); } +void test_bounded_preload(Fixture& fixture) { + namespace tp = psx_texpreload; + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + HdTexturePreloadResult r{}; + /* One valid 64x8 PNG; the other keys are empty (malformed) files. */ + const int status = hd_texture_pack_preload(pack, size_t{64} << 20, 4, &r); + check(status >= 0 && r.loaded == 1 && r.bytes == 64u * 8u * 4u, + "preload pins the valid replacement"); + check(r.requested == r.loaded + r.failed + r.over_budget && + (r.requested == 1 ? status == 1 : (status == 0 && r.failed >= 1)), + "malformed replacements are reported, not counted as preloaded"); + hd_texture_pack_set_decode_budget(pack, 1); + check(hd_texture_pack_track_upload(pack, 10, 20, 8, 4, + fixture.upload_words.data(), + fixture.upload_words.size(), nullptr) == 1, + "preload fixture upload tracks"); + HdTexturePixels pixels{}; + check(hd_texture_pack_acquire_decoded(pack, fixture.upload_hash, fixture.palette4_hash, + &pixels) == HD_TEXTURE_LOOKUP_FOUND && + pixels.width == 64, + "a preloaded image is resident and survives budget eviction"); + hd_texture_pixels_release(&pixels); + hd_texture_pack_destroy(pack); + } + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + HdTexturePreloadResult r{}; + check(hd_texture_pack_preload(pack, 64, 4, &r) == 0 && r.loaded == 0 && + r.over_budget == 1 && r.bytes == 0, + "the aggregate cap skips an oversized image"); + hd_texture_pack_destroy(pack); + } + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + tp::fault_mode() = 1; + HdTexturePreloadResult r{}; + const int status = hd_texture_pack_preload(pack, size_t{64} << 20, 4, &r); + tp::fault_mode() = 0; + check(status == 0 && r.worker_errors == 1 && r.loaded + 1 + r.over_budget <= r.requested, + "an injected worker failure is caught and reported as partial"); + hd_texture_pack_destroy(pack); + } +} + } // namespace int main() { @@ -523,6 +572,7 @@ int main() { test_wrapping(fixture); test_ambiguity_fallback(); test_bounded_async_decode(fixture); + test_bounded_preload(fixture); if (failures) { std::fprintf(stderr, "test_hd_texture_pack: %d failure(s)\n", failures); From a7f7a4bed31ad59b2e0af4afaa169718dcccc36c Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Thu, 22 Oct 2026 17:03:10 -0700 Subject: [PATCH 3/4] Account retained preload bytes after publication failure --- runtime/include/texture_pack_preload.hpp | 6 +++++- runtime/src/duckstation_texture_pack.cpp | 21 ++++++++++++++----- runtime/src/hd_texture_pack.cpp | 20 ++++++++++++++---- .../tests/test_duckstation_texture_pack.cpp | 9 ++++++++ runtime/tests/test_hd_texture_pack.cpp | 11 ++++++++++ 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/runtime/include/texture_pack_preload.hpp b/runtime/include/texture_pack_preload.hpp index 6517802b..38b355ef 100644 --- a/runtime/include/texture_pack_preload.hpp +++ b/runtime/include/texture_pack_preload.hpp @@ -17,11 +17,15 @@ namespace psx_texpreload { /* Test-only fault injection. 1: the first item's decode throws - * std::bad_alloc. 2: creating the second pool thread throws std::system_error. */ + * std::bad_alloc. 2: creating the second pool thread throws std::system_error. + * 3: publication fails after the first pinned image. */ inline std::atomic& fault_mode() { static std::atomic mode{0}; return mode; } +inline void after_publish(size_t count) { + if (count == 1 && fault_mode().load() == 3) throw std::bad_alloc(); +} /* Aggregate decoded bytes, reserved before an image is decoded. */ class Budget { diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index 73ee9319..0b9ba668 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -1189,20 +1189,31 @@ int duck_texture_pack_preload(DuckTexturePack* p, size_t max_bytes, unsigned max /* Already pinned, or an on-demand decode in flight: leave it. */ if (it->second.pinned || it->second.status == DecodeStatus::Queued || it->second.status == DecodeStatus::Loading) continue; - if (it->second.image) w.used -= it->second.image->rgba.size(); } - Decode& entry = w.decoded[ids[k]]; + const size_t previous_bytes = it != w.decoded.end() && it->second.image + ? it->second.image->rgba.size() : 0; + Decode entry; + /* Prepare throwing metadata before touching an existing entry. */ entry.path = p->entries[ids[k] - 1].path; - r.bytes += images[k]->rgba.size(); entry.image = std::move(images[k]); entry.status = DecodeStatus::Ready; entry.pinned = true; entry.use = ++w.tick; + const size_t bytes = entry.image->rgba.size(); + if (it == w.decoded.end()) w.decoded.emplace(ids[k], std::move(entry)); + else it->second = std::move(entry); + w.used -= previous_bytes; + r.bytes += bytes; ++w.pinned_count; ++r.loaded; + ++w.decoded_images; + tp::after_publish(r.loaded); } - w.decoded_images += static_cast(r.loaded); r.failed = r.requested - r.loaded - r.over_budget; if (out) *out = r; return r.loaded == r.requested ? 1 : 0; - } catch (...) { return -1; } + } catch (...) { + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; /* Account the pinned prefix even on publication failure. */ + return -1; + } } int duck_texture_pack_acquire_decoded(DuckTexturePack* p, uint64_t id, DuckTexturePixels* out) { if (out) *out = {}; diff --git a/runtime/src/hd_texture_pack.cpp b/runtime/src/hd_texture_pack.cpp index ce3f0859..95f042d3 100644 --- a/runtime/src/hd_texture_pack.cpp +++ b/runtime/src/hd_texture_pack.cpp @@ -1363,20 +1363,32 @@ int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_ /* Already pinned, or an on-demand decode in flight: leave it. */ if (it->second.pinned || it->second.state == DecodeState::Queued || it->second.state == DecodeState::Loading) continue; - if (it->second.state == DecodeState::Ready) pack->decode.used -= it->second.bytes; } - DecodeItem& item = pack->decode.items[work_items[k].first]; + const size_t previous_bytes = it != pack->decode.items.end() && + it->second.state == DecodeState::Ready ? it->second.bytes : 0; + DecodeItem item; + /* Prepare throwing metadata before touching an existing entry. */ item.path = work_items[k].second; item.bytes = images[k]->rgba.size(); item.image = std::move(images[k]); item.state = DecodeState::Ready; item.pinned = true; item.last_used = ++pack->decode.tick; - r.bytes += item.bytes; ++r.loaded; + const size_t bytes = item.bytes; + if (it == pack->decode.items.end()) + pack->decode.items.emplace(work_items[k].first, std::move(item)); + else it->second = std::move(item); + pack->decode.used -= previous_bytes; + r.bytes += bytes; ++r.loaded; + tp::after_publish(r.loaded); } r.failed = r.requested - r.loaded - r.over_budget; if (out) *out = r; return r.loaded == r.requested ? 1 : 0; - } catch (...) { return -1; } + } catch (...) { + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; /* Account the pinned prefix even on publication failure. */ + return -1; + } #endif } diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index 5a808774..b1ff1186 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -154,6 +154,15 @@ void preload(const fs::path& root) { check(status == 0 && r.worker_errors == 1 && r.failed == 1 && r.loaded == 0, "a throwing decode is caught and reported as a partial preload"); } + { /* Publication can fail after retaining a pinned prefix. */ + auto pack = create(root); + tp::fault_mode() = 3; + HdTexturePreloadResult r{}; + const int status = duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r); + tp::fault_mode() = 0; + check(status == -1 && r.loaded == 1 && r.bytes == 16*4*4, + "publication failure still reports every retained byte"); + } { /* Malformed image: failed, not pinned, async decode reports failure. */ std::ofstream(root/"replacements"/name,std::ios::binary|std::ios::trunc) << "not a png"; auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); diff --git a/runtime/tests/test_hd_texture_pack.cpp b/runtime/tests/test_hd_texture_pack.cpp index 3165dd9f..73563dae 100644 --- a/runtime/tests/test_hd_texture_pack.cpp +++ b/runtime/tests/test_hd_texture_pack.cpp @@ -559,6 +559,17 @@ void test_bounded_preload(Fixture& fixture) { "an injected worker failure is caught and reported as partial"); hd_texture_pack_destroy(pack); } + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + tp::fault_mode() = 3; + HdTexturePreloadResult r{}; + const int status = hd_texture_pack_preload(pack, size_t{64} << 20, 4, &r); + tp::fault_mode() = 0; + check(status == -1 && r.loaded == 1 && r.bytes == 64u * 8u * 4u, + "publication failure still accounts the retained pinned image"); + hd_texture_pack_destroy(pack); + } } } // namespace From 214e5cdbe66c9e283a13d5e3e33e2b10e737d899 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 24 Oct 2026 08:31:16 -0700 Subject: [PATCH 4/4] Account existing pinned textures across preload retries Bound reservations against total retained residency, skip decoding pinned images, and report complete repeated loads truthfully. Cover cap reductions and publication retry in both formats. Tracking: beads-eio.3.386. --- runtime/include/hd_texture_pack.h | 9 +++-- runtime/src/duckstation_texture_pack.cpp | 33 +++++++++++++++---- runtime/src/hd_texture_pack.cpp | 25 +++++++++++--- .../tests/test_duckstation_texture_pack.cpp | 8 +++++ runtime/tests/test_hd_texture_pack.cpp | 11 +++++++ 5 files changed, 71 insertions(+), 15 deletions(-) diff --git a/runtime/include/hd_texture_pack.h b/runtime/include/hd_texture_pack.h index aa0c2ff2..9aa098cc 100644 --- a/runtime/include/hd_texture_pack.h +++ b/runtime/include/hd_texture_pack.h @@ -187,14 +187,17 @@ void hd_texture_pack_set_decode_budget(HdTexturePack* pack, * preloaded (over the cap, unreadable, malformed, a worker failure) are left * to the normal asynchronous, budgeted decode path; nothing is marked failed * by the preload. Returns 1 when every image was preloaded, 0 for a partial - * preload (see out), -1 on error. */ + * preload (see out), -1 on error. Repeated calls count already pinned images + * without reading/decoding them again; max_bytes caps the pack's total pinned + * residency. A cap smaller than existing residency is rejected with -1 and + * the existing residency reported; it does not discard live pinned images. */ typedef struct HdTexturePreloadResult { int requested; /* unambiguous replacements considered */ - int loaded; /* decoded and pinned */ + int loaded; /* resident and pinned, including earlier calls */ int over_budget; /* skipped: the aggregate max_bytes was reached */ int failed; /* unreadable, malformed, too large, or worker error */ int worker_errors; /* of failed: a decode that threw (caught) */ - size_t bytes; /* decoded RGBA pinned */ + size_t bytes; /* total decoded RGBA pinned by this pack */ } HdTexturePreloadResult; int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_threads, HdTexturePreloadResult* out); diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index 0b9ba668..9dfaa249 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -1142,17 +1142,37 @@ int duck_texture_pack_preload(DuckTexturePack* p, size_t max_bytes, unsigned max if (!p) return -1; try { namespace tp = psx_texpreload; + Worker& w = p->worker; std::vector ids; - for (size_t i = 0; i < p->entries.size(); ++i) - if (!p->entries[i].ambiguous) ids.push_back(i + 1); size_t per_image = 0; { - std::lock_guard lock(p->worker.mutex); - per_image = p->worker.budget; + std::lock_guard lock(w.mutex); + per_image = w.budget; + /* The cap and result describe the pack's total pinned residency, + * including earlier successful calls or publication prefixes. */ + for (const auto& item : w.decoded) + if (item.second.pinned && item.second.image) + r.bytes += item.second.image->rgba.size(); + for (size_t i = 0; i < p->entries.size(); ++i) { + if (p->entries[i].ambiguous) continue; + ++r.requested; + const auto it = w.decoded.find(i + 1); + if (it != w.decoded.end() && it->second.pinned) ++r.loaded; + } + if (r.bytes > max_bytes) { + r.failed = r.requested - r.loaded; + if (out) *out = r; + return -1; /* A smaller cap cannot discard live pinned data. */ + } + for (size_t i = 0; i < p->entries.size(); ++i) { + if (p->entries[i].ambiguous) continue; + const auto it = w.decoded.find(i + 1); + if (it == w.decoded.end() || !it->second.pinned) ids.push_back(i + 1); + } } std::vector> images(ids.size()); std::vector status(ids.size(), tp::kPending); - tp::Budget budget(max_bytes); + tp::Budget budget(max_bytes - r.bytes); r.worker_errors = static_cast(tp::run(ids.size(), max_threads, [&](size_t k) { status[k] = tp::kFailed; const std::string& path = p->entries[ids[k] - 1].path; @@ -1179,8 +1199,7 @@ int duck_texture_pack_preload(DuckTexturePack* p, size_t max_bytes, unsigned max reservation.keep = true; status[k] = tp::kLoaded; })); - r.requested = static_cast(ids.size()); - Worker& w = p->worker; std::lock_guard lock(w.mutex); + std::lock_guard lock(w.mutex); for (size_t k = 0; k < ids.size(); ++k) { if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } if (status[k] != tp::kLoaded) continue; diff --git a/runtime/src/hd_texture_pack.cpp b/runtime/src/hd_texture_pack.cpp index 95f042d3..225c2b37 100644 --- a/runtime/src/hd_texture_pack.cpp +++ b/runtime/src/hd_texture_pack.cpp @@ -1305,17 +1305,33 @@ int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_ try { namespace tp = psx_texpreload; std::vector> work_items; - for (const auto& entry : pack->entries) - if (!entry.second.ambiguous) - work_items.emplace_back(entry.first, entry.second.replacement_path); size_t per_image = 0; { std::lock_guard lock(pack->decode.mutex); per_image = pack->decode.budget; + for (const auto& item : pack->decode.items) + if (item.second.pinned) r.bytes += item.second.bytes; + for (const auto& entry : pack->entries) { + if (entry.second.ambiguous) continue; + ++r.requested; + const auto it = pack->decode.items.find(entry.first); + if (it != pack->decode.items.end() && it->second.pinned) ++r.loaded; + } + if (r.bytes > max_bytes) { + r.failed = r.requested - r.loaded; + if (out) *out = r; + return -1; + } + for (const auto& entry : pack->entries) { + if (entry.second.ambiguous) continue; + const auto it = pack->decode.items.find(entry.first); + if (it == pack->decode.items.end() || !it->second.pinned) + work_items.emplace_back(entry.first, entry.second.replacement_path); + } } std::vector> images(work_items.size()); std::vector status(work_items.size(), tp::kPending); - tp::Budget budget(max_bytes); + tp::Budget budget(max_bytes - r.bytes); r.worker_errors = static_cast(tp::run(work_items.size(), max_threads, [&](size_t k) { status[k] = tp::kFailed; std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); @@ -1353,7 +1369,6 @@ int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_ reservation.keep = true; status[k] = tp::kLoaded; })); - r.requested = static_cast(work_items.size()); std::lock_guard lock(pack->decode.mutex); for (size_t k = 0; k < work_items.size(); ++k) { if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index b1ff1186..4a23ed53 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -130,6 +130,11 @@ void preload(const fs::path& root) { pixels.width == 16,"preloaded image is resident without a decode request"); duck_texture_pixels_release(&pixels); duck_texture_pack_set_decode_budget(pack.get(),1); + check(duck_texture_pack_preload(pack.get(),256,1,&r) == 1 && r.loaded == 1 && + r.failed == 0 && r.bytes == 256 && r.worker_errors == 0, + "repeat counts existing residency without decoding under the reduced budget"); + check(duck_texture_pack_preload(pack.get(),255,1,&r) == -1 && r.bytes == 256, + "a smaller lifetime cap rejects without hiding pinned residency"); check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, "a preloaded image survives budget eviction"); duck_texture_pixels_release(&pixels); @@ -162,6 +167,9 @@ void preload(const fs::path& root) { tp::fault_mode() = 0; check(status == -1 && r.loaded == 1 && r.bytes == 16*4*4, "publication failure still reports every retained byte"); + check(duck_texture_pack_preload(pack.get(),256,1,&r) == 1 && + r.loaded == 1 && r.failed == 0 && r.bytes == 256, + "publication retry accounts the retained prefix without decoding again"); } { /* Malformed image: failed, not pinned, async decode reports failure. */ std::ofstream(root/"replacements"/name,std::ios::binary|std::ios::trunc) << "not a png"; diff --git a/runtime/tests/test_hd_texture_pack.cpp b/runtime/tests/test_hd_texture_pack.cpp index 73563dae..383eeb13 100644 --- a/runtime/tests/test_hd_texture_pack.cpp +++ b/runtime/tests/test_hd_texture_pack.cpp @@ -527,6 +527,14 @@ void test_bounded_preload(Fixture& fixture) { (r.requested == 1 ? status == 1 : (status == 0 && r.failed >= 1)), "malformed replacements are reported, not counted as preloaded"); hd_texture_pack_set_decode_budget(pack, 1); + HdTexturePreloadResult repeat{}; + check(hd_texture_pack_preload(pack,64u*8u*4u,1,&repeat) >= 0 && + repeat.loaded == 1 && repeat.bytes == 64u*8u*4u && + repeat.failed == repeat.requested - 1, + "repeat counts pinned images without decoding them under a reduced budget"); + check(hd_texture_pack_preload(pack,64u*8u*4u-1,1,&repeat) == -1 && + repeat.bytes == 64u*8u*4u, + "a smaller lifetime cap rejects and reports retained residency"); check(hd_texture_pack_track_upload(pack, 10, 20, 8, 4, fixture.upload_words.data(), fixture.upload_words.size(), nullptr) == 1, @@ -568,6 +576,9 @@ void test_bounded_preload(Fixture& fixture) { tp::fault_mode() = 0; check(status == -1 && r.loaded == 1 && r.bytes == 64u * 8u * 4u, "publication failure still accounts the retained pinned image"); + check(hd_texture_pack_preload(pack,64u*8u*4u,1,&r) >= 0 && + r.loaded == 1 && r.bytes == 64u*8u*4u && r.failed == r.requested-1, + "publication retry accounts existing pinned residency"); hd_texture_pack_destroy(pack); } }