From 954e28b051b039b6a7bbc4f9fe0c7faef9da3bf9 Mon Sep 17 00:00:00 2001 From: Shokunin <30949000+tetrisgm@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:27:29 -0700 Subject: [PATCH] HD preload: bounded contract, safe threads, explicit partial result Both preload functions now take an aggregate decoded-byte cap and a thread cap (runtime: 512 MiB, 4 threads). Each image is probed and its RGBA reserved against the cap before decoding; per-image limits follow the pack's decode budget, which is no longer grown to usage (pinned images sit outside it, and outside the Duck 512-entry cap). The pool (texture_pack_ preload.hpp) joins every started thread on unwinding, survives thread creation failure, and catches worker exceptions. Images not preloaded are left to the asynchronous budgeted path instead of being marked failed, and HdTexturePreloadResult reports loaded / over-cap / failed / worker errors with a complete-or-partial status. The Beetle async worker no longer lets allocation failures escape its thread entry. Tests cover both formats: complete preload, over-cap skip with on-demand fallback, malformed images, injected worker and thread-creation failures, and budget reservation bounds. Co-Authored-By: Claude Opus 5.5 --- runtime/include/duckstation_texture_pack.h | 10 +- runtime/include/hd_texture_pack.h | 21 ++- runtime/include/texture_pack_preload.hpp | 96 ++++++++++ runtime/src/duckstation_texture_pack.cpp | 106 ++++++----- runtime/src/gpu_hd_textures.cpp | 32 +++- runtime/src/hd_texture_pack.cpp | 171 +++++++++++------- .../tests/test_duckstation_texture_pack.cpp | 86 +++++++-- runtime/tests/test_hd_texture_pack.cpp | 50 +++++ 8 files changed, 433 insertions(+), 139 deletions(-) create mode 100644 runtime/include/texture_pack_preload.hpp diff --git a/runtime/include/duckstation_texture_pack.h b/runtime/include/duckstation_texture_pack.h index b552e914..d0569139 100644 --- a/runtime/include/duckstation_texture_pack.h +++ b/runtime/include/duckstation_texture_pack.h @@ -170,10 +170,12 @@ int duck_texture_pack_match_parts(DuckTexturePack* pack, * Release the lease even after pack destruction. */ void duck_texture_pack_set_decode_budget(DuckTexturePack* pack, size_t bytes); int duck_texture_pack_request_decode(DuckTexturePack* pack, uint64_t entry_id); -/* Decode every unambiguous replacement now (all cores) and pin it in RAM, so - * no draw waits for or evicts an image. Returns the count loaded, -1 on error; - * bytes_out gets the decoded RGBA total. */ -int duck_texture_pack_preload(DuckTexturePack* pack, size_t* bytes_out); +/* Elective full preload with the hd_texture_pack_preload contract: bounded + * aggregate decoded bytes and concurrency, pinned images outside the decode + * budget, everything not preloaded left to the asynchronous decode path. + * Returns 1 complete, 0 partial (see out), -1 on error. */ +int duck_texture_pack_preload(DuckTexturePack* pack, size_t max_bytes, + unsigned max_threads, HdTexturePreloadResult* out); int duck_texture_pack_acquire_decoded(DuckTexturePack* pack, uint64_t entry_id, DuckTexturePixels* out_pixels); void duck_texture_pixels_release(DuckTexturePixels* pixels); diff --git a/runtime/include/hd_texture_pack.h b/runtime/include/hd_texture_pack.h index da8ad462..165adb63 100644 --- a/runtime/include/hd_texture_pack.h +++ b/runtime/include/hd_texture_pack.h @@ -189,9 +189,24 @@ typedef struct HdTexturePixels { void hd_texture_pack_set_decode_budget(HdTexturePack* pack, size_t budget_bytes); -/* Decode every unambiguous replacement now (all cores) and pin it in RAM. - * Returns the count loaded, -1 on error; bytes_out gets the RGBA total. */ -int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out); +/* Elective full preload: decode unambiguous replacements up front and pin + * them, bounded by max_bytes of decoded RGBA in aggregate and by max_threads + * concurrent decodes (the caller's thread included). Pinned images are kept + * outside the on-demand decode budget, which stays as configured. Images not + * preloaded (over the cap, unreadable, malformed, a worker failure) are left + * to the normal asynchronous, budgeted decode path; nothing is marked failed + * by the preload. Returns 1 when every image was preloaded, 0 for a partial + * preload (see out), -1 on error. */ +typedef struct HdTexturePreloadResult { + int requested; /* unambiguous replacements considered */ + int loaded; /* decoded and pinned */ + int over_budget; /* skipped: the aggregate max_bytes was reached */ + int failed; /* unreadable, malformed, too large, or worker error */ + int worker_errors; /* of failed: a decode that threw (caught) */ + size_t bytes; /* decoded RGBA pinned */ +} HdTexturePreloadResult; +int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, + unsigned max_threads, HdTexturePreloadResult* out); int hd_texture_pack_request_decode(HdTexturePack* pack, uint32_t texture_hash, uint32_t palette_hash); diff --git a/runtime/include/texture_pack_preload.hpp b/runtime/include/texture_pack_preload.hpp new file mode 100644 index 00000000..6517802b --- /dev/null +++ b/runtime/include/texture_pack_preload.hpp @@ -0,0 +1,96 @@ +#ifndef PSX_TEXTURE_PACK_PRELOAD_HPP +#define PSX_TEXTURE_PACK_PRELOAD_HPP + +/* Shared machinery for the elective full texture-pack preload + * (hd_texture_pack_preload, duck_texture_pack_preload): an aggregate + * decoded-byte reservation and a bounded worker pool that can neither leak a + * joinable std::thread nor let an exception escape a thread entry. */ + +#include +#include +#include +#include +#include +#include +#include + +namespace psx_texpreload { + +/* Test-only fault injection. 1: the first item's decode throws + * std::bad_alloc. 2: creating the second pool thread throws std::system_error. */ +inline std::atomic& fault_mode() { + static std::atomic mode{0}; + return mode; +} + +/* Aggregate decoded bytes, reserved before an image is decoded. */ +class Budget { +public: + explicit Budget(size_t cap) : cap_(cap) {} + bool reserve(size_t bytes) { + size_t cur = used_.load(); + do { + if (bytes > cap_ || cur > cap_ - bytes) return false; + } while (!used_.compare_exchange_weak(cur, cur + bytes)); + return true; + } + void release(size_t bytes) { used_.fetch_sub(bytes); } +private: + size_t cap_; + std::atomic used_{0}; +}; + +/* Per-item outcome, written by exactly one worker. */ +enum ItemStatus : unsigned char { kPending = 0, kLoaded, kFailed, kOverBudget }; + +/* Joins every started thread when the pool goes out of scope, including while + * unwinding from a failed std::thread construction. */ +struct Joiner { + std::vector& threads; + ~Joiner() { + for (auto& t : threads) + if (t.joinable()) t.join(); + } +}; + +/* Runs item(k) for every k < n on at most max_threads threads, the caller + * being one of them. An exception from item(k) is caught and counted, never + * propagated out of a thread entry. A failure to start a thread only lowers + * concurrency. Returns the number of items whose item(k) threw. */ +template +size_t run(size_t n, unsigned max_threads, F&& item) { + std::atomic next{0}, errors{0}; + auto work = [&]() noexcept { + for (size_t k; (k = next.fetch_add(1)) < n;) { + try { + if (k == 0 && fault_mode().load() == 1) throw std::bad_alloc(); + item(k); + } catch (...) { + errors.fetch_add(1); + } + } + }; + unsigned hw = std::thread::hardware_concurrency(); + if (hw == 0) hw = 1; + const unsigned threads = std::max(1u, std::min(max_threads ? max_threads : 1u, hw)); + std::vector pool; + { + Joiner joiner{pool}; + for (unsigned t = 1; t < threads && t < n; ++t) { + try { + if (t == 1 && fault_mode().load() == 2) + throw std::system_error( + std::make_error_code(std::errc::resource_unavailable_try_again)); + pool.emplace_back(work); + } catch (...) { + break; + } + } + work(); + } + return errors.load(); +} + +} // namespace psx_texpreload + +#endif diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index 9aa8206f..ceb138ee 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -1,5 +1,6 @@ #include "duckstation_texture_pack.h" #include "texture_image_decode.h" +#include "texture_pack_preload.hpp" #define XXH_INLINE_ALL #include "../third_party/xxhash.h" #include "png_write.h" @@ -232,6 +233,7 @@ struct Worker { size_t dump_bytes = 0; uint64_t tick = 0; uint64_t decoded_images = 0, decode_evictions = 0; + size_t pinned_count = 0; /* preloaded entries: outside the 512-entry cap */ bool stop = false; bool dump_busy = false; std::string dump_error; @@ -1142,7 +1144,7 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { } if (!w.budget) return HD_TEXTURE_LOOKUP_ERROR; if (w.decode_queue.size() >= kMaxDecodeQueue) return HD_TEXTURE_LOOKUP_NONE; - if (w.decoded.size() >= 512) { + if (w.decoded.size() - w.pinned_count >= 512) { auto victim = w.decoded.end(); for (auto item = w.decoded.begin(); item != w.decoded.end(); ++item) if ((item->second.status == DecodeStatus::Ready || item->second.status == DecodeStatus::Failed) && @@ -1156,53 +1158,73 @@ int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { w.wake.notify_one(); return HD_TEXTURE_LOOKUP_NONE; } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } } -int duck_texture_pack_preload(DuckTexturePack* p, size_t* bytes_out) { - if (bytes_out) *bytes_out = 0; +int duck_texture_pack_preload(DuckTexturePack* p, size_t max_bytes, unsigned max_threads, + HdTexturePreloadResult* out) { + HdTexturePreloadResult r{}; + if (out) *out = r; if (!p) return -1; try { - /* Decode every replacement before the game runs, across all cores, - * so no draw ever waits for (or misses) an image mid-game. */ + namespace tp = psx_texpreload; std::vector ids; for (size_t i = 0; i < p->entries.size(); ++i) if (!p->entries[i].ambiguous) ids.push_back(i + 1); - std::vector> images(ids.size()); - std::atomic next{0}; - auto work = [&] { - for (size_t k; (k = next++) < ids.size();) { - try { - const std::string& path = p->entries[ids[k] - 1].path; - std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); - const auto length = input ? input.tellg() : std::streampos(-1); - if (length <= 0 || static_cast(length) > kMaxEncoded) continue; - std::vector encoded(static_cast(length)); - input.seekg(0); - DecodedTextureImage decoded; std::string error; - if (!input.read(reinterpret_cast(encoded.data()), length) || - !texture_image_decode(encoded.data(), encoded.size(), SIZE_MAX, decoded, error)) continue; - auto image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; - image->rgba = std::move(decoded.rgba); images[k] = std::move(image); - } catch (...) {} - } - }; - unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); - std::vector pool; - for (unsigned t = 1; t < threads && t < ids.size(); ++t) pool.emplace_back(work); - work(); - for (auto& t : pool) t.join(); - Worker& w = p->worker; std::lock_guard lock(w.mutex); - int loaded = 0; size_t bytes = 0; - for (size_t k = 0; k < ids.size(); ++k) { - Decode& d = w.decoded[ids[k]]; - d.path = p->entries[ids[k] - 1].path; - if (!images[k]) { d.status = DecodeStatus::Failed; continue; } - if (d.image) w.used -= d.image->rgba.size(); - bytes += images[k]->rgba.size(); w.used += images[k]->rgba.size(); - d.image = std::move(images[k]); d.status = DecodeStatus::Ready; d.pinned = true; ++loaded; + size_t per_image = 0; + { + std::lock_guard lock(p->worker.mutex); + per_image = p->worker.budget; } - if (w.budget < w.used) w.budget = w.used; - w.decoded_images += loaded; - if (bytes_out) *bytes_out = bytes; - return loaded; + std::vector> images(ids.size()); + std::vector status(ids.size(), tp::kPending); + tp::Budget budget(max_bytes); + r.worker_errors = static_cast(tp::run(ids.size(), max_threads, [&](size_t k) { + status[k] = tp::kFailed; + const std::string& path = p->entries[ids[k] - 1].path; + std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); + const auto length = input ? input.tellg() : std::streampos(-1); + if (length <= 0 || static_cast(length) > kMaxEncoded) return; + std::vector encoded(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) return; + TextureImageProbe probe; std::string error; + if (!texture_image_probe(encoded.data(), encoded.size(), per_image, probe, error)) return; + const size_t need = static_cast(probe.width) * probe.height * 4u; + if (!budget.reserve(need)) { status[k] = tp::kOverBudget; return; } + struct Reservation { + tp::Budget& b; size_t n; bool keep = false; + ~Reservation() { if (!keep) b.release(n); } + } reservation{budget, need}; + DecodedTextureImage decoded; + if (!texture_image_decode(encoded.data(), encoded.size(), need, decoded, error) || + decoded.rgba.size() != need) return; + auto image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; + image->rgba = std::move(decoded.rgba); + images[k] = std::move(image); + reservation.keep = true; + status[k] = tp::kLoaded; + })); + r.requested = static_cast(ids.size()); + Worker& w = p->worker; std::lock_guard lock(w.mutex); + for (size_t k = 0; k < ids.size(); ++k) { + if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } + if (status[k] != tp::kLoaded) continue; + auto it = w.decoded.find(ids[k]); + if (it != w.decoded.end()) { + /* Already pinned, or an on-demand decode in flight: leave it. */ + if (it->second.pinned || it->second.status == DecodeStatus::Queued || + it->second.status == DecodeStatus::Loading) continue; + if (it->second.image) w.used -= it->second.image->rgba.size(); + } + Decode& entry = w.decoded[ids[k]]; + entry.path = p->entries[ids[k] - 1].path; + r.bytes += images[k]->rgba.size(); + entry.image = std::move(images[k]); entry.status = DecodeStatus::Ready; + entry.pinned = true; entry.use = ++w.tick; + ++w.pinned_count; ++r.loaded; + } + w.decoded_images += static_cast(r.loaded); + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; + return r.loaded == r.requested ? 1 : 0; } catch (...) { return -1; } } int duck_texture_pack_acquire_decoded(DuckTexturePack* p, uint64_t id, DuckTexturePixels* out) { diff --git a/runtime/src/gpu_hd_textures.cpp b/runtime/src/gpu_hd_textures.cpp index 036da2c4..5430dfb3 100644 --- a/runtime/src/gpu_hd_textures.cpp +++ b/runtime/src/gpu_hd_textures.cpp @@ -26,6 +26,11 @@ constexpr size_t kCompositionBudget = 64u * 1024u * 1024u; constexpr size_t kCompositionCount = 256; constexpr uint32_t kCompositionDimension = 8192; constexpr size_t kMaxParts = 64; +/* Elective full preload of an installed pack: at most this much decoded RGBA + * pinned across both formats, decoded by at most this many threads. Anything + * beyond decodes on demand under the normal budget. */ +constexpr size_t kPreloadMaxBytes = size_t{512} * 1024u * 1024u; +constexpr unsigned kPreloadThreads = 4; struct Composition { std::vector signature; std::vector rgba; @@ -291,19 +296,30 @@ extern "C" int gpu_hd_textures_configure(const char* root, int replacements, } if (!duck_texture_pack_create(root, &next->duck, error, capacity)) return 0; if (next->replacements) { - /* Every replacement decoded and resident before the game draws. */ + /* Replacements decoded and resident before the game draws, within + * kPreloadMaxBytes; the rest stays on the budgeted async path. */ const auto t0 = std::chrono::steady_clock::now(); - size_t bytes = 0, beetle_bytes = 0; - int loaded = duck_texture_pack_preload(next->duck, &bytes); + HdTexturePreloadResult dr{}, br{}; + const int dstatus = duck_texture_pack_preload(next->duck, kPreloadMaxBytes, + kPreloadThreads, &dr); + int bstatus = 1; if (next->beetle) { - const int beetle_loaded = hd_texture_pack_preload(next->beetle, &beetle_bytes); - if (beetle_loaded > 0) { loaded = (loaded > 0 ? loaded : 0) + beetle_loaded; bytes += beetle_bytes; } + const size_t left = dr.bytes < kPreloadMaxBytes ? kPreloadMaxBytes - dr.bytes : 0; + bstatus = hd_texture_pack_preload(next->beetle, left, kPreloadThreads, &br); } const auto ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0).count(); - if (loaded > 0) - std::fprintf(stdout, "psxrecomp: texture pack: %d images preloaded (%.1f MiB) in %lld ms\n", - loaded, bytes / 1048576.0, static_cast(ms)); + const int requested = dr.requested + br.requested; + if (requested > 0 || dstatus < 0 || bstatus < 0) + std::fprintf(stdout, "psxrecomp: texture pack: preloaded %d/%d images (%.1f MiB) in %lld ms" + "%s; %d over the %u MiB preload cap, %d failed (%d worker errors);" + " the rest decode on demand\n", + dr.loaded + br.loaded, requested, (dr.bytes + br.bytes) / 1048576.0, + static_cast(ms), + (dstatus < 0 || bstatus < 0) ? " (preload error)" : + (dstatus == 1 && bstatus == 1) ? " (complete)" : " (partial)", + dr.over_budget + br.over_budget, static_cast(kPreloadMaxBytes / (1024u * 1024u)), + dr.failed + br.failed, dr.worker_errors + br.worker_errors); } /* Quiesce renderer users (GL render thread: dump_draw appends to the * old session's sources) before flushing its dumps or reading its diff --git a/runtime/src/hd_texture_pack.cpp b/runtime/src/hd_texture_pack.cpp index 062069dc..af7b6874 100644 --- a/runtime/src/hd_texture_pack.cpp +++ b/runtime/src/hd_texture_pack.cpp @@ -1,4 +1,5 @@ #include "hd_texture_pack.h" +#include "texture_pack_preload.hpp" #ifndef HD_TEXTURE_PACK_DISABLE_PNG_DECODE #include "../third_party/stb_image.h" /* declarations only; implementation is shared */ @@ -517,38 +518,46 @@ void decode_worker(DecodeCache* cache) { path = it->second.path; } - std::vector encoded; - { - std::ifstream input(path, std::ios::binary | std::ios::ate); - std::streamoff length = -1; - if (input) length = static_cast(input.tellg()); - if (length > 0 && static_cast(length) <= kMaxPngFileBytes && - static_cast(length) <= - static_cast(std::numeric_limits::max())) { - encoded.resize(static_cast(length)); - input.seekg(0); - if (!input.read(reinterpret_cast(encoded.data()), length)) - encoded.clear(); - } - } - int width = 0; - int height = 0; - int components = 0; - stbi_uc* loaded = encoded.empty() ? nullptr : stbi_load_from_memory( - encoded.data(), static_cast(encoded.size()), - &width, &height, &components, 4); std::shared_ptr decoded; size_t bytes = 0; - if (loaded && width > 0 && height > 0 && - static_cast(width) * static_cast(height) <= - std::numeric_limits::max() / 4) { - bytes = static_cast(width) * static_cast(height) * 4; - decoded = std::make_shared(); - decoded->width = static_cast(width); - decoded->height = static_cast(height); - decoded->rgba.assign(loaded, loaded + bytes); + /* Allocation failure (encoded buffer, RGBA copy) must not escape the + * thread entry: the item fails like an undecodable image. */ + try { + std::vector encoded; + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + std::streamoff length = -1; + if (input) length = static_cast(input.tellg()); + if (length > 0 && static_cast(length) <= kMaxPngFileBytes && + static_cast(length) <= + static_cast(std::numeric_limits::max())) { + encoded.resize(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) + encoded.clear(); + } + } + int width = 0; + int height = 0; + int components = 0; + std::unique_ptr loaded( + encoded.empty() ? nullptr : stbi_load_from_memory( + encoded.data(), static_cast(encoded.size()), + &width, &height, &components, 4), + stbi_image_free); + if (loaded && width > 0 && height > 0 && + static_cast(width) * static_cast(height) <= + std::numeric_limits::max() / 4) { + bytes = static_cast(width) * static_cast(height) * 4; + decoded = std::make_shared(); + decoded->width = static_cast(width); + decoded->height = static_cast(height); + decoded->rgba.assign(loaded.get(), loaded.get() + bytes); + } + } catch (...) { + decoded.reset(); + bytes = 0; } - if (loaded) stbi_image_free(loaded); std::lock_guard lock(cache->mutex); auto it = cache->items.find(key); @@ -1319,63 +1328,89 @@ void hd_texture_pack_set_decode_budget(HdTexturePack* pack, evict_decode_cache(pack->decode); } -int hd_texture_pack_preload(HdTexturePack* pack, size_t* bytes_out) { - if (bytes_out) *bytes_out = 0; +int hd_texture_pack_preload(HdTexturePack* pack, size_t max_bytes, unsigned max_threads, + HdTexturePreloadResult* out) { + HdTexturePreloadResult r{}; + if (out) *out = r; #ifdef HD_TEXTURE_PACK_DISABLE_PNG_DECODE - (void)pack; + (void)pack; (void)max_bytes; (void)max_threads; return -1; #else if (!pack) return -1; try { - /* Decode every replacement before the game runs, across all cores. */ + namespace tp = psx_texpreload; std::vector> work_items; for (const auto& entry : pack->entries) if (!entry.second.ambiguous) work_items.emplace_back(entry.first, entry.second.replacement_path); + size_t per_image = 0; + { + std::lock_guard lock(pack->decode.mutex); + per_image = pack->decode.budget; + } std::vector> images(work_items.size()); - std::atomic next{0}; - auto work = [&] { - for (size_t k; (k = next++) < work_items.size();) { - std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); - const std::streamoff length = input ? static_cast(input.tellg()) : -1; - if (length <= 0 || static_cast(length) > kMaxPngFileBytes) continue; - std::vector encoded(static_cast(length)); - input.seekg(0); - if (!input.read(reinterpret_cast(encoded.data()), length)) continue; - int width = 0, height = 0, components = 0; - stbi_uc* loaded = stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), - &width, &height, &components, 4); - if (!loaded) continue; - if (width > 0 && height > 0) { - auto image = std::make_shared(); - image->width = static_cast(width); - image->height = static_cast(height); - image->rgba.assign(loaded, loaded + static_cast(width) * height * 4); - images[k] = std::move(image); - } - stbi_image_free(loaded); - } - }; - const unsigned threads = std::max(1u, std::min(std::thread::hardware_concurrency(), 16u)); - std::vector pool; - for (unsigned t = 1; t < threads && t < work_items.size(); ++t) pool.emplace_back(work); - work(); - for (auto& t : pool) t.join(); + std::vector status(work_items.size(), tp::kPending); + tp::Budget budget(max_bytes); + r.worker_errors = static_cast(tp::run(work_items.size(), max_threads, [&](size_t k) { + status[k] = tp::kFailed; + std::ifstream input(work_items[k].second, std::ios::binary | std::ios::ate); + const std::streamoff length = input ? static_cast(input.tellg()) : -1; + if (length <= 0 || static_cast(length) > kMaxPngFileBytes || + static_cast(length) > static_cast(std::numeric_limits::max())) + return; + std::vector encoded(static_cast(length)); + input.seekg(0); + if (!input.read(reinterpret_cast(encoded.data()), length)) return; + int width = 0, height = 0, components = 0; + if (!stbi_info_from_memory(encoded.data(), static_cast(encoded.size()), + &width, &height, &components) || + width <= 0 || height <= 0 || + static_cast(width) * static_cast(height) > + static_cast(per_image) / 4) + return; + const size_t need = static_cast(width) * static_cast(height) * 4; + if (!budget.reserve(need)) { status[k] = tp::kOverBudget; return; } + struct Reservation { + tp::Budget& b; size_t n; bool keep = false; + ~Reservation() { if (!keep) b.release(n); } + } reservation{budget, need}; + int lw = 0, lh = 0, lc = 0; + std::unique_ptr loaded( + stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), + &lw, &lh, &lc, 4), + stbi_image_free); + if (!loaded || lw != width || lh != height) return; + auto image = std::make_shared(); + image->width = static_cast(width); + image->height = static_cast(height); + image->rgba.assign(loaded.get(), loaded.get() + need); + images[k] = std::move(image); + reservation.keep = true; + status[k] = tp::kLoaded; + })); + r.requested = static_cast(work_items.size()); std::lock_guard lock(pack->decode.mutex); - int loaded = 0; size_t bytes = 0; for (size_t k = 0; k < work_items.size(); ++k) { + if (status[k] == tp::kOverBudget) { ++r.over_budget; continue; } + if (status[k] != tp::kLoaded) continue; + auto it = pack->decode.items.find(work_items[k].first); + if (it != pack->decode.items.end()) { + /* Already pinned, or an on-demand decode in flight: leave it. */ + if (it->second.pinned || it->second.state == DecodeState::Queued || + it->second.state == DecodeState::Loading) continue; + if (it->second.state == DecodeState::Ready) pack->decode.used -= it->second.bytes; + } DecodeItem& item = pack->decode.items[work_items[k].first]; item.path = work_items[k].second; - if (!images[k]) { item.state = DecodeState::Failed; continue; } - if (item.state == DecodeState::Ready) pack->decode.used -= item.bytes; item.bytes = images[k]->rgba.size(); item.image = std::move(images[k]); item.state = DecodeState::Ready; item.pinned = true; - pack->decode.used += item.bytes; bytes += item.bytes; ++loaded; + item.last_used = ++pack->decode.tick; + r.bytes += item.bytes; ++r.loaded; } - if (pack->decode.budget < pack->decode.used) pack->decode.budget = pack->decode.used; - if (bytes_out) *bytes_out = bytes; - return loaded; + r.failed = r.requested - r.loaded - r.over_budget; + if (out) *out = r; + return r.loaded == r.requested ? 1 : 0; } catch (...) { return -1; } #endif } diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index a67daae3..7e894cc2 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -1,6 +1,9 @@ #define STB_IMAGE_IMPLEMENTATION #include "../third_party/stb_image.h" #include "duckstation_texture_pack.h" +#include "texture_pack_preload.hpp" +#include +#include #include "../src/png_write.h" #include @@ -110,23 +113,78 @@ bool decode_fails(DuckTexturePack* pack, uint64_t id) { } void preload(const fs::path& root) { + namespace tp = psx_texpreload; const char* name = "texpage-P4-3631D2A04EC85311-9CBCDEA9305D7528-64x256-4-2-8x2-P0-15.png"; png(root/"replacements"/name,16,4); std::vector vram(1024*512); place(vram,65,258,2,words(4)); place(vram,512,400,16,words(16)); - auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); - size_t bytes = 0; - check(duck_texture_pack_preload(pack.get(),&bytes) == 1 && bytes == 16*4*4,"preload decodes the pack"); - DuckTextureMatch match{}; - check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"preloaded fixture matches"); - DuckTexturePixels pixels{}; - check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND && - pixels.width == 16,"preloaded image is resident without a decode request"); - duck_texture_pixels_release(&pixels); - duck_texture_pack_set_decode_budget(pack.get(),1); - check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, - "a preloaded image survives budget eviction"); - duck_texture_pixels_release(&pixels); + { + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r) == 1 && r.loaded == 1 && + r.requested == 1 && r.bytes == 16*4*4 && r.failed == 0,"preload decodes the pack"); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"preloaded fixture matches"); + DuckTexturePixels pixels{}; + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND && + pixels.width == 16,"preloaded image is resident without a decode request"); + duck_texture_pixels_release(&pixels); + duck_texture_pack_set_decode_budget(pack.get(),1); + check(duck_texture_pack_acquire_decoded(pack.get(),match.entry_id,&pixels) == HD_TEXTURE_LOOKUP_FOUND, + "a preloaded image survives budget eviction"); + duck_texture_pixels_release(&pixels); + } + { /* Over the aggregate cap: partial, nothing pinned, async path still decodes. */ + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),16,4,&r) == 0 && r.loaded == 0 && r.over_budget == 1 && + r.bytes == 0,"oversized image is skipped by the preload cap"); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"capped fixture matches"); + DuckTexturePixels pixels = decoded(pack.get(),match.entry_id); + check(pixels.width == 16,"an image the preload skipped decodes on demand"); + duck_texture_pixels_release(&pixels); + } + { /* Injected worker failure: caught, reported, image left to the async path. */ + auto pack = create(root); + tp::fault_mode() = 1; + HdTexturePreloadResult r{}; + const int status = duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r); + tp::fault_mode() = 0; + check(status == 0 && r.worker_errors == 1 && r.failed == 1 && r.loaded == 0, + "a throwing decode is caught and reported as a partial preload"); + } + { /* Malformed image: failed, not pinned, async decode reports failure. */ + std::ofstream(root/"replacements"/name,std::ios::binary|std::ios::trunc) << "not a png"; + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + HdTexturePreloadResult r{}; + check(duck_texture_pack_preload(pack.get(),size_t(64)<<20,4,&r) == 0 && r.failed == 1 && + r.worker_errors == 0,"malformed image fails the preload item"); + DuckTextureMatch match{}; + if (duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND) + check(decode_fails(pack.get(),match.entry_id),"malformed image still fails on demand"); + } +} + +void preload_pool() { + namespace tp = psx_texpreload; + std::atomic ran{0}; + tp::fault_mode() = 2; /* the second thread cannot be created */ + size_t errors = tp::run(32,8,[&](size_t){ ++ran; }); + check(errors == 0 && ran == 32,"thread creation failure lowers concurrency, all items still run"); + tp::fault_mode() = 1; /* the first item throws bad_alloc */ + ran = 0; + errors = tp::run(32,8,[&](size_t){ ++ran; }); + tp::fault_mode() = 0; + check(errors == 1 && ran == 31,"a throwing item is caught and counted"); + ran = 0; + errors = tp::run(32,8,[&](size_t k){ if (k % 4 == 0) throw std::runtime_error("x"); ++ran; }); + check(errors == 8 && ran == 24,"item exceptions never escape a pool thread"); + tp::Budget budget(100); + check(budget.reserve(60) && !budget.reserve(41) && budget.reserve(40) && !budget.reserve(1), + "aggregate budget reservations are bounded"); + budget.release(50); + check(budget.reserve(50),"released bytes are reusable"); } void fixed_hash_vectors() { @@ -644,7 +702,7 @@ int main(int argc, char** argv) { palettes(root/"palettes"); st_alias_duplicates(root/"keys"); failed_payloads(root/"invalid"); dumping(root/"dump"); official_binary_oracle(root/"official-oracle"); defaults_and_options(root/"defaults"); immutable_palettes_and_pages(root/"snapshots"); split_coalesce_copy_parts(root/"lifecycle"); copied_pack_ownership(root/"ownership"); - resident_checkpoint(root/"resident"); preload(root/"preload"); + resident_checkpoint(root/"resident"); preload(root/"preload"); preload_pool(); release_while_another_source_splits(root/"release-split"); /* Every artifact is created beneath this fresh, explicitly named test * root. No user-provided paths participate in recursive cleanup. */ diff --git a/runtime/tests/test_hd_texture_pack.cpp b/runtime/tests/test_hd_texture_pack.cpp index a3695756..ab581aee 100644 --- a/runtime/tests/test_hd_texture_pack.cpp +++ b/runtime/tests/test_hd_texture_pack.cpp @@ -1,6 +1,7 @@ #define STB_IMAGE_IMPLEMENTATION #include "../third_party/stb_image.h" #include "hd_texture_pack.h" +#include "texture_pack_preload.hpp" #include #include @@ -530,6 +531,54 @@ void test_bounded_async_decode(Fixture& fixture) { hd_texture_pixels_release(&pixels); } +void test_bounded_preload(Fixture& fixture) { + namespace tp = psx_texpreload; + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + HdTexturePreloadResult r{}; + /* One valid 64x8 PNG; the other keys are empty (malformed) files. */ + const int status = hd_texture_pack_preload(pack, size_t{64} << 20, 4, &r); + check(status >= 0 && r.loaded == 1 && r.bytes == 64u * 8u * 4u, + "preload pins the valid replacement"); + check(r.requested == r.loaded + r.failed + r.over_budget && + (r.requested == 1 ? status == 1 : (status == 0 && r.failed >= 1)), + "malformed replacements are reported, not counted as preloaded"); + hd_texture_pack_set_decode_budget(pack, 1); + check(hd_texture_pack_track_upload(pack, 10, 20, 8, 4, + fixture.upload_words.data(), + fixture.upload_words.size(), nullptr) == 1, + "preload fixture upload tracks"); + HdTexturePixels pixels{}; + check(hd_texture_pack_acquire_decoded(pack, fixture.upload_hash, fixture.palette4_hash, + &pixels) == HD_TEXTURE_LOOKUP_FOUND && + pixels.width == 64, + "a preloaded image is resident and survives budget eviction"); + hd_texture_pixels_release(&pixels); + hd_texture_pack_destroy(pack); + } + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + HdTexturePreloadResult r{}; + check(hd_texture_pack_preload(pack, 64, 4, &r) == 0 && r.loaded == 0 && + r.over_budget == 1 && r.bytes == 0, + "the aggregate cap skips an oversized image"); + hd_texture_pack_destroy(pack); + } + { + HdTexturePack* pack = open_pack(fixture.root); + if (!pack) return; + tp::fault_mode() = 1; + HdTexturePreloadResult r{}; + const int status = hd_texture_pack_preload(pack, size_t{64} << 20, 4, &r); + tp::fault_mode() = 0; + check(status == 0 && r.worker_errors == 1 && r.loaded + 1 + r.over_budget <= r.requested, + "an injected worker failure is caught and reported as partial"); + hd_texture_pack_destroy(pack); + } +} + } // namespace int main() { @@ -542,6 +591,7 @@ int main() { test_wrapping(fixture); test_ambiguity_fallback(); test_bounded_async_decode(fixture); + test_bounded_preload(fixture); if (failures) { std::fprintf(stderr, "test_hd_texture_pack: %d failure(s)\n", failures);