From ed398cc37e2ae2f385b76d7f6be7de9891b27fba Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Wed, 7 Oct 2026 13:40:57 -0700 Subject: [PATCH] Improve DuckStation texture capture and replacement fidelity --- THIRD_PARTY_ATTRIBUTION.md | 14 + docs/DUCKSTATION_TEXTURE_FORMAT.md | 155 ++++- docs/HD_TEXTURE_PACKS.md | 33 +- docs/internal/FAITHFUL_TIMING_PLAN.md | 13 + .../duckstation-texture-format-and-xxhash.md | 42 +- .../1.0.0/manifest.toml | 10 +- runtime/CMakeLists.txt | 7 + runtime/include/duckstation_texture_pack.h | 87 ++- runtime/include/gpu_hd_textures.h | 13 +- runtime/include/texture_image_decode.h | 37 + runtime/licenses/libwebp-NOTICES.txt | 134 ++++ runtime/runtime.cmake | 3 +- runtime/src/debug_server.c | 9 +- runtime/src/duckstation_texture_pack.cpp | 654 ++++++++++++++---- runtime/src/gpu_gl_renderer.c | 2 + runtime/src/gpu_hd_textures.cpp | 248 ++++++- runtime/src/gpu_render.c | 11 +- runtime/src/psx_window_icon.cpp | 1 + runtime/src/texture_image_decode.cpp | 107 +++ .../tests/fixtures/texture_images/README.txt | 4 + .../fixtures/texture_images/animated.webp | Bin 0 -> 172 bytes .../fixtures/texture_images/opaque-lossy.webp | Bin 0 -> 72 bytes .../tests/fixtures/texture_images/opaque.jpg | Bin 0 -> 638 bytes .../fixtures/texture_images/progressive.jpg | Bin 0 -> 524 bytes .../tests/fixtures/texture_images/rgba.png | Bin 0 -> 82 bytes .../tests/fixtures/texture_images/rgba.webp | Bin 0 -> 68 bytes runtime/tests/run_gl_texture_filter.py | 8 +- .../tests/test_duckstation_texture_pack.cpp | 280 +++++++- runtime/tests/test_gl_hd_texture_pack.c | 167 ++++- runtime/tests/test_texture_image_decode.cpp | 60 ++ runtime/texture_image_dependency.cmake | 37 + third_party/deps.manifest | 1 + 32 files changed, 1924 insertions(+), 213 deletions(-) create mode 100644 runtime/include/texture_image_decode.h create mode 100644 runtime/licenses/libwebp-NOTICES.txt create mode 100644 runtime/src/texture_image_decode.cpp create mode 100644 runtime/tests/fixtures/texture_images/README.txt create mode 100644 runtime/tests/fixtures/texture_images/animated.webp create mode 100644 runtime/tests/fixtures/texture_images/opaque-lossy.webp create mode 100644 runtime/tests/fixtures/texture_images/opaque.jpg create mode 100644 runtime/tests/fixtures/texture_images/progressive.jpg create mode 100644 runtime/tests/fixtures/texture_images/rgba.png create mode 100644 runtime/tests/fixtures/texture_images/rgba.webp create mode 100644 runtime/tests/test_texture_image_decode.cpp create mode 100644 runtime/texture_image_dependency.cmake diff --git a/THIRD_PARTY_ATTRIBUTION.md b/THIRD_PARTY_ATTRIBUTION.md index 1d71d291..ccbe9a2c 100644 --- a/THIRD_PARTY_ATTRIBUTION.md +++ b/THIRD_PARTY_ATTRIBUTION.md @@ -44,6 +44,20 @@ netplay binary: `runtime/licenses/libjuice-NOTICES.txt`. MPL-2.0 is file-level copyleft: a change to a libjuice file must be published under MPL-2.0. The build changes none. +## libwebp - static texture-image decoder + +[libwebp 1.6.0](https://chromium.googlesource.com/webm/libwebp/+/refs/tags/v1.6.0) +by Google Inc. and WebM contributors is licensed **BSD-3-Clause**. The runtime +builds only its static decoder, without command-line tools, the encoder, or a +libwebp DLL. The dependency is unmodified and pinned to commit +`4fa21912338357f89e4fd51cf2368325b59e9bd9`, with immutable archive SHA-256 +`923f3382a47a2af185c3240c954cf004428b237bd7317413a95146d01eb4b94b` in +`third_party/deps.manifest`. Its original COPYING, PATENTS, and AUTHORS texts +ship in `runtime/licenses/libwebp-NOTICES.txt`, which release packagers carry +into `licenses/`. Developers can use the verified archive or an explicit +`FETCHCONTENT_SOURCE_DIR_PSX_LIBWEBP` / `PSX_LIBWEBP_SOURCE_DIR` source override. +PNG and JPEG continue to use the existing shared stb_image implementation. + ## Vendored libraries These are checked in under `recompiler/lib/` and `runtime/third_party/` with diff --git a/docs/DUCKSTATION_TEXTURE_FORMAT.md b/docs/DUCKSTATION_TEXTURE_FORMAT.md index 92e287b0..b0c78648 100644 --- a/docs/DUCKSTATION_TEXTURE_FORMAT.md +++ b/docs/DUCKSTATION_TEXTURE_FORMAT.md @@ -28,10 +28,19 @@ made XXH3 output stable across future versions. Thus the upstream v0.8.3 implementation supplies the unseeded XXH3 behavior used by DuckStation's vendored v0.8.0 without importing any emulator code. +WebP decoding uses only the static decoder from the official +[libwebp 1.6.0 tag](https://chromium.googlesource.com/webm/libwebp/+/refs/tags/v1.6.0), +commit `4fa21912338357f89e4fd51cf2368325b59e9bd9`, under BSD-3-Clause. +The immutable source archive has SHA-256 +`923f3382a47a2af185c3240c954cf004428b237bd7317413a95146d01eb4b94b`. +Its copyright, COPYING, PATENTS, and AUTHORS notices ship in +`runtime/licenses/libwebp-NOTICES.txt`. No encoder, command-line utility, or +libwebp DLL is required. PNG/JPEG use the existing shared stb_image implementation. + ## Supported files Choose a game/package folder containing `replacements/` and `dumps/`, or choose -`replacements/` itself. Replacement PNGs can live in subdirectories. See the +`replacements/` itself. Replacement PNG/JPEG/static WebP images can live in subdirectories. See the [player workflow](HD_TEXTURE_PACKS.md) for dumping, editing and reloading. DuckStation documents its serial-based folders and authoring process in its [texture replacement wiki](https://github.com/stenzek/duckstation/wiki/Texture-Replacement). @@ -51,7 +60,7 @@ Direct stems omit the palette fields: mode = C16 | STC16 ``` -Append `.png`. Hash fields contain exactly 16 hexadecimal digits; either case +Append `.png`, `.jpg`/`.jpeg`, or `.webp`. Hash fields contain exactly 16 hexadecimal digits; either case is accepted. Dimensions, offsets and palette endpoints are decimal. Source width is in native VRAM words. Rectangle width and X offset are expanded texels: four per word for P4, two for P8, one for C16. Height and Y offset are @@ -60,7 +69,7 @@ These definitions follow the [upstream filename documentation](https://github.com/stenzek/duckstation/wiki/Texture-Replacement). The loader validates nonzero sizes, native VRAM bounds, word-aligned horizontal -rectangles, page bounds and palette endpoints. The replacement PNG's actual +rectangles, page bounds and palette endpoints. The replacement image's actual size controls its independent X/Y scale; integer and equal-axis scale factors are not required. The renderer applies the declared native rectangle's origin and extent when mapping UVs. @@ -80,7 +89,8 @@ For reduced `Pmin-max` ranges, the reference implementation hashes the **first maximum crosses the edge does not match. It does not revalidate used indices against the declared range during lookup. psxrecomp preserves this behavior for interoperability. Changing a palette entry outside that prefix can leave -a replacement matched; use full ranges when authoring new packs here. These +a replacement matched. Set `ReducePaletteRange: false` to capture full ranges +when that tradeoff is preferable. These are observations of the pinned [texture-cache behavior](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_hw_texture_cache.cpp). @@ -95,27 +105,38 @@ with nonzero RGB in an ST image remains a semitransparent texel. Actual blending still uses the primitive's PSX transparency mode. These rules follow the pinned [replacement merge shader](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_hw_shadergen.cpp). -When both ordinary and ST records match, psxrecomp prefers the primitive's -convention; if only the other convention exists it can still supply the image. -Multiple matching records with the same preferred convention fall back to -native rendering. This deterministic policy avoids dependence on directory -enumeration or unordered-map order. +When ordinary and ST records cover the same identity and rectangle, psxrecomp +prefers the primitive's convention; if only the other convention exists it can +still supply the image. Duplicate canonical identities remain ambiguous. +Distinct matching parts are clipped and composed in a deterministic order. -The dumper writes rounded 5-bit-to-8-bit native RGB, full CLUT ranges, alpha +The finalized dumper writes rounded 5-bit-to-8-bit native RGB, reduced CLUT +ranges by default, alpha zero for native color zero, alpha 255 for occupied ordinary texels, and alpha -128 for occupied STP texels in ST dumps. The pinned upstream dump produces -143 in that last case because of an incidental mask expression; both values -have the same replacement classification. Dumps here are compatible authoring -inputs, not a promise of byte-identical upstream dump PNGs. Native conversion +143 for occupied STP texels in ST dumps, matching the pinned upstream dump's +mask-expression result. `DumpTextureForceAlphaChannel` instead emits ordinary +names and makes every pixel's alpha 255. Native conversion was checked against the pinned [GPU helpers](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_helpers.h). ## Current compatibility limits -The module accepts a single replacement covering the complete draw query. +The module accepts bounded replacement parts clipped to the draw query. It supports ordinary unchanged uploads and nonwrapping page subrectangles. -Every touched upload loses its identity after a partial overwrite; it does -not reconstruct split or coalesced upload histories. A same-session pack reload +OpenGL composes up to 64 matching parts over a native-pixel fallback, with at +most 16 cached compositions, 64 MiB of composed RGBA, and 8192 pixels per side. +`ReplacementScaleLinearFilter` enables color interpolation when scaling parts +while keeping cutout, STP, and opaque classification discrete. The default is +nearest scaling. Decode or composition limits retain native rendering. +Bounded copy handling, partial-overwrite splitting, and adjacent upload +coalescing can be enabled with their configuration keys; their defaults are +off. Splits retain the immutable original source hash and words. Exceeding a +split limit finalizes observed usage and drops the remaining tracked identity. +Coalescing accepts unused, unsplit uploads adjacent to the right or below, +within the configured incoming-write dimensions. Copies normally invalidate +their destination; `ConvertCopiesToWrites` can refresh an existing destination +source's original extent when its active region contains the copy. It does not +clone the source identity. A same-session pack reload can copy intact upload identities into the newly indexed pack. Loading a savestate clears them; page matching remains available, while upload matching needs subsequent guest uploads. The module's rectangle and query caches must @@ -125,18 +146,46 @@ The following cases use native rendering or report an unsupported-file/option diagnostic: - Wrapped UV intervals, horizontally wrapped pages and wrapped uploads. -- Draws requiring composition from several replacement images. -- `vram-write-` XXH3-128 images, JPEG/WebP payloads and legacy unnamed layouts. -- Copy-to-upload conversion, upload coalescing and split-upload identities. -- Other `config.yaml` behavior options, including replacement bilinear filtering. +- `vram-write-` XXH3-128 images and legacy unnamed layouts. +- Animated WebP and image payloads outside PNG/JPEG/static WebP. +- Unrecognized or unimplemented `config.yaml` behavior. -`config.yaml` supports only a bounded `Aliases:` mapping with literal or quoted -filename keys and relative PNG paths. A single-line literal/folded block value +`config.yaml` supports root-level scalar options and a bounded `Aliases:` mapping +with literal or quoted filename keys and relative PNG/JPEG/WebP paths under +`replacements/`. A single-line literal/folded block value is accepted. Escaped strings, multiline values, anchors and general YAML features are outside the subset. Absolute paths and `..` traversal are rejected. A direct canonical replacement takes precedence over its alias. Unknown top-level options are reported by name and are not applied. +The root-level authoring keys and defaults are: + +| Key | Default | Behavior | +| --- | --- | --- | +| `DumpTexturePages` | `false` | Select page capture instead of upload tracking. | +| `DumpFullTexturePages` | `false` | Capture full 256x256 pages in page mode. | +| `DumpTextureForceAlphaChannel` | `false` | Force opaque alpha and ordinary names for texture dumps. | +| `DumpC16Textures` | `false` | Include direct-color texture captures. | +| `ReducePaletteRange` | `true` | Emit the used palette-index range, with the pinned prefix-hash convention. | +| `ConvertCopiesToWrites` | `false` | Refresh bounded existing destination-source tracking after copies. | +| `MaxVRAMWriteSplits` | `0` | Bound partial-overwrite source splitting; zero disables it. | +| `MaxVRAMWriteCoalesceWidth`, `MaxVRAMWriteCoalesceHeight` | `0` | Bound incoming uploads eligible for adjacent coalescing. | +| `DumpTextureWidthThreshold`, `DumpTextureHeightThreshold` | `16` | Minimum finalized texture dimensions in texels. | +| `DumpVRAMWriteForceAlphaChannel` | `true` | Recognized upstream background-write option; that separate dump mode is unsupported. | +| `DumpVRAMWriteWidthThreshold`, `DumpVRAMWriteHeightThreshold` | `128` | Recognized background-write thresholds; do not enable XXH3-128 dumping. | +| `ReplacementScaleLinearFilter` | `false` | Interpolate composition colors within the nearest pixel's PSX alpha class. | +| `MaxHashCacheEntries` | `256` | Rectangle-hash cache entries, capped at 4096. | +| `MaxHashCacheVRAMUsageMB` | `64` | Source-snapshot budget in MiB, capped at 64. | +| `MaxReplacementCacheVRAMUsage` | `64` | Decoded replacement budget in MiB, capped at 64. | + +Use these keys at the root, not inside `Options:`. General YAML structures are +unsupported. Existing full-range replacement filenames remain valid even when +new captures use reduced ranges. The Tomba example's `config.yaml.example` +documents the defaults without activating or overwriting a user's config. +The cache keys are compatibility mappings to this implementation's bounded +CPU storage: `MaxHashCacheVRAMUsageMB` controls source snapshots, rather than +reproducing DuckStation's GPU hash-cache allocation. + ## Bounds and lifecycle The loader hard-fails above 262144 candidate files or entries, scans at most @@ -147,22 +196,35 @@ Native page/CLUT writes invalidate affected cached queries. A query requiring over 4096 candidate page geometries or more than 4 MiB of new hashing reports an error and renders natively. -One low-priority background worker performs PNG decoding and dumping. The +One low-priority background worker performs PNG/JPEG/WebP decoding and PNG dumping. The decode queue is bounded to 32, decoded metadata to 512 entries, and decoded pixels to a default 64 MiB budget. Encoded files are limited to 64 MiB and each image dimension to 8192 before decoding. Pixel leases survive eviction and pack destruction. Decoding failure leaves native rendering active. -Dumps use word-aligned draw rectangles, choose `texupload` when an intact -tracked source contains the draw, and otherwise choose `texpage`. The queue -holds at most eight snapshots. A bounded set deduplicates 8192 dump identities -per session; reaching it reports a diagnostic until dump tracking is reset. -PNG writing finishes on pack destruction. A temporary file is published +Dumps accumulate the union of word-aligned used rectangles per source and +palette, including semitransparent draw usage. They finalize when a source +retires or capture is flushed, rather than writing a PNG for each draw. +Default upload mode emits `texupload`; optional page mode emits `texpage`. +Original source snapshots are bounded to 64 MiB by default, with at most 8192 +tracked sources/uploads and 8192 palette records; pressure retires sources and +finalizes their observed usage. The write queue is bounded to eight jobs and +32 MiB of queued RGBA, applying backpressure instead of silently losing captures, +and dump deduplication to a rolling 8192 identities. Exiting the game flushes +remaining sources and waits for PNG writing. A temporary file is published atomically without replacing an existing final file. Write failures remove -only the created temporary, print an error even on shutdown, and are returned -by the next dump call. All snapshots decode native VRAM rather than replacement +only the created temporary and are returned by capture/flush operations. +All snapshots decode native VRAM rather than replacement pixels. +Texture identities describe upload/page pixels, palette ranges, source sizes, +and used rectangles. A PSX sprite assembled from several uploads or palettes +can still produce separate parts; palette animation can produce several IDs. +Different histories, supported options, wrapping, or composition requirements +can still differ from DuckStation. This is bounded interoperability, not a +promise that every real game's dumps have identical filenames or reconstructed +sprite images. Alias names can make editing easier without changing identity. + ## Independent validation `runtime/tests/test_duckstation_texture_pack.cpp` contains fixed XXH3 vectors @@ -184,3 +246,34 @@ invalidation, reload identity copying, duplicate ambiguity, alias diagnostics, raw alpha thresholds, Unicode folders, background decoding, native dump pixels, worker shutdown and preservation of existing edited files. Renderer fixtures exercise the alpha classification and native-VRAM authority separately. + +`texture_image_decode_test` uses synthetic literal-pixel fixtures generated by +Pillow 12.3.0. Windows Clang and Linux GCC checks cover raw PNG/lossless WebP +RGBA (including alpha 0, 1, 128, 254), baseline/progressive JPEG, lossy WebP, +dimensions, encoded/decoded bounds, truncated input, and explicit animation +rejection. Decoders accept bytes supplied by the background loader and perform +no filesystem access themselves. + +An official pinned DuckStation binary was also used as an external oracle: +`0.1-12074-g697599c47 (dev)`, development ZIP SHA-256 +`4813f22823e6aa262c46a4a516f10b01e1e2ddf6fcea82763e359ef9ac812a9d`. +The owned standalone PS-X probe contains no game artwork. It uploads 10x56 +words at VRAM (640,0), repeating `3210 7654 BA98 FEDC` 140 times. Palette A at +(0,256) is zero followed by `(i*1537)&0x7fff` for indices 1..15; palette B at +(0,257) is zero followed by `((i*293)&0x7fff)|0x8000`. It draws a 32x32 ordinary +head and nested 36x32 semitransparent head at UV (0,0) with A, then a 40x24 +ordinary body at UV (0,32) with B. Overwriting the complete upload retires it. +The reference produces exactly these two lifetime-union files: + +| Reference filename | Decoded RGBA SHA-256 | +| --- | --- | +| `texupload-STP4-18DDF9B731440D20-0DAE5F73C88719B1-10x56-0-0-36x32-P0-15.png` | `5d1bec29956fb3979547f18c9ccc719903924bd9c9d70584913d363b67248ca7` | +| `texupload-P4-18DDF9B731440D20-AF506869B80F5CE7-10x56-0-32-40x24-P0-15.png` | `1a26a7b32dc147cd952364bc59290eaaa8f973fb1c2201c84b9e2756a7bc3479` | + +psxrecomp produces the same two filenames and dimensions, with every decoded +RGBA byte identical to those reference PNGs. The module regression preserves +the literal oracle names; the primary audit compared the actual files independently. + +This controlled oracle isolates naming, lifetime union, palette hashing, and +native pixel conversion. It does not establish full Tomba gameplay identity +equivalence, and no reference binary or proprietary game pixels are distributed. diff --git a/docs/HD_TEXTURE_PACKS.md b/docs/HD_TEXTURE_PACKS.md index 41d71c95..7731e30a 100644 --- a/docs/HD_TEXTURE_PACKS.md +++ b/docs/HD_TEXTURE_PACKS.md @@ -1,6 +1,6 @@ # HD texture packs -The **HD Texture Packs** mod loads compatible DuckStation texture PNGs and can +The **HD Texture Packs** mod loads compatible DuckStation PNG/JPEG/WebP images and can dump original textures encountered during gameplay. It is disabled by default. Use **OpenGL** to display replacements. Software and Vulkan can collect dumps but continue to display the original artwork. Tomba USA (`SCUS-94236`) is the @@ -19,7 +19,7 @@ and select OpenGL to see the example title textures. 2. Under **Texture pack folder**, click **Open folder**. On first use the launcher creates `mods/texture-packs/` beside the executable, with empty `replacements` and `dumps` directories. -3. Copy the pack's replacement PNGs into `replacements`, preserving their +3. Copy the pack's replacement PNG, JPEG, or WebP images into `replacements`, preserving their filenames. Subdirectories are supported. If the download already contains `replacements`, copy that directory's contents into this one. 4. Leave **Load replacements** enabled and **Dump textures** disabled. @@ -34,9 +34,11 @@ beside it. Keep editable packs outside `mods/bundled`, which builds regenerate. 1. Enable **HD Texture Packs**, disable **Load replacements**, and enable **Dump textures**. -2. Click **Play** and visit the scenes you want to capture. Original texture - PNGs appear under `dumps` as the game uses them. -3. Exit to the launcher. Copy selected PNGs into `replacements`, then edit or +2. Click **Play** and visit the scenes you want to capture. The runtime collects + the used area of each texture source across its lifetime. Some files appear + when the game overwrites or retires that source; others are still pending. +3. Exit the game to finish capture and allow the writer to complete. Copy + selected PNGs from `dumps` into `replacements`, then edit or enlarge them while keeping the filenames, aspect ratio, and an integer size multiplier. 4. Enable **Load replacements**, disable **Dump textures**, and click **Play** @@ -47,10 +49,25 @@ the runtime scans them again. Disabling the mod restores the original artwork. The plugin does not write replacement pixels into native VRAM or change the save namespace. +Capture defaults follow the referenced DuckStation policy: track uploads, +union their used rectangles per palette, skip direct-color C16 textures, ignore +results smaller than 16x16, and reduce palette ranges to used indices. Small +sprites or parts can therefore be absent until you lower the thresholds. +Dumping texture pages is optional and often produces more duplicate-looking +images. PSX sprites are assembled from texture parts, palettes, and drawing +commands; dumps are editable source textures, not reconstructed characters. + +The Tomba example includes an inactive `config.yaml.example`. Copy it to +`config.yaml` at the pack root to choose authoring settings. Option keys go +directly at the YAML root, without an `Options:` section. `Aliases:` can map a +canonical identity to a friendly PNG/JPEG/WebP filename relative to +`replacements`; aliases do not alter the matching identity. See the format +guide for the supported keys and their bounds. + ## Compatibility and diagnostics Pack artwork is supplied separately. Matching depends on native game pixels -and palettes; a different region or revision may not match. PNG is supported, +and palettes; a different region or revision may not match. PNG, JPEG, and static WebP are supported, with a maximum of 8192 pixels per side, 64 MiB encoded, and 64 MiB decoded RGBA. Unsupported identities, configuration features, or images fall back to the original artwork. After loading a savestate, upload-based replacements need @@ -62,5 +79,7 @@ Development builds expose the [TCP `hd_textures` command](TCP_COMMANDS.md). It reports the selected root, switches, backend support, pack size, and matched, ready, and applied draw counts. Optional integer fields `replacements: 0|1`, `dump: 0|1`, and `reload: 1` control an already configured pack during a test -session. Use `present_shot` to capture the composed OpenGL window; native VRAM +session. `pending_dump_sources` counts sources with capture usage awaiting +finalization; `dumped_textures` counts queued PNG jobs, which can precede files +appearing. Exit the game to finish capture. Use `present_shot` to capture the composed OpenGL window; native VRAM screenshots retain the original artwork. diff --git a/docs/internal/FAITHFUL_TIMING_PLAN.md b/docs/internal/FAITHFUL_TIMING_PLAN.md index 30371383..a61a0bc4 100644 --- a/docs/internal/FAITHFUL_TIMING_PLAN.md +++ b/docs/internal/FAITHFUL_TIMING_PLAN.md @@ -213,6 +213,19 @@ on a fixed region -> next. ## 5. Status / Log (update every session) +- **2026-10-07 — DuckStation texture authoring fidelity (beads-qi66).** + The early Tomba capture report exposed per-draw crops and automatic page + fallback in the opt-in HD texture enhancement. An independent synthetic + PS-X GPU sequence run in the official DuckStation binary at reference + `697599c47a646a6cfcc4d246018adf4904d55001` produced one lifetime-unioned + 36x32 STP4 image from nested 32x32/36x32 draws and a separate 40x24 image + for another palette. The new module matches both literal filenames, + dimensions, and every decoded RGBA byte from that binary. Focused module, + image-decoder (Windows/Linux), and real OpenGL checks pass. A Tomba title + and Village smoke displayed checkerboards, preserved the native texture + atlas, and flushed pending native PNG captures without errors. Canonical native VRAM remains + authoritative; this enhancement changes no guest CPU, BIOS, or cycle model. + - **2026-10-03 - Render replay validation and texture streaming.** Increased the bounded small-transfer VRAM journal from 16 to 64 entries; Jersey Devil streams 31 texture/CLUT uploads in a draw. The 32-transfer rollback regression compares all VRAM bytes. Jersey Devil configures the SDK draw environment before replay packet allocation, preventing its command storage from overwriting a world quad. Final live runs completed at least 1,040 Jersey Devil and 1,088 A Bug's Life extra passes with zero state-verification mismatches, VRAM leaks, watchdogs or span failures. Changes remain local beyond base 9c1e716a. - **2026-10-03 - Ape Escape wall-flicker regression.** Dense frame/VRAM capture isolated world GT4s incorrectly promoted to UI behind a front-layer fade. UI grouping now selects the actual populated drawing layer, retaining empty trailing OT behavior. Captured-packet tests fail on the prior core and pass on the fix; software and 4K OpenGL/PGXP captures show no gaps in the affected camera interval. No guest timing or draw replay changes. diff --git a/docs/internal/upstream/duckstation-texture-format-and-xxhash.md b/docs/internal/upstream/duckstation-texture-format-and-xxhash.md index e252cae2..53102bc4 100644 --- a/docs/internal/upstream/duckstation-texture-format-and-xxhash.md +++ b/docs/internal/upstream/duckstation-texture-format-and-xxhash.md @@ -1,4 +1,4 @@ -# DuckStation texture format and xxHash provenance +# DuckStation texture format, xxHash, and libwebp provenance The independent format integration is described in [DuckStation texture format notes](../../DUCKSTATION_TEXTURE_FORMAT.md). @@ -11,13 +11,14 @@ implementation, translated implementation, patch, or binary is included, and no DuckStation co-author trailer is claimed for this independent work. Implemented behavior: bounded modern texture filenames and palette ranges, -XXH3 identities over native pixel sources and palettes, supported alpha -classification, replacement loading, and original-texture dumping. Related -coalescing, copy/split identities, multi-image composition, wrapped footprints, -JPEG/WebP decoding, and DuckStation rendering/cache implementation are excluded. -The format guide records the exact compatibility limits. +XXH3 identities over native pixel sources and palettes, alpha classification, +replacement loading, and source-lifetime original-texture dumping. The fidelity +followup independently adds root-level authoring options, bounded copy/split +and coalescing tracking, and multiple replacement parts. Wrapped footprints, +XXH3-128 background-write images, general YAML, and DuckStation's rendering/cache +implementation remain excluded. The format guide records the exact limits. -The only vendored implementation is unmodified +The checked-in hash implementation is unmodified [xxHash v0.8.3](https://github.com/Cyan4973/xxHash/tree/v0.8.3), authored by Yann Collet and contributors under [BSD-2-Clause](https://github.com/Cyan4973/xxHash/blob/v0.8.3/LICENSE). @@ -28,6 +29,33 @@ Player packages carry `runtime/licenses/xxhash-NOTICES.txt` as a runtime notice. Vendoring preserves authorship in the unmodified source and notices, rather than attributing that source to this change's author. +Static WebP decoding uses unmodified +[libwebp 1.6.0](https://chromium.googlesource.com/webm/libwebp/+/refs/tags/v1.6.0), +commit [4fa21912338357f89e4fd51cf2368325b59e9bd9](https://chromium.googlesource.com/webm/libwebp/+/4fa21912338357f89e4fd51cf2368325b59e9bd9), +authored by Google Inc. and WebM contributors under BSD-3-Clause. The immutable +archive SHA-256 is +`923f3382a47a2af185c3240c954cf004428b237bd7317413a95146d01eb4b94b`. +Authorship and license credit remain in the upstream source and original +COPYING, PATENTS, and AUTHORS texts in `runtime/licenses/libwebp-NOTICES.txt`. +No decoder code is translated or attributed to this integration's author. +Only the static decoder is linked; encoder, utilities, and DLL dependencies +are excluded. PNG/JPEG reuse stb_image's existing implementation. + +The bounded image helper was rebuilt and checked with Windows Clang and Linux +GCC against synthetic fixtures: raw PNG/lossless WebP RGBA, baseline/progressive +JPEG, lossy WebP, header dimensions, encoded/decoded bounds, truncated images, +and explicit animated-WebP rejection. It reads supplied bytes and performs no +filesystem access. This focused check does not claim full-game dump parity. + +The fidelity reference was additionally run as the official development binary +`0.1-12074-g697599c47`, ZIP SHA-256 +`4813f22823e6aa262c46a4a516f10b01e1e2ddf6fcea82763e359ef9ac812a9d`. +An owned synthetic PS-X probe supplies a 10x56-word upload, nested head draws, +a separate body palette, and a full overwrite. Its input formulas, exact two +output identities, and decoded RGBA digests are documented in the format guide. +The reference confirms lifetime unions and ST dump alpha 143 for that controlled +case. It supplies no proprietary artwork and makes no full-game parity claim. + Validation on current framework master `92c0f3cd`: registered DuckStation module and OpenGL renderer tests pass, alongside mod package, builtin manifest, runtime, and resident tests. Independent xxHash wheel vectors, diff --git a/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml b/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml index 6132b392..c42d9808 100644 --- a/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml +++ b/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml @@ -3,7 +3,7 @@ id = "psx.enhancement.hd-textures" version = "1.0.0" name = "HD Texture Packs" author = "psxrecomp" -description = "Load compatible DuckStation PNG texture packs with OpenGL or dump original textures during gameplay. No artwork is bundled." +description = "Load compatible DuckStation PNG/JPEG/WebP packs with OpenGL or capture originals during gameplay. Exit the game to finish capture PNGs. No artwork is bundled." resolver = "declarative" save_compatibility = "shared" @@ -13,7 +13,7 @@ game_id = "*" [[feature]] id = "hd-textures" name = "HD Texture Packs" -description = "Disabled by default. Open the pack folder to install PNG artwork in replacements, or enable Dump textures to collect originals in dumps. Options apply on Play. OpenGL displays replacements; other backends retain original artwork." +description = "Disabled by default. Install PNG/JPEG/WebP artwork in replacements, or enable Dump textures to collect originals. Exit the game to finish capture PNGs in dumps. Options apply on Play. OpenGL displays replacements; other backends retain original artwork." group = "Visual" default_enabled = false @@ -21,7 +21,7 @@ default_enabled = false feature = "hd-textures" id = "pack" label = "Texture pack folder" -description = "The launcher creates mods/texture-packs/ beside the executable on first use. Open folder to add replacement PNGs or read captured originals; Change folder selects another compatible pack or empty directory." +description = "The launcher creates mods/texture-packs/ beside the executable on first use. Open folder to add replacement images or read captured originals; Change folder selects another compatible pack or empty directory." format = "directory" required = true @@ -29,7 +29,7 @@ required = true feature = "hd-textures" id = "replacements" label = "Load replacements" -description = "Display matching PNG artwork from replacements in OpenGL. Unsupported or missing replacements retain the original artwork." +description = "Display matching PNG/JPEG/WebP artwork from replacements in OpenGL. Unsupported or missing replacements retain the original artwork." type = "boolean" default = "true" @@ -37,7 +37,7 @@ default = "true" feature = "hd-textures" id = "dump" label = "Dump textures" -description = "Write original texture PNGs under dumps as they are encountered while playing. Disable after collecting the areas you need." +description = "Collect original texture usage while playing. Exit the game to finish capture PNGs under dumps, then edit the files you need." type = "boolean" default = "false" diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 01780a6c..014d978c 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -230,6 +230,8 @@ if(BUILD_TESTING) "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_gl_texture_filter.py" --hd-pack --fixture "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_gl_hd_texture_pack.c" --cc "${CMAKE_C_COMPILER}" --cxx "${CMAKE_CXX_COMPILER}" + --webp-include "$" + --webp-decoder "$" --sdl-include "$" --sdl-library "$" --output "${CMAKE_CURRENT_BINARY_DIR}/gl-hd-texture-results") @@ -463,7 +465,12 @@ if(BUILD_TESTING) src/duckstation_texture_pack.cpp) target_include_directories(duckstation_texture_pack_test PRIVATE include) target_link_libraries(duckstation_texture_pack_test PRIVATE Threads::Threads) + target_link_libraries(duckstation_texture_pack_test PRIVATE psx_texture_image_decode) add_test(NAME duckstation_texture_pack_test COMMAND duckstation_texture_pack_test) + add_executable(texture_image_decode_test tests/test_texture_image_decode.cpp) + target_link_libraries(texture_image_decode_test PRIVATE psx_texture_image_decode) + add_test(NAME texture_image_decode_test COMMAND texture_image_decode_test + "${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/texture_images") # Standalone serial-pad protocol tests use explicit SIO/device stubs; they # cover both DualShock rumble and JogCon wire/state behavior. diff --git a/runtime/include/duckstation_texture_pack.h b/runtime/include/duckstation_texture_pack.h index cb23a9c4..ff1e4810 100644 --- a/runtime/include/duckstation_texture_pack.h +++ b/runtime/include/duckstation_texture_pack.h @@ -36,6 +36,10 @@ typedef struct DuckTexturePackInfo { size_t replacement_count; size_t ambiguous_count; size_t ignored_count; + size_t queued_dump_count; /* Cumulative jobs actually submitted to writer. */ + size_t pending_dump_sources; + size_t pending_palette_records; + size_t snapshot_bytes; } DuckTexturePackInfo; typedef struct DuckTextureMatch { @@ -48,8 +52,32 @@ typedef struct DuckTextureMatch { int32_t origin_v; uint16_t source_width; uint16_t source_height; + /* Intersection with this draw and the surviving native source region. + * These page-space texels clip the full replacement mapping above. */ + int32_t clip_u; + int32_t clip_v; + uint16_t clip_width; + uint16_t clip_height; } DuckTextureMatch; +typedef struct DuckTextureConfig { + uint16_t max_vram_write_splits; + uint16_t max_vram_write_coalesce_width; + uint16_t max_vram_write_coalesce_height; + uint16_t texture_dump_width_threshold; + uint16_t texture_dump_height_threshold; + uint16_t vram_write_dump_width_threshold; + uint16_t vram_write_dump_height_threshold; + uint8_t dump_texture_pages; + uint8_t dump_full_texture_pages; + uint8_t dump_texture_force_alpha_channel; + uint8_t dump_vram_write_force_alpha_channel; + uint8_t dump_c16_textures; + uint8_t reduce_palette_range; + uint8_t convert_copies_to_writes; + uint8_t replacement_scale_linear_filter; +} DuckTextureConfig; + typedef struct DuckTexturePixels { const uint8_t* rgba; uint32_t width; @@ -60,13 +88,18 @@ typedef struct DuckTexturePixels { /* Accepts a game/package folder containing replacements/ and dumps/, or the * replacements directory itself. Missing folders are valid empty packs, so - * a new pack can start by dumping. Recursive PNG scan; no game assets bundled. - * Non-PNG payloads, unsupported options and invalid names are diagnosed. */ + * a new pack can start by dumping. Recursive image scan; no game assets bundled. + * PNG, JPEG and WebP payloads; unsupported options and invalid names are + * diagnosed. Upstream config.yaml keys are root-level scalar options. */ int duck_texture_pack_create(const char* root, DuckTexturePack** out_pack, char* error, size_t error_capacity); void duck_texture_pack_destroy(DuckTexturePack* pack); void duck_texture_pack_get_info(const DuckTexturePack* pack, DuckTexturePackInfo* out_info); +void duck_texture_pack_get_config(const DuckTexturePack* pack, + DuckTextureConfig* out_config); +int duck_texture_pack_linear_filter(const DuckTexturePack* pack); +uint64_t duck_texture_pack_revision(const DuckTexturePack* pack); int duck_texture_parse_name(const char* filename, DuckTextureKey* out_key); int duck_texture_format_name(const DuckTextureKey* key, char* output, @@ -79,9 +112,8 @@ uint64_t duck_texture_hash_rect(const uint16_t* vram, size_t count, uint16_t width_words, uint16_t height); /* Track post-write native VRAM, after PSX set/check-mask semantics. Wrapped - * uploads are invalidated but deliberately not tracked. Copies, coalescing, - * and split-upload identities are outside this initial compatibility subset. - * Any touched upload is dropped, so partial overwrites fail conservatively. */ + * uploads are invalidated but deliberately not tracked. Immutable original + * words survive partial overwrites and configurable active-region splits. */ int duck_texture_pack_track_upload(DuckTexturePack* pack, uint16_t x, uint16_t y, uint16_t width_words, uint16_t height, @@ -90,11 +122,23 @@ void duck_texture_pack_invalidate(DuckTexturePack* pack, uint16_t x, uint16_t y, uint16_t width_words, uint16_t height); void duck_texture_pack_reset_tracking(DuckTexturePack* pack); -/* Same-session reload only: copy immutable upload identities while the caller - * keeps native VRAM unchanged. Lookup, image and dump caches stay independent. +/* Same-session reload only: clone immutable sources and owned observations + * while the caller keeps native VRAM unchanged. Lookup/image caches remain + * independent; only same-root filename deduplication history is shared. * This is not a savestate restoration API. Strong replacement on success. */ int duck_texture_pack_copy_tracking(DuckTexturePack* destination, const DuckTexturePack* source); +/* Begin before changing native VRAM, end after PSX mask semantics. Default + * copies invalidate destination identities. ConvertCopiesToWrites updates an + * existing destination write's original extent when its active region covers + * the copy; it does not clone the source upload into the destination. */ +int duck_texture_pack_begin_copy(DuckTexturePack* pack, + uint16_t source_x, uint16_t source_y, + uint16_t destination_x, uint16_t destination_y, + uint16_t width_words, uint16_t height, + const uint16_t* vram, size_t count); +int duck_texture_pack_end_copy(DuckTexturePack* pack, + const uint16_t* vram, size_t count); /* Only one replacement covering the entire inclusive query is accepted. * UV wrapping, edge-wrapped pages and multi-image composition fall back. @@ -108,6 +152,15 @@ int duck_texture_pack_match(DuckTexturePack* pack, int duck_texture_pack_match_draw(DuckTexturePack* pack, const HdTextureDrawQuery* query, int semitransparent, DuckTextureMatch* out_match); +/* All intersecting parts, in composition order: pages before uploads, larger + * rectangles before smaller ones. Same-coverage ST variants prefer this draw's + * convention. An insufficient capacity returns ERROR and zero parts, never a + * silently truncated composition. At most 64 parts are supported. */ +int duck_texture_pack_match_parts(DuckTexturePack* pack, + const HdTextureDrawQuery* query, + int semitransparent, + DuckTextureMatch* out_matches, + size_t capacity, size_t* out_count); /* One bounded background worker; no PNG decoding in the render thread. * Pixels retain raw DuckStation alpha bytes. The renderer interprets the @@ -119,18 +172,22 @@ int duck_texture_pack_acquire_decoded(DuckTexturePack* pack, uint64_t entry_id, DuckTexturePixels* out_pixels); void duck_texture_pixels_release(DuckTexturePixels* pixels); -/* Dumps the word-aligned draw rectangle. Uses texupload when exactly one - * unchanged tracked upload covers it; otherwise texpage. Full palette range - * is emitted; transparent native zero stays zero, native STP uses alpha 128 - * for ST names. Existing files are never overwritten. Writes native decoded - * pixels only, never already-replaced pixels. Returns FOUND for a new dump, - * NONE for an existing/skipped dump, ERROR on invalid input or write failure. - * FOUND means queued; bounded background writes finish on pack destruction. - * Any asynchronous write failure is reported by the next dump call. */ +/* Records a draw's used region and captured palette, unioning nested crops + * until the upload/page retires or a flush occurs. Upload-only by default; + * minimum 16x16, C16 off and reduced palette ranges on. Any semi draw selects + * ST unless ForceAlpha is enabled. FOUND means new usage was recorded, not + * that a file was written. Existing files are never overwritten. */ int duck_texture_pack_dump_draw(DuckTexturePack* pack, const HdTextureDrawQuery* query, int semitransparent, char* error, size_t error_capacity); +/* Finalize current usage without discarding live identities, then drain dump + * writes. Call before disabling dumps/changing folders. Reset/destruction also + * finalize. Reload copies own independent sources and share same-root filename + * history, so destroying a copied pack never steals the original's records. + * Returns FOUND on success (including empty flush), ERROR on failure. */ +int duck_texture_pack_flush_dumps(DuckTexturePack* pack, + char* error, size_t error_capacity); void duck_texture_pack_reset_dump(DuckTexturePack* pack); #ifdef __cplusplus diff --git a/runtime/include/gpu_hd_textures.h b/runtime/include/gpu_hd_textures.h index 825910b0..6804a70c 100644 --- a/runtime/include/gpu_hd_textures.h +++ b/runtime/include/gpu_hd_textures.h @@ -25,6 +25,8 @@ typedef struct GpuHdTextureDiag { int active, replacements, dump, format; uint64_t draw_queries, matched_draws, ready_draws, applied_draws, dumped_textures; size_t replacement_count; + const char* diagnostic; + uint64_t pending_dump_sources; } GpuHdTextureDiag; void gpu_hd_textures_get_diag(GpuHdTextureDiag* out); void gpu_hd_textures_note_applied(void); @@ -39,6 +41,11 @@ void gpu_hd_textures_observe_draw(uint16_t texpage, uint16_t clut_x, void gpu_hd_textures_set_vram(const uint16_t* vram); void gpu_hd_textures_track_upload(int x, int y, int width_words, int height, const uint16_t* words); +/* Header-before-payload invalidation and pre/post native copy observations. */ +void gpu_hd_textures_begin_upload(int x, int y, int width_words, int height); +void gpu_hd_textures_begin_copy(int source_x, int source_y, int destination_x, + int destination_y, int width_words, int height); +void gpu_hd_textures_end_copy(void); void gpu_hd_textures_invalidate(int x, int y, int width_words, int height); void gpu_hd_textures_reset_tracking(void); @@ -49,14 +56,16 @@ typedef struct GpuHdTextureImage { uint64_t generation; int32_t origin_u, origin_v; uint32_t source_width, source_height; - /* 1: Beetle (native STP), 2: Duck non-ST, 3: Duck ST alpha encoding. */ + /* 1: Beetle (native STP), 2: Duck non-ST, 3: Duck ST alpha encoding, + * 4: composed cutout/STP/opaque classes (alpha 0/128/255). */ int alpha_mode; void* lease; } GpuHdTextureImage; /* Inclusive page-UV bounds, before native raster writes the destination. * Texture-window masks are applied to the queried footprint. Pending decode, - * ambiguity and unsupported geometry all return 0 for native sampling. */ + * ambiguity and unsupported geometry return native sampling. Ready partial + * replacements compose with native holes; pending parts never decode here. */ int gpu_hd_textures_acquire_draw(uint16_t texpage, uint16_t clut_x, uint16_t clut_y, const int limits[4], uint32_t texture_window, int semitransparent, diff --git a/runtime/include/texture_image_decode.h b/runtime/include/texture_image_decode.h new file mode 100644 index 00000000..848e7d4c --- /dev/null +++ b/runtime/include/texture_image_decode.h @@ -0,0 +1,37 @@ +#ifndef PSX_TEXTURE_IMAGE_DECODE_H +#define PSX_TEXTURE_IMAGE_DECODE_H + +#include +#include +#include +#include + +enum class TextureImageFormat { Png, Jpeg, WebP }; + +struct TextureImageProbe { + uint32_t width = 0; + uint32_t height = 0; + TextureImageFormat format = TextureImageFormat::Png; +}; + +struct DecodedTextureImage { + uint32_t width = 0; + uint32_t height = 0; + std::vector rgba; +}; + +constexpr size_t TEXTURE_IMAGE_MAX_ENCODED_BYTES = 64u * 1024u * 1024u; +constexpr size_t TEXTURE_IMAGE_DEFAULT_DECODED_BUDGET = 64u * 1024u * 1024u; +constexpr uint32_t TEXTURE_IMAGE_MAX_DIMENSION = 8192; + +// Encoded bytes are supplied by the caller: these helpers perform no I/O. +// Limits are checked before allocating decoded pixels. RGBA alpha is retained +// verbatim; the texture renderer separately classifies PSX transparency. +bool texture_image_probe(const uint8_t* encoded, size_t encoded_size, + size_t decoded_budget, TextureImageProbe& probe, + std::string& error); +bool texture_image_decode(const uint8_t* encoded, size_t encoded_size, + size_t decoded_budget, DecodedTextureImage& image, + std::string& error); + +#endif diff --git a/runtime/licenses/libwebp-NOTICES.txt b/runtime/licenses/libwebp-NOTICES.txt new file mode 100644 index 00000000..c6a25ef1 --- /dev/null +++ b/runtime/licenses/libwebp-NOTICES.txt @@ -0,0 +1,134 @@ +libwebp 1.6.0 (WebM Project authors), static decoder only. +Source commit: 4fa21912338357f89e4fd51cf2368325b59e9bd9 +Source: https://chromium.googlesource.com/webm/libwebp/+/4fa21912338357f89e4fd51cf2368325b59e9bd9 + +=== COPYING === +Copyright (c) 2010, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +=== PATENTS === +Additional IP Rights Grant (Patents) +------------------------------------ + +"These implementations" means the copyrightable works that implement the WebM +codecs distributed by Google as part of the WebM Project. + +Google hereby grants to you a perpetual, worldwide, non-exclusive, no-charge, +royalty-free, irrevocable (except as stated in this section) patent license to +make, have made, use, offer to sell, sell, import, transfer, and otherwise +run, modify and propagate the contents of these implementations of WebM, where +such license applies only to those patent claims, both currently owned by +Google and acquired in the future, licensable by Google that are necessarily +infringed by these implementations of WebM. This grant does not include claims +that would be infringed only as a consequence of further modification of these +implementations. If you or your agent or exclusive licensee institute or order +or agree to the institution of patent litigation or any other patent +enforcement activity against any entity (including a cross-claim or +counterclaim in a lawsuit) alleging that any of these implementations of WebM +or any code incorporated within any of these implementations of WebM +constitute direct or contributory patent infringement, or inducement of +patent infringement, then any patent rights granted to you under this License +for these implementations of WebM shall terminate as of the date such +litigation is filed. + +=== AUTHORS === +Contributors: +- Aidan O'Loan (aidanol at gmail dot com) +- Alan Browning (browning at google dot com) +- Alexandru Ardelean (ardeleanalex at gmail dot com) +- Anuraag Agrawal (anuraaga at gmail dot com) +- Arthur Eubanks (aeubanks at google dot com) +- Brian Ledger (brianpl at google dot com) +- Charles Munger (clm at google dot com) +- Cheng Yi (cyi at google dot com) +- Christian Duvivier (cduvivier at google dot com) +- Christopher Degawa (ccom at randomderp dot com) +- Clement Courbet (courbet at google dot com) +- devtools-clrobot at google dot com (devtools-clrobot@google dot com) +- Djordje Pesut (djordje dot pesut at imgtec dot com) +- Frank (1433351828 at qq dot com) +- Frank Barchard (fbarchard at google dot com) +- Henner Zeller (hzeller at google dot com) +- Hui Su (huisu at google dot com) +- H. Vetinari (h dot vetinari at gmx dot com) +- Ilya Kurdyukov (jpegqs at gmail dot com) +- Ingvar Stepanyan (rreverser at google dot com) +- Istvan Stefan (Istvan dot Stefan at arm dot com) +- James Zern (jzern at google dot com) +- Jan Engelhardt (jengelh at medozas dot de) +- Jehan (jehan at girinstud dot io) +- Jeremy Maitin-Shepard (jbms at google dot com) +- Johann Koenig (johann dot koenig at duck dot com) +- Jonathan Grant (jgrantinfotech at gmail dot com) +- Jonliu1993 (13720414433 at 163 dot com) +- Jovan Zelincevic (jovan dot zelincevic at imgtec dot com) +- Jyrki Alakuijala (jyrki at google dot com) +- Konstantin Ivlev (tomskside at gmail dot com) +- Lode Vandevenne (lode at google dot com) +- Lou Quillio (louquillio at google dot com) +- Mans Rullgard (mans at mansr dot com) +- Marcin Kowalczyk (qrczak at google dot com) +- Martin Olsson (mnemo at minimum dot se) +- Maryla Ustarroz-Calonge (maryla at google dot com) +- Masahiro Hanada (hanada at atmark-techno dot com) +- Mikołaj Zalewski (mikolajz at google dot com) +- Mislav Bradac (mislavm at google dot com) +- natewood (natewood at fb dot com) +- Nico Weber (thakis at chromium dot org) +- Noel Chromium (noel at chromium dot org) +- Nozomi Isozaki (nontan at pixiv dot co dot jp) +- Oliver Wolff (oliver dot wolff at qt dot io) +- Owen Rodley (orodley at google dot com) +- Ozkan Sezer (sezeroz at gmail dot com) +- Parag Salasakar (img dot mips1 at gmail dot com) +- Pascal Massimino (pascal dot massimino at gmail dot com) +- Paweł Hajdan, Jr (phajdan dot jr at chromium dot org) +- Pierre Joye (pierre dot php at gmail dot com) +- Roberto Alanis (alanisbaez at google dot com) +- Sam Clegg (sbc at chromium dot org) +- Scott Hancher (seh at google dot com) +- Scott LaVarnway (slavarnway at google dot com) +- Scott Talbot (s at chikachow dot org) +- Slobodan Prijic (slobodan dot prijic at imgtec dot com) +- Somnath Banerjee (somnath dot banerjee at gmail dot com) +- Sriraman Tallam (tmsriram at google dot com) +- Tamar Levy (tamar dot levy at intel dot com) +- Thiago Perrotta (tperrotta at google dot com) +- Timothy Gu (timothygu99 at gmail dot com) +- Urvang Joshi (urvang at google dot com) +- Vikas Arora (vikasa at google dot com) +- Vincent Rabaud (vrabaud at google dot com) +- Vlad Tsyrklevich (vtsyrklevich at chromium dot org) +- Wan-Teh Chang (wtc at google dot com) +- wrv (wrv at utexas dot edu) +- Yang Zhang (yang dot zhang at arm dot com) +- Yannis Guyon (yguyon at google dot com) +- Zhi An Ng (zhin at chromium dot org) diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index b9fdb59b..5090635c 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -21,6 +21,7 @@ include("${PSXRECOMP_ROOT}/runtime/chd_dependency.cmake") include("${PSXRECOMP_ROOT}/runtime/openxr_dependency.cmake") include("${PSXRECOMP_ROOT}/runtime/overlay_static_sources.cmake") include("${PSXRECOMP_ROOT}/runtime/netplay_dependency.cmake") +include("${PSXRECOMP_ROOT}/runtime/texture_image_dependency.cmake") # Default to an optimized build. The recompiled game is a huge (~270 MB) block of # generated C; with no CMAKE_BUILD_TYPE the compiler emits it at -O0 and the game @@ -1581,7 +1582,7 @@ function(psxrecomp_add_runtime_target target) ${generated_sources} ${PSXRT_EXTRAS_SOURCES} ) - target_link_libraries(${target} PRIVATE chdr-static) + target_link_libraries(${target} PRIVATE chdr-static psx_texture_image_decode) # audio_trace.c uses C11 atomics. Make the runtime's actual language # requirement explicit instead of relying on a parent project's global # CMAKE_C_STANDARD setting. cxx_std_17 likewise — game CMakeLists may omit diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index e24ed90a..ff44bfe7 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -10034,13 +10034,17 @@ static void handle_hd_textures(int id, const char *json) } char root_json[8192]; json_escape_string(root_json, sizeof(root_json), info.root ? info.root : ""); + char diagnostic_json[4096]; + json_escape_string(diagnostic_json, sizeof(diagnostic_json), + info.diagnostic ? info.diagnostic : ""); const int backend = gr_backend(); send_fmt("{\"id\":%d,\"ok\":true,\"root\":\"%s\",\"active\":%s," "\"replacements\":%s,\"dump\":%s,\"format\":%d," "\"backend\":\"%s\",\"replacement_backend_supported\":%s," "\"replacement_count\":%llu,\"draw_queries\":%llu," "\"matched_draws\":%llu,\"ready_draws\":%llu," - "\"applied_draws\":%llu,\"dumped_textures\":%llu}", + "\"applied_draws\":%llu,\"dumped_textures\":%llu," + "\"pending_dump_sources\":%llu,\"diagnostic\":\"%s\"}", id, root_json, info.active ? "true" : "false", info.replacements ? "true" : "false", info.dump ? "true" : "false", info.format, backend == GR_BACKEND_OPENGL ? "opengl" : @@ -10049,7 +10053,8 @@ static void handle_hd_textures(int id, const char *json) (unsigned long long)info.replacement_count, (unsigned long long)info.draw_queries, (unsigned long long)info.matched_draws, (unsigned long long)info.ready_draws, (unsigned long long)info.applied_draws, - (unsigned long long)info.dumped_textures); + (unsigned long long)info.dumped_textures, + (unsigned long long)info.pending_dump_sources, diagnostic_json); } static void handle_video_info(int id, const char *json) diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp index 6dfbfed6..fa0f86da 100644 --- a/runtime/src/duckstation_texture_pack.cpp +++ b/runtime/src/duckstation_texture_pack.cpp @@ -1,5 +1,5 @@ #include "duckstation_texture_pack.h" -#include "../third_party/stb_image.h" +#include "texture_image_decode.h" #define XXH_INLINE_ALL #include "../third_party/xxhash.h" #include "png_write.h" @@ -46,7 +46,10 @@ constexpr size_t kMaxDumpQueue = 8; constexpr size_t kMaxDecodeQueue = 32; constexpr size_t kDefaultBudget = 64u * 1024u * 1024u; constexpr size_t kMaxEncoded = 64u * 1024u * 1024u; -constexpr uint32_t kMaxImageDimension = 8192; +constexpr size_t kMaxSnapshots = 64u * 1024u * 1024u; +constexpr size_t kMaxPaletteRecords = 8192; +constexpr size_t kMaxDumpBytes = 32u * 1024u * 1024u; +constexpr size_t kMaxParts = 64; void error_text(char* dest, size_t capacity, const std::string& text) { if (dest && capacity) std::snprintf(dest, capacity, "%s", text.c_str()); @@ -68,6 +71,16 @@ bool intersects(Rect a, Rect b) { bool contains(Rect a, Rect b) { return b.x >= a.x && b.y >= a.y && b.x + b.w <= a.x + a.w && b.y + b.h <= a.y + a.h; } +Rect intersection(Rect a, Rect b) { + const unsigned x = std::max(a.x,b.x), y = std::max(a.y,b.y); + const unsigned right = std::min(a.x+a.w,b.x+b.w), bottom = std::min(a.y+a.h,b.y+b.h); + return {x,y,right > x ? right-x : 0,bottom > y ? bottom-y : 0}; +} +Rect united(Rect a, Rect b) { + if (!a.w || !a.h) return b; + const unsigned x = std::min(a.x,b.x), y = std::min(a.y,b.y); + return {x,y,std::max(a.x+a.w,b.x+b.w)-x,std::max(a.y+a.h,b.y+b.h)-y}; +} bool valid_rect(Rect r) { return r.w && r.h && r.x + r.w <= 1024 && r.y + r.h <= 512; } template bool number(const std::string& text, T& value, int base = 10) { @@ -130,7 +143,27 @@ uint64_t palette_hash(const HdTextureDrawQuery& q, unsigned first, unsigned last } struct Entry { DuckTextureKey key{}; std::string path; bool ambiguous = false; }; -struct Upload { Rect rect; uint64_t hash; }; +struct Observation { + Rect used{}; /* Original source-space native words. */ + std::array palette{}; + uint64_t palette_hash = 0; + uint16_t palette_size = 0, clut_x = 0, clut_y = 0; + uint16_t page_x = 0, page_y = 0; + uint8_t depth = 0; + bool semi = false; +}; +struct Source { + Rect rect{}; + uint64_t hash = 0; + std::vector words; + std::vector records; + DuckTexturePack* dump_owner = nullptr; + uint8_t kind = DUCK_TEXTURE_UPLOAD; + bool used = false; +}; +struct Upload { std::shared_ptr source; Rect active{}; unsigned splits = 0; }; +struct PageSource { std::shared_ptr source; uint8_t depth = 0; uint16_t clut_x = 0, clut_y = 0; }; +struct CopyUpdate { Rect original, active; unsigned splits; }; struct RectHash { Rect rect; uint64_t hash; const uint16_t* vram; }; struct PageGroup { Rect texels; @@ -139,7 +172,7 @@ struct PageGroup { struct MatchCache { HdTextureDrawQuery query; int st, status; - DuckTextureMatch result; + std::vector results; }; struct Image { std::vector rgba; uint32_t w = 0, h = 0; }; struct Lease { std::shared_ptr image; }; @@ -151,6 +184,7 @@ struct Decode { uint64_t use = 0; }; struct Dump { fs::path path; std::shared_ptr image; }; +struct DumpHistory { std::unordered_set names; std::deque order; }; /* png_write.h's C-compatible CRC table is mutable during initialization. * Different pack workers therefore serialize this TU's calls to the writer. */ std::mutex png_writer_mutex; @@ -186,15 +220,17 @@ bool publish_exclusive(const fs::path& temp, const fs::path& target) { } struct Worker { - std::mutex mutex; + mutable std::mutex mutex; std::condition_variable wake; std::unordered_map decoded; std::deque decode_queue; std::deque dump_queue; std::thread thread; size_t budget = kDefaultBudget, used = 0; + size_t dump_bytes = 0; uint64_t tick = 0; bool stop = false; + bool dump_busy = false; std::string dump_error; ~Worker() { { std::lock_guard lock(mutex); stop = true; } @@ -225,13 +261,16 @@ void worker_main(Worker* w) { w->wake.wait(lock, [&] { return w->stop || !w->decode_queue.empty() || !w->dump_queue.empty(); }); /* Finish queued dumps on shutdown so the final frame is durable. */ if (w->stop && w->dump_queue.empty()) return; - if (!w->stop && !w->decode_queue.empty()) { + if (!w->stop && w->dump_queue.empty() && !w->decode_queue.empty()) { id = w->decode_queue.front(); w->decode_queue.pop_front(); auto it = w->decoded.find(id); if (it == w->decoded.end()) continue; it->second.status = DecodeStatus::Loading; path = it->second.path; budget = w->budget; - } else { dump = std::move(w->dump_queue.front()); w->dump_queue.pop_front(); } + } else { + dump = std::move(w->dump_queue.front()); w->dump_queue.pop_front(); w->dump_busy = true; + w->wake.notify_all(); + } } if (id) { std::shared_ptr image; @@ -241,21 +280,11 @@ void worker_main(Worker* w) { if (length > 0 && static_cast(length) <= kMaxEncoded) { std::vector encoded(static_cast(length)); input.seekg(0); - int width = 0, height = 0, channels = 0; - constexpr uint8_t png_signature[] = {137,80,78,71,13,10,26,10}; + DecodedTextureImage decoded; std::string error; if (input.read(reinterpret_cast(encoded.data()), length) && - encoded.size() >= sizeof(png_signature) && - !std::memcmp(encoded.data(),png_signature,sizeof(png_signature)) && - stbi_info_from_memory(encoded.data(), static_cast(encoded.size()), &width, &height, &channels) && - width > 0 && height > 0 && width <= int(kMaxImageDimension) && height <= int(kMaxImageDimension) && - uint64_t(width) * height * 4 <= budget) { - stbi_uc* pixels = stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), - &width, &height, &channels, 4); - if (pixels) { - std::unique_ptr owned(pixels, &stbi_image_free); - image = std::make_shared(); image->w = width; image->h = height; - image->rgba.assign(pixels, pixels + size_t(width) * height * 4); - } + texture_image_decode(encoded.data(),encoded.size(),budget,decoded,error)) { + image = std::make_shared(); image->w = decoded.width; image->h = decoded.height; + image->rgba = std::move(decoded.rgba); } } } catch (...) { image.reset(); } @@ -274,7 +303,7 @@ void worker_main(Worker* w) { std::error_code ec; fs::create_directories(dump.path.parent_path(), ec); if (ec) throw std::runtime_error("cannot create texture dump directory: " + ec.message()); - if (fs::exists(dump.path,ec) && !ec) continue; + if (!(fs::exists(dump.path,ec) && !ec)) { FILE* file = create_exclusive(temporary); if (!file) throw std::runtime_error("cannot create temporary texture dump PNG"); created = true; @@ -286,15 +315,19 @@ void worker_main(Worker* w) { if (std::fclose(file) != 0 || !success) throw std::runtime_error("cannot write texture dump PNG"); publish_exclusive(temporary,dump.path); fs::remove(temporary,ec); + } } catch (const std::exception& e) { if (created) { std::error_code ignored; fs::remove(temporary,ignored); } std::lock_guard lock(w->mutex); w->dump_error = e.what(); - std::fprintf(stderr,"[texture-pack] dump failed: %s\n",e.what()); } catch (...) { if (created) { std::error_code ignored; fs::remove(temporary,ignored); } std::lock_guard lock(w->mutex); w->dump_error = "texture dump allocation failed"; - std::fprintf(stderr,"[texture-pack] dump failed: allocation failed\n"); } + { + std::lock_guard lock(w->mutex); + w->dump_bytes -= dump.image->rgba.size(); w->dump_busy = false; + } + w->wake.notify_all(); } } } @@ -302,15 +335,27 @@ void worker_main(Worker* w) { struct DuckTexturePack { std::string root, diagnostic; + mutable std::string reported_diagnostic; fs::path replacements, dumps; std::vector entries; std::array>, 3> uploads_by_hash; std::array, 3> page_groups; std::unordered_map by_name; std::vector uploads; + std::vector pages; + std::deque> sources; + std::vector copy_updates; + Rect copy_destination{}; + bool copy_pending = false; std::vector rect_cache; std::vector match_cache; - std::unordered_set dumped; + std::shared_ptr dump_history = std::make_shared(); + DuckTextureConfig config{0,0,0,16,16,128,128,0,0,0,1,0,1,0,0}; + size_t snapshot_bytes = 0, palette_records = 0; + size_t queued_dumps = 0; + size_t snapshot_budget = kMaxSnapshots, rect_cache_limit = kMaxRectCache; + uint64_t revision = 1; + std::string lifecycle_error; size_t ignored = 0, ambiguous = 0; bool geometry_limit_reported = false, dump_limit_reported = false; Worker worker; @@ -345,8 +390,10 @@ uint64_t cached_hash(DuckTexturePack& p, const HdTextureDrawQuery& q, Rect r) { for (const auto& saved : p.rect_cache) if (saved.vram == q.vram && saved.rect.x == r.x && saved.rect.y == r.y && saved.rect.w == r.w && saved.rect.h == r.h) return saved.hash; const uint64_t value = duck_texture_hash_rect(q.vram, q.vram_word_count, r.x, r.y, r.w, r.h); - if (p.rect_cache.size() >= kMaxRectCache) p.rect_cache.erase(p.rect_cache.begin()); - p.rect_cache.push_back({r, value, q.vram}); + if (p.rect_cache_limit) { + if (p.rect_cache.size() >= p.rect_cache_limit) p.rect_cache.erase(p.rect_cache.begin()); + p.rect_cache.push_back({r, value, q.vram}); + } return value; } std::string unquote(std::string s) { @@ -356,6 +403,53 @@ std::string unquote(std::string s) { const auto comment = s.find(" #"); return comment == std::string::npos ? s : trim(s.substr(0, comment)); } +bool image_extension(const fs::path& path) { + const std::string ext = lower(path.extension().u8string()); + return ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".webp"; +} +bool option(DuckTexturePack& p, const std::string& key, const std::string& value) { + const std::pair integers[] = { + {"MaxVRAMWriteSplits",&DuckTextureConfig::max_vram_write_splits}, + {"MaxVRAMWriteCoalesceWidth",&DuckTextureConfig::max_vram_write_coalesce_width}, + {"MaxVRAMWriteCoalesceHeight",&DuckTextureConfig::max_vram_write_coalesce_height}, + {"DumpTextureWidthThreshold",&DuckTextureConfig::texture_dump_width_threshold}, + {"DumpTextureHeightThreshold",&DuckTextureConfig::texture_dump_height_threshold}, + {"DumpVRAMWriteWidthThreshold",&DuckTextureConfig::vram_write_dump_width_threshold}, + {"DumpVRAMWriteHeightThreshold",&DuckTextureConfig::vram_write_dump_height_threshold} + }; + for (const auto& item : integers) if (key == item.first) { + uint16_t parsed = 0; + if (number(value,parsed)) p.config.*item.second = parsed; + else note(p,"invalid config.yaml integer: " + key); + return true; + } + const std::pair booleans[] = { + {"DumpTexturePages",&DuckTextureConfig::dump_texture_pages}, + {"DumpFullTexturePages",&DuckTextureConfig::dump_full_texture_pages}, + {"DumpTextureForceAlphaChannel",&DuckTextureConfig::dump_texture_force_alpha_channel}, + {"DumpVRAMWriteForceAlphaChannel",&DuckTextureConfig::dump_vram_write_force_alpha_channel}, + {"DumpC16Textures",&DuckTextureConfig::dump_c16_textures}, + {"ReducePaletteRange",&DuckTextureConfig::reduce_palette_range}, + {"ConvertCopiesToWrites",&DuckTextureConfig::convert_copies_to_writes}, + {"ReplacementScaleLinearFilter",&DuckTextureConfig::replacement_scale_linear_filter} + }; + for (const auto& item : booleans) if (key == item.first) { + const std::string parsed = lower(value); + if (parsed == "true" || parsed == "1" || parsed == "yes" || parsed == "on") p.config.*item.second = 1; + else if (parsed == "false" || parsed == "0" || parsed == "no" || parsed == "off") p.config.*item.second = 0; + else note(p,"invalid config.yaml boolean: " + key); + return true; + } + if (key == "MaxHashCacheEntries" || key == "MaxHashCacheVRAMUsageMB" || key == "MaxReplacementCacheVRAMUsage") { + uint32_t parsed = 0; + if (!number(value,parsed)) note(p,"invalid config.yaml cache limit: " + key); + else if (key == "MaxHashCacheEntries") p.rect_cache_limit = std::min(parsed,4096); + else if (key == "MaxHashCacheVRAMUsageMB") p.snapshot_budget = std::min(uint64_t(parsed)*1024*1024,kMaxSnapshots); + else p.worker.budget = std::min(uint64_t(parsed)*1024*1024,kDefaultBudget); + return true; + } + return false; +} void load_config(DuckTexturePack& p, const fs::path& path) { std::error_code ec; if (!fs::exists(path, ec)) return; @@ -370,8 +464,12 @@ void load_config(DuckTexturePack& p, const fs::path& path) { const std::string key = unquote(text.substr(0, colon)); std::string value = unquote(text.substr(colon + 1)); const bool indented = line.front() == ' ' || line.front() == '\t'; - if (!indented) { aliases = key == "Aliases"; if (!aliases) note(p, "config.yaml option not applied: " + key); continue; } - if (!aliases) continue; + if (!indented) { + aliases = key == "Aliases"; + if (!aliases && !option(p,key,value)) note(p,"unsupported config.yaml option: " + key); + continue; + } + if (!aliases) { note(p,"unsupported nested config.yaml option: " + key); continue; } DuckTextureKey parsed{}; if (!duck_texture_parse_name(key.c_str(), &parsed)) { note(p, "invalid alias key"); continue; } if (value == "|" || value == "|-" || value == ">" || value == ">-") { @@ -381,7 +479,7 @@ void load_config(DuckTexturePack& p, const fs::path& path) { fs::path relative = fs::u8path(value); bool unsafe = relative.empty() || relative.is_absolute() || relative.has_root_name(); for (const auto& part : relative) if (part == "..") unsafe = true; - if (unsafe || lower(relative.extension().string()) != ".png") { note(p, "unsupported alias path"); continue; } + if (unsafe || !image_extension(relative)) { note(p, "unsupported alias path"); continue; } const fs::path target = p.replacements / relative; if (!fs::is_regular_file(target, ec) || ec) { note(p, "alias target missing"); ec.clear(); continue; } const auto old = p.by_name.find(stem(parsed)); @@ -389,6 +487,169 @@ void load_config(DuckTexturePack& p, const fs::path& path) { if (old == p.by_name.end()) add_entry(p, parsed, target); } } + +uint64_t snapshot_hash(const Source& source, Rect crop) { + XXH3_state_t state; XXH3_64bits_reset(&state); + std::array bytes{}; + for (unsigned y = crop.y; y < crop.y+crop.h; ++y) { + const uint16_t* row = source.words.data()+size_t(y)*source.rect.w+crop.x; + for (unsigned x = 0; x < crop.w; ++x) { bytes[2*x] = row[x]&255; bytes[2*x+1] = row[x]>>8; } + XXH3_64bits_update(&state,bytes.data(),crop.w*2); + } + return XXH3_64bits_digest(&state); +} +bool enqueue_dump(DuckTexturePack& p, const std::string& name, std::shared_ptr image) { + if (p.dump_history->names.count(name)) return true; + Worker& w = p.worker; + std::unique_lock lock(w.mutex); + if (!w.thread.joinable()) w.thread = std::thread(worker_main,&w); + const size_t bytes = image->rgba.size(); + w.wake.wait(lock,[&] { return !w.dump_error.empty() || + (w.dump_queue.size() < kMaxDumpQueue && w.dump_bytes+bytes <= kMaxDumpBytes); }); + if (!w.dump_error.empty()) { p.lifecycle_error = w.dump_error; return false; } + /* Bound metadata without permanently stopping authoring. Old names may + * be requeued, but exclusive publication still preserves existing files. */ + if (p.dump_history->names.size() >= kMaxDumps) { p.dump_history->names.erase(p.dump_history->order.front()); p.dump_history->order.pop_front(); } + p.dump_history->names.insert(name); p.dump_history->order.push_back(name); + w.dump_bytes += bytes; + w.dump_queue.push_back({p.dumps/(name+".png"),std::move(image)}); + ++p.queued_dumps; + w.wake.notify_one(); + return true; +} +bool finalize_source(DuckTexturePack& p, Source& source) { + if (source.dump_owner != &p) return true; + bool success = true; + for (const Observation& record : source.records) { + const unsigned ppw = expansion(record.depth); + Rect crop = record.used; + if (source.kind == DUCK_TEXTURE_PAGE && p.config.dump_full_texture_pages) + crop = {0,0,source.rect.w,source.rect.h}; + if (record.depth == 2 && !p.config.dump_c16_textures) continue; + if (crop.w*ppw < p.config.texture_dump_width_threshold || crop.h < p.config.texture_dump_height_threshold) continue; + DuckTextureKey key{}; + key.kind = source.kind; key.depth = record.depth; + key.semitransparent = record.semi && !p.config.dump_texture_force_alpha_channel; + key.source_hash = source.kind == DUCK_TEXTURE_UPLOAD ? source.hash : snapshot_hash(source,crop); + key.source_width_words = static_cast(source.rect.w); + key.source_height = static_cast(source.rect.h); + key.offset_x = static_cast(crop.x*ppw); key.offset_y = static_cast(crop.y); + key.width = static_cast(crop.w*ppw); key.height = static_cast(crop.h); + key.palette_hash = record.palette_hash; + if (record.depth < 2) { + unsigned minimum = 0, maximum = record.depth == 0 ? 15 : 255; + if (p.config.reduce_palette_range) { + minimum = maximum; maximum = 0; + const unsigned mask = record.depth == 0 ? 15 : 255; + const unsigned bits = record.depth == 0 ? 4 : 8; + for (unsigned y = crop.y; y < crop.y+crop.h; ++y) + for (unsigned x = crop.x; x < crop.x+crop.w; ++x) { + const uint16_t word = source.words[size_t(y)*source.rect.w+x]; + for (unsigned n = 0; n < ppw; ++n) { + const unsigned index = (word>>(n*bits))&mask; + minimum = std::min(minimum,index); maximum = std::max(maximum,index); + } + } + minimum = std::min(minimum,record.palette_size-1); + maximum = std::min(maximum,record.palette_size-1); + /* Pinned upstream hashes the FIRST range-size entries, not an + * offset into the palette. Keep that wire-format quirk. */ + key.palette_hash = duck_texture_hash_words_le(record.palette.data(),maximum-minimum+1); + } + key.palette_min = static_cast(minimum); key.palette_max = static_cast(maximum); + } + const std::string name = stem(key); + if (name.empty() || p.dump_history->names.count(name)) continue; + auto image = std::make_shared(); image->w = key.width; image->h = key.height; + image->rgba.resize(size_t(image->w)*image->h*4); + for (unsigned y = 0; y < image->h; ++y) for (unsigned x = 0; x < image->w; ++x) { + uint16_t color = source.words[size_t(crop.y+y)*source.rect.w+crop.x+x/ppw]; + if (record.depth < 2) { + const unsigned bits = record.depth == 0 ? 4 : 8; + const unsigned index = (color>>((x%ppw)*bits))&(record.depth == 0 ? 15 : 255); + color = record.palette[index]; + } + uint8_t* pixel = image->rgba.data()+(size_t(y)*image->w+x)*4; + for (unsigned c = 0; c < 3; ++c) pixel[c] = static_cast((((color>>(5*c))&31)*255+15)/31); + pixel[3] = p.config.dump_texture_force_alpha_channel ? 255 : color == 0 ? 0 : + key.semitransparent && (color&0x8000) ? 143 : 255; + } + if (!enqueue_dump(p,name,std::move(image))) success = false; + } + if (success) { p.palette_records -= source.records.size(); source.records.clear(); } + return success; +} +bool has_residency(const DuckTexturePack& p, const Source* source) { + for (const auto& upload : p.uploads) if (upload.source.get() == source) return true; + for (const auto& page : p.pages) if (page.source.get() == source) return true; + return false; +} +void retire_unused(DuckTexturePack& p) { + for (auto it = p.sources.begin(); it != p.sources.end();) { + if (has_residency(p,it->get())) { ++it; continue; } + if (!finalize_source(p,**it)) { + /* Keep failed pending records bounded and visible to the caller. */ + p.lifecycle_error = p.lifecycle_error.empty() ? "texture dump could not be finalized" : p.lifecycle_error; + ++it; continue; + } + p.snapshot_bytes -= (*it)->words.size()*sizeof(uint16_t); it = p.sources.erase(it); + } +} +std::shared_ptr capture_source(DuckTexturePack& p, Rect rect, const uint16_t* vram, uint8_t kind) { + const size_t bytes = size_t(rect.w)*rect.h*sizeof(uint16_t); + if (bytes > p.snapshot_budget) { + p.lifecycle_error = "upload snapshot exceeds bounded authoring budget"; + return {}; + } + while (p.snapshot_bytes+bytes > p.snapshot_budget || p.uploads.size() >= kMaxUploads || p.sources.size() >= kMaxUploads) { + if (p.sources.empty()) return {}; + const auto victim = p.sources.front(); + if (!finalize_source(p,*victim)) return {}; + p.uploads.erase(std::remove_if(p.uploads.begin(),p.uploads.end(),[&](const Upload& u) { return u.source == victim; }),p.uploads.end()); + p.pages.erase(std::remove_if(p.pages.begin(),p.pages.end(),[&](const PageSource& u) { return u.source == victim; }),p.pages.end()); + p.snapshot_bytes -= victim->words.size()*sizeof(uint16_t); p.sources.pop_front(); + p.match_cache.clear(); + } + auto source = std::make_shared(); source->rect = rect; source->kind = kind; source->dump_owner = &p; + source->words.resize(size_t(rect.w)*rect.h); + for (unsigned y = 0; y < rect.h; ++y) + std::copy_n(vram+(rect.y+y)*1024+rect.x,rect.w,source->words.data()+size_t(y)*rect.w); + source->hash = duck_texture_hash_words_le(source->words.data(),source->words.size()); + p.snapshot_bytes += bytes; p.sources.push_back(source); + return source; +} +bool record_usage(DuckTexturePack& p, Source& source, Rect global, const HdTextureDrawQuery& q, bool semi) { + source.used = true; + Observation current{}; current.depth = q.depth; current.semi = semi; + current.page_x = q.page_x; current.page_y = q.page_y; + current.clut_x = q.clut_x; current.clut_y = q.clut_y; + current.used = {global.x-source.rect.x,global.y-source.rect.y,global.w,global.h}; + if (q.depth < 2) { + if (q.clut_x >= 1024 || q.clut_x%16 || q.clut_y >= 512) return false; + current.palette_size = static_cast(std::min(q.depth == 0 ? 16u : 256u,1024u-q.clut_x)); + std::copy_n(q.vram+q.clut_y*1024+q.clut_x,current.palette_size,current.palette.data()); + current.palette_hash = duck_texture_hash_words_le(current.palette.data(),current.palette_size); + } + for (Observation& previous : source.records) { + if (previous.depth != current.depth || previous.palette_hash != current.palette_hash || + previous.palette != current.palette || (q.depth < 2 && + (previous.clut_x != q.clut_x || previous.clut_y != q.clut_y))) continue; + const Rect combined = united(previous.used,current.used); + const bool changed = combined.x != previous.used.x || combined.y != previous.used.y || + combined.w != previous.used.w || combined.h != previous.used.h || (!previous.semi && semi); + previous.used = combined; previous.semi |= semi; + return changed; + } + if (p.palette_records >= kMaxPaletteRecords) { + for (const auto& saved : p.sources) if (saved->dump_owner == &p && !saved->records.empty()) { + if (!finalize_source(p,*saved)) return false; + if (p.palette_records < kMaxPaletteRecords) break; + } + } + if (p.palette_records >= kMaxPaletteRecords) { p.lifecycle_error = "palette record budget exhausted"; return false; } + source.records.push_back(std::move(current)); ++p.palette_records; + return true; +} } // namespace extern "C" { @@ -399,7 +660,7 @@ int duck_texture_parse_name(const char* filename, DuckTextureKey* out) { if (text.size() > 192 || text.find_first_of("/\\") != std::string::npos) return 0; const auto dot = text.find('.'); if (dot != std::string::npos) { - if (lower(text.substr(dot)) != ".png") return 0; + if (!image_extension(fs::u8path(text))) return 0; text.resize(dot); } std::vector parts; @@ -499,7 +760,7 @@ int duck_texture_pack_create(const char* root, DuckTexturePack** out, char* erro if (it->is_regular_file(ec)) { const std::string name = it->path().filename().u8string(); if (name.rfind("texupload-", 0) != 0 && name.rfind("texpage-", 0) != 0 && name.rfind("vram-write-", 0) != 0) continue; - if (lower(it->path().extension().string()) != ".png") { note(*p, "replacement format outside PNG subset"); continue; } + if (!image_extension(it->path())) { note(*p, "unsupported replacement image format"); continue; } if (paths.size() >= kMaxEntries) throw std::runtime_error("texture replacement file limit (262144) exceeded"); paths.push_back(it->path()); } @@ -519,41 +780,109 @@ int duck_texture_pack_create(const char* root, DuckTexturePack** out, char* erro } catch (const std::exception& e) { error_text(error, capacity, e.what()); return 0; } catch (...) { error_text(error, capacity, "texture pack allocation failed"); return 0; } } -void duck_texture_pack_destroy(DuckTexturePack* p) { delete p; } +void duck_texture_pack_destroy(DuckTexturePack* p) { + if (p) { duck_texture_pack_flush_dumps(p,nullptr,0); delete p; } +} void duck_texture_pack_get_info(const DuckTexturePack* p, DuckTexturePackInfo* out) { if (!out) return; *out = {}; - if (p) { out->root = p->root.c_str(); out->diagnostic = p->diagnostic.c_str(); out->replacement_count = p->entries.size(); - out->ambiguous_count = p->ambiguous; out->ignored_count = p->ignored; } + if (p) { + p->reported_diagnostic = p->diagnostic; + if (!p->lifecycle_error.empty()) p->reported_diagnostic += "; " + p->lifecycle_error; + { std::lock_guard lock(p->worker.mutex); + if (!p->worker.dump_error.empty()) p->reported_diagnostic += "; " + p->worker.dump_error; } + out->root = p->root.c_str(); out->diagnostic = p->reported_diagnostic.c_str(); out->replacement_count = p->entries.size(); + out->ambiguous_count = p->ambiguous; out->ignored_count = p->ignored; out->queued_dump_count = p->queued_dumps; + out->snapshot_bytes = p->snapshot_bytes; + for (const auto& source : p->sources) if (source->dump_owner == p && !source->records.empty()) { + ++out->pending_dump_sources; out->pending_palette_records += source->records.size(); + } + } } +void duck_texture_pack_get_config(const DuckTexturePack* p, DuckTextureConfig* out) { if (out) *out = p ? p->config : DuckTextureConfig{}; } +int duck_texture_pack_linear_filter(const DuckTexturePack* p) { return p && p->config.replacement_scale_linear_filter; } +uint64_t duck_texture_pack_revision(const DuckTexturePack* p) { return p ? p->revision : 0; } void duck_texture_pack_invalidate(DuckTexturePack* p, uint16_t x, uint16_t y, uint16_t w, uint16_t h) { if (!p || !w || !h) return; if (w > 1024 || h > 512) { duck_texture_pack_reset_tracking(p); return; } + try { const unsigned px = x % 1024, py = y % 512; const unsigned w0 = std::min(unsigned(w), 1024 - px), h0 = std::min(unsigned(h), 512 - py); const std::array regions{{{px,py,w0,h0},{0,py,unsigned(w)-w0,h0}, {px,0,w0,unsigned(h)-h0},{0,0,unsigned(w)-w0,unsigned(h)-h0}}}; + ++p->revision; for (const auto& r : regions) if (r.w && r.h) { - const size_t old_uploads = p->uploads.size(); - p->uploads.erase(std::remove_if(p->uploads.begin(), p->uploads.end(), [&](const Upload& u){ return intersects(u.rect,r); }), p->uploads.end()); + std::vector survivors; survivors.reserve(p->uploads.size()); + for (const auto& upload : p->uploads) { + const Rect cut = intersection(upload.active,r); + if (!cut.w || !cut.h) { if (survivors.size() < kMaxUploads) survivors.push_back(upload); continue; } + if (upload.splits >= p->config.max_vram_write_splits || contains(r,upload.active)) continue; + const Rect a = upload.active; + const Rect fragments[] = { + {a.x,a.y,a.w,cut.y-a.y}, {a.x,cut.y+cut.h,a.w,a.y+a.h-cut.y-cut.h}, + {a.x,cut.y,cut.x-a.x,cut.h}, {cut.x+cut.w,cut.y,a.x+a.w-cut.x-cut.w,cut.h} + }; + if (survivors.size()+4 > kMaxUploads) continue; + for (const Rect& fragment : fragments) if (fragment.w && fragment.h) + survivors.push_back({upload.source,fragment,upload.splits+1}); + } + p->uploads.swap(survivors); + p->pages.erase(std::remove_if(p->pages.begin(),p->pages.end(),[&](const PageSource& page) { + if (intersects(page.source->rect,r)) return true; + for (const auto& record : page.source->records) + if (record.depth < 2 && intersects({record.clut_x,record.clut_y,record.palette_size,1},r)) return true; + return false; + }),p->pages.end()); p->rect_cache.erase(std::remove_if(p->rect_cache.begin(), p->rect_cache.end(), [&](const RectHash& v){ return intersects(v.rect,r); }), p->rect_cache.end()); - if (p->uploads.size() != old_uploads) p->match_cache.clear(); - else p->match_cache.erase(std::remove_if(p->match_cache.begin(), p->match_cache.end(), [&](const MatchCache& c) { + p->match_cache.erase(std::remove_if(p->match_cache.begin(), p->match_cache.end(), [&](const MatchCache& c) { const auto& q = c.query; const Rect page{q.page_x,q.page_y,256 / expansion(q.depth),256}; const Rect clut{q.clut_x,q.clut_y,std::min(q.depth == 0 ? 16u : 256u,1024u-q.clut_x),1}; return intersects(page,r) || (q.depth < 2 && intersects(clut,r)); }), p->match_cache.end()); } + retire_unused(*p); + } catch (...) { + /* Allocation/finalization failure must never preserve stale residency + * after a native write. Pending immutable observations remain owned. */ + p->uploads.clear(); p->pages.clear(); p->rect_cache.clear(); p->match_cache.clear(); + } +} +void duck_texture_pack_reset_tracking(DuckTexturePack* p) { + if (!p) return; + const bool flushed = duck_texture_pack_flush_dumps(p,nullptr,0) >= 0; + p->uploads.clear(); p->pages.clear(); p->copy_updates.clear(); p->copy_pending = false; + if (flushed) { p->sources.clear(); p->snapshot_bytes = 0; p->palette_records = 0; } + p->rect_cache.clear(); p->match_cache.clear(); ++p->revision; } -void duck_texture_pack_reset_tracking(DuckTexturePack* p) { if (p) { p->uploads.clear(); p->rect_cache.clear(); p->match_cache.clear(); } } int duck_texture_pack_copy_tracking(DuckTexturePack* destination, const DuckTexturePack* source) { if (!destination || !source) return 0; + if (destination == source) return 1; try { auto uploads = source->uploads; + auto pages = source->pages; + std::deque> sources; + std::unordered_map> cloned; + size_t bytes = 0, records = 0; + const auto clone = [&](const std::shared_ptr& item) { + const auto found = cloned.find(item.get()); + if (found != cloned.end()) return found->second; + auto copy = std::make_shared(*item); copy->dump_owner = destination; + cloned.emplace(item.get(),copy); sources.push_back(copy); + bytes += copy->words.size()*sizeof(uint16_t); records += copy->records.size(); + return copy; + }; + for (auto& upload : uploads) upload.source = clone(upload.source); + for (auto& page : pages) page.source = clone(page.source); + if (bytes > destination->snapshot_budget || duck_texture_pack_flush_dumps(destination,nullptr,0) < 0) return 0; destination->uploads.swap(uploads); + destination->pages.swap(pages); destination->sources.swap(sources); + destination->snapshot_bytes = bytes; destination->palette_records = records; + if (destination->root == source->root) destination->dump_history = source->dump_history; + destination->copy_updates.clear(); destination->copy_pending = false; destination->rect_cache.clear(); destination->match_cache.clear(); + ++destination->revision; return 1; } catch (...) { return 0; } } @@ -563,11 +892,60 @@ int duck_texture_pack_track_upload(DuckTexturePack* p, uint16_t x, uint16_t y, u duck_texture_pack_invalidate(p, x, y, w, h); if (!vram || count < kWords || !valid_rect({x,y,w,h})) return 0; try { - if (p->uploads.size() >= kMaxUploads) p->uploads.erase(p->uploads.begin()); - p->uploads.push_back({{x,y,w,h},duck_texture_hash_rect(vram,count,x,y,w,h)}); + const Rect written{x,y,w,h}; + if (!p->uploads.empty()) { + const Upload last = p->uploads.back(); const Rect before = last.source->rect; + const bool vertical = h <= p->config.max_vram_write_coalesce_height && before.x == x && before.w == w && before.y+before.h == y; + const bool horizontal = w <= p->config.max_vram_write_coalesce_width && before.y == y && before.h == h && before.x+before.w == x; + if (!last.splits && !last.source->used && (vertical || horizontal)) { + const Rect combined = united(before,written); + p->uploads.pop_back(); retire_unused(*p); + auto merged = capture_source(*p,combined,vram,DUCK_TEXTURE_UPLOAD); + if (!merged) return 0; + p->uploads.push_back({merged,combined,0}); p->match_cache.clear(); return 1; + } + } + auto captured = capture_source(*p,written,vram,DUCK_TEXTURE_UPLOAD); + if (!captured) return 0; + p->uploads.push_back({captured,written,0}); p->match_cache.clear(); return 1; } catch (...) { return 0; } } +int duck_texture_pack_begin_copy(DuckTexturePack* p, uint16_t sx, uint16_t sy, uint16_t dx, uint16_t dy, + uint16_t w, uint16_t h, const uint16_t* vram, size_t count) { + if (!p || !vram || count < kWords) return 0; + p->copy_updates.clear(); p->copy_pending = false; + const Rect destination{dx,dy,w,h}; + if (!valid_rect({sx,sy,w,h}) || !valid_rect(destination)) { duck_texture_pack_invalidate(p,dx,dy,w,h); return 0; } + try { + if (p->config.convert_copies_to_writes) { + for (const auto& upload : p->uploads) if (intersects(upload.active,destination)) { + if (!finalize_source(*p,*upload.source)) return 0; + if (contains(upload.active,destination)) + p->copy_updates.push_back({upload.source->rect,upload.active,upload.splits}); + } + p->uploads.erase(std::remove_if(p->uploads.begin(),p->uploads.end(),[&](const Upload& upload) { + return contains(upload.active,destination); + }),p->uploads.end()); + } + duck_texture_pack_invalidate(p,dx,dy,w,h); + p->copy_destination = destination; p->copy_pending = true; + return 1; + } catch (...) { p->copy_updates.clear(); return 0; } +} +int duck_texture_pack_end_copy(DuckTexturePack* p, const uint16_t* vram, size_t count) { + if (!p || !p->copy_pending || !vram || count < kWords) return 0; + p->copy_pending = false; + try { + for (const auto& update : p->copy_updates) { + auto replacement = capture_source(*p,update.original,vram,DUCK_TEXTURE_UPLOAD); + if (!replacement) { p->copy_updates.clear(); return 0; } + p->uploads.push_back({replacement,update.active,update.splits}); + } + p->copy_updates.clear(); p->match_cache.clear(); ++p->revision; + return 1; + } catch (...) { p->copy_updates.clear(); return 0; } +} int duck_texture_pack_match(DuckTexturePack* p, const HdTextureDrawQuery* q, DuckTextureMatch* out) { return duck_texture_pack_match_draw(p,q,0,out); @@ -575,6 +953,22 @@ int duck_texture_pack_match(DuckTexturePack* p, const HdTextureDrawQuery* q, Duc int duck_texture_pack_match_draw(DuckTexturePack* p, const HdTextureDrawQuery* q, int st, DuckTextureMatch* out) { if (out) *out = {}; if (!p || !q || !out) return HD_TEXTURE_LOOKUP_ERROR; + std::array parts{}; size_t count = 0; + const int status = duck_texture_pack_match_parts(p,q,st,parts.data(),parts.size(),&count); + if (status != HD_TEXTURE_LOOKUP_FOUND) return status; + size_t winner = size_t(-1); + for (size_t i = 0; i < count; ++i) if (parts[i].clip_u <= q->u_first && parts[i].clip_v <= q->v_first && + parts[i].clip_u+parts[i].clip_width > q->u_last && parts[i].clip_v+parts[i].clip_height > q->v_last) { + if (winner != size_t(-1)) return HD_TEXTURE_LOOKUP_AMBIGUOUS; + winner = i; + } + if (winner == size_t(-1)) return HD_TEXTURE_LOOKUP_NONE; + *out = parts[winner]; return HD_TEXTURE_LOOKUP_FOUND; +} +int duck_texture_pack_match_parts(DuckTexturePack* p, const HdTextureDrawQuery* q, int st, + DuckTextureMatch* output, size_t capacity, size_t* out_count) { + if (out_count) *out_count = 0; + if (!p || !q || !out_count || (!output && capacity)) return HD_TEXTURE_LOOKUP_ERROR; Rect wanted{}; if (!query_rect(q, wanted)) return HD_TEXTURE_LOOKUP_NONE; try { for (const auto& c : p->match_cache) { @@ -582,61 +976,96 @@ int duck_texture_pack_match_draw(DuckTexturePack* p, const HdTextureDrawQuery* q if (c.st == (st != 0) && a.vram == q->vram && a.page_x == q->page_x && a.page_y == q->page_y && a.depth == q->depth && a.clut_x == q->clut_x && a.clut_y == q->clut_y && a.u_first == q->u_first && a.u_last == q->u_last && a.v_first == q->v_first && a.v_last == q->v_last) { - *out = c.result; return c.status; + if (c.results.size() > capacity) return HD_TEXTURE_LOOKUP_ERROR; + std::copy(c.results.begin(),c.results.end(),output); *out_count = c.results.size(); return c.status; } } const unsigned ppw = expansion(q->depth); - size_t winner = size_t(-1); int origin_u = 0, origin_v = 0, rank = 2; - bool ambiguous = false; - const auto consider = [&](size_t index, int u, int v) { + std::vector candidates; + bool over_budget = false; size_t checked = 0; + const auto consider = [&](size_t index, int u, int v, Rect surviving) { + if (candidates.size() >= kMaxParts*4) { over_budget = true; return; } const Entry& e = p->entries[index]; const auto& k = e.key; - const int candidate_rank = k.semitransparent == (st != 0) ? 0 : 1; - if (candidate_rank > rank) return; bool valid = false; if (palette_hash(*q,k.palette_min,k.palette_max,valid) != k.palette_hash || !valid) return; - if (candidate_rank < rank) { rank = candidate_rank; winner = size_t(-1); ambiguous = false; } - if (e.ambiguous || winner != size_t(-1)) ambiguous = true; - winner = index; origin_u = u; origin_v = v; + const int left = std::max({u,(int(surviving.x)-q->page_x)*int(ppw),int(q->u_first)}); + const int top = std::max({v,int(surviving.y)-q->page_y,int(q->v_first)}); + const int right = std::min({u+k.width,(int(surviving.x+surviving.w)-q->page_x)*int(ppw),int(q->u_last)+1}); + const int bottom = std::min({v+k.height,int(surviving.y+surviving.h)-q->page_y,int(q->v_last)+1}); + if (right <= left || bottom <= top) return; + DuckTextureMatch part{}; part.key = k; part.entry_id = index+1; part.replacement_path = e.path.c_str(); + part.origin_u = u; part.origin_v = v; part.source_width = k.width; part.source_height = k.height; + part.clip_u = left; part.clip_v = top; part.clip_width = static_cast(right-left); part.clip_height = static_cast(bottom-top); + candidates.push_back(part); }; for (const auto& upload : p->uploads) { - if (!contains(upload.rect,wanted)) continue; - const auto found = p->uploads_by_hash[q->depth].find(upload.hash); + if (!intersects(upload.active,wanted)) continue; + upload.source->used = true; + const Rect source = upload.source->rect; + const auto found = p->uploads_by_hash[q->depth].find(upload.source->hash); if (found == p->uploads_by_hash[q->depth].end()) continue; for (const size_t index : found->second) { + if (++checked > 4096) { over_budget = true; break; } const auto& k = p->entries[index].key; - if (upload.rect.w != k.source_width_words || upload.rect.h != k.source_height) continue; - Rect rect{upload.rect.x+k.offset_x/ppw,upload.rect.y+k.offset_y,unsigned(k.width)/ppw,k.height}; - if (contains(rect,wanted)) consider(index,(int(upload.rect.x)-q->page_x)*int(ppw)+k.offset_x, - int(upload.rect.y)-q->page_y+k.offset_y); + if (source.w != k.source_width_words || source.h != k.source_height) continue; + Rect rect{source.x+k.offset_x/ppw,source.y+k.offset_y,unsigned(k.width)/ppw,k.height}; + if (intersects(rect,wanted)) consider(index,(int(source.x)-q->page_x)*int(ppw)+k.offset_x, + int(source.y)-q->page_y+k.offset_y,upload.active); + if (over_budget) break; } + if (over_budget) break; } - size_t groups_checked = 0, hashed_bytes = 0; bool over_budget = false; + size_t groups_checked = 0, hashed_bytes = 0; for (const auto& item : p->page_groups[q->depth]) { const auto& group = item.second; const auto& r = group.texels; Rect rect{q->page_x+r.x/ppw,q->page_y+r.y,r.w/ppw,r.h}; - if (!contains(rect,wanted)) continue; + if (!intersects(rect,wanted)) continue; const bool cached = std::any_of(p->rect_cache.begin(),p->rect_cache.end(),[&](const RectHash& c) { return c.vram == q->vram && c.rect.x == rect.x && c.rect.y == rect.y && c.rect.w == rect.w && c.rect.h == rect.h; }); if (!cached) hashed_bytes += size_t(rect.w) * rect.h * 2; if (++groups_checked > 4096 || hashed_bytes > 4u*1024u*1024u) { over_budget = true; break; } const auto found = group.hashes.find(cached_hash(*p,*q,rect)); - if (found != group.hashes.end()) for (const size_t index : found->second) consider(index,r.x,r.y); + if (found != group.hashes.end()) for (const size_t index : found->second) { + if (++checked > 4096) { over_budget = true; break; } + consider(index,r.x,r.y,rect); + } + if (over_budget) break; } if (over_budget && !p->geometry_limit_reported) { note(*p,"page candidate geometry/hash budget exceeded"); - std::fprintf(stderr,"[texture-pack] page candidate geometry/hash budget exceeded; using native texture\n"); p->geometry_limit_reported = true; } - int status = over_budget ? HD_TEXTURE_LOOKUP_ERROR : winner == size_t(-1) ? HD_TEXTURE_LOOKUP_NONE : - ambiguous ? HD_TEXTURE_LOOKUP_AMBIGUOUS : HD_TEXTURE_LOOKUP_FOUND; - if (status == HD_TEXTURE_LOOKUP_FOUND) { - const Entry& e = p->entries[winner]; out->key = e.key; out->entry_id = winner + 1; - out->replacement_path = e.path.c_str(); out->origin_u = origin_u; out->origin_v = origin_v; - out->source_width = e.key.width; out->source_height = e.key.height; + const auto same_coverage = [](const DuckTextureMatch& a, const DuckTextureMatch& b) { + const auto& x = a.key; const auto& y = b.key; + return x.kind == y.kind && x.depth == y.depth && x.source_hash == y.source_hash && x.palette_hash == y.palette_hash && + x.palette_min == y.palette_min && x.palette_max == y.palette_max && a.origin_u == b.origin_u && a.origin_v == b.origin_v && + a.source_width == b.source_width && a.source_height == b.source_height && a.clip_u == b.clip_u && a.clip_v == b.clip_v && + a.clip_width == b.clip_width && a.clip_height == b.clip_height; + }; + std::vector parts; + for (const auto& candidate : candidates) { + auto previous = std::find_if(parts.begin(),parts.end(),[&](const DuckTextureMatch& a) { return same_coverage(a,candidate); }); + if (previous == parts.end()) parts.push_back(candidate); + else if (candidate.key.semitransparent == (st != 0) && previous->key.semitransparent != (st != 0)) *previous = candidate; } + bool ambiguous = false; + for (const auto& part : parts) ambiguous |= p->entries[part.entry_id-1].ambiguous; + int status = over_budget || parts.size() > kMaxParts ? HD_TEXTURE_LOOKUP_ERROR : ambiguous ? HD_TEXTURE_LOOKUP_AMBIGUOUS : + parts.empty() ? HD_TEXTURE_LOOKUP_NONE : HD_TEXTURE_LOOKUP_FOUND; + if (status != HD_TEXTURE_LOOKUP_FOUND) parts.clear(); + std::sort(parts.begin(),parts.end(),[](const DuckTextureMatch& a,const DuckTextureMatch& b) { + if (a.key.kind != b.key.kind) return a.key.kind == DUCK_TEXTURE_PAGE; + const unsigned area_a = unsigned(a.source_width)*a.source_height, area_b = unsigned(b.source_width)*b.source_height; + if (area_a != area_b) return area_a > area_b; + if (a.entry_id != b.entry_id) return a.entry_id < b.entry_id; + if (a.clip_v != b.clip_v) return a.clip_v < b.clip_v; + return a.clip_u < b.clip_u; + }); if (p->match_cache.size() >= 256) p->match_cache.erase(p->match_cache.begin()); - p->match_cache.push_back({*q,st != 0,status,*out}); + p->match_cache.push_back({*q,st != 0,status,parts}); + if (parts.size() > capacity) return HD_TEXTURE_LOOKUP_ERROR; + std::copy(parts.begin(),parts.end(),output); *out_count = parts.size(); return status; } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } } @@ -689,62 +1118,55 @@ int duck_texture_pack_dump_draw(DuckTexturePack* p, const HdTextureDrawQuery* q, if (!p || !q) return HD_TEXTURE_LOOKUP_ERROR; Rect rect{}; if (!query_rect(q,rect)) return HD_TEXTURE_LOOKUP_NONE; try { - Worker& w = p->worker; - { std::lock_guard lock(w.mutex); - if (!w.dump_error.empty()) { error_text(error,capacity,w.dump_error); return HD_TEXTURE_LOOKUP_ERROR; } - if (w.dump_queue.size() >= kMaxDumpQueue) return HD_TEXTURE_LOOKUP_NONE; } - const unsigned ppw = expansion(q->depth); const Upload* upload = nullptr; - for (const auto& u : p->uploads) if (contains(u.rect,rect)) { if (upload) { upload = nullptr; break; } upload = &u; } - DuckTextureKey k{}; k.kind = upload ? DUCK_TEXTURE_UPLOAD : DUCK_TEXTURE_PAGE; - k.depth = q->depth; k.semitransparent = st != 0; - k.source_hash = upload ? upload->hash : cached_hash(*p,*q,rect); - k.source_width_words = upload ? upload->rect.w : 256 / ppw; k.source_height = upload ? upload->rect.h : 256; - k.offset_x = (rect.x - (upload ? upload->rect.x : q->page_x)) * ppw; - k.offset_y = rect.y - (upload ? upload->rect.y : q->page_y); k.width = rect.w * ppw; k.height = rect.h; - k.palette_max = q->depth == 0 ? 15 : q->depth == 1 ? 255 : 0; - bool palette_valid = false; k.palette_hash = palette_hash(*q,0,k.palette_max,palette_valid); - if (!palette_valid) return HD_TEXTURE_LOOKUP_NONE; - const std::string name = stem(k); - if (name.empty()) return HD_TEXTURE_LOOKUP_NONE; - if (p->dumped.count(name)) return HD_TEXTURE_LOOKUP_NONE; - if (p->dumped.size() >= kMaxDumps) { - if (!p->dump_limit_reported) { - note(*p,"dump session limit (8192) reached; reset dump tracking to continue"); - std::fprintf(stderr,"[texture-pack] dump session limit (8192) reached; reset dump tracking to continue\n"); - p->dump_limit_reported = true; + { std::lock_guard lock(p->worker.mutex); + if (!p->worker.dump_error.empty()) { error_text(error,capacity,p->worker.dump_error); return HD_TEXTURE_LOOKUP_ERROR; } } + bool changed = false; + for (const auto& upload : p->uploads) if (intersects(upload.active,rect)) upload.source->used = true; + if (q->depth == 2 && !p->config.dump_c16_textures) return HD_TEXTURE_LOOKUP_NONE; + if (!p->config.dump_texture_pages) { + for (const auto& upload : p->uploads) { + const Rect observed = intersection(upload.active,rect); + if (observed.w && observed.h) changed |= record_usage(*p,*upload.source,observed,*q,st != 0); } - error_text(error,capacity,"texture dump session limit (8192) reached"); - return HD_TEXTURE_LOOKUP_ERROR; + } else { + const Rect page{q->page_x,q->page_y,256/expansion(q->depth),256}; + auto existing = std::find_if(p->pages.begin(),p->pages.end(),[&](const PageSource& value) { + return value.depth == q->depth && value.source->rect.x == page.x && value.source->rect.y == page.y && + (q->depth == 2 || (value.clut_x == q->clut_x && value.clut_y == q->clut_y)); + }); + std::shared_ptr source; + if (existing != p->pages.end()) source = existing->source; + else { + source = capture_source(*p,page,q->vram,DUCK_TEXTURE_PAGE); + if (source) p->pages.push_back({source,q->depth,q->clut_x,q->clut_y}); + } + if (source) changed = record_usage(*p,*source,rect,*q,st != 0); } - auto image = std::make_shared(); image->w = k.width; image->h = k.height; - image->rgba.resize(size_t(image->w) * image->h * 4); - for (unsigned y = 0; y < image->h; ++y) for (unsigned x = 0; x < image->w; ++x) { - uint16_t color = q->vram[(rect.y+y)*1024 + rect.x + x/ppw]; - if (q->depth < 2) { - const unsigned index = q->depth == 0 ? ((color >> ((x%4)*4)) & 15) : ((color >> ((x%2)*8)) & 255); - color = q->clut_x + index < 1024 ? q->vram[q->clut_y*1024+q->clut_x+index] : 0; - } - uint8_t* pixel = image->rgba.data() + (size_t(y)*image->w+x)*4; - for (unsigned c = 0; c < 3; ++c) { - const unsigned value = (color >> (c*5)) & 31; - pixel[c] = static_cast((value * 255u + 15u) / 31u); - } - pixel[3] = color == 0 ? 0 : (st && (color & 0x8000)) ? 128 : 255; - } - { std::lock_guard lock(w.mutex); - if (w.dump_queue.size() >= kMaxDumpQueue) return HD_TEXTURE_LOOKUP_NONE; - if (!w.thread.joinable()) w.thread = std::thread(worker_main,&w); - w.dump_queue.push_back({p->dumps / (name + ".png"), std::move(image)}); - p->dumped.insert(name); - w.wake.notify_one(); } - return HD_TEXTURE_LOOKUP_FOUND; + if (!p->lifecycle_error.empty()) { error_text(error,capacity,p->lifecycle_error); return HD_TEXTURE_LOOKUP_ERROR; } + return changed ? HD_TEXTURE_LOOKUP_FOUND : HD_TEXTURE_LOOKUP_NONE; } catch (const std::exception& e) { error_text(error,capacity,e.what()); return HD_TEXTURE_LOOKUP_ERROR; } catch (...) { error_text(error,capacity,"texture dump allocation failed"); return HD_TEXTURE_LOOKUP_ERROR; } } +int duck_texture_pack_flush_dumps(DuckTexturePack* p, char* error, size_t capacity) { + if (!p) return HD_TEXTURE_LOOKUP_ERROR; + try { + bool success = true; + for (const auto& source : p->sources) success &= finalize_source(*p,*source); + Worker& worker = p->worker; + std::unique_lock lock(worker.mutex); + worker.wake.wait(lock,[&] { return worker.dump_queue.empty() && !worker.dump_busy; }); + if (!worker.dump_error.empty()) { error_text(error,capacity,worker.dump_error); return HD_TEXTURE_LOOKUP_ERROR; } + if (!success) { error_text(error,capacity,p->lifecycle_error); return HD_TEXTURE_LOOKUP_ERROR; } + return HD_TEXTURE_LOOKUP_FOUND; + } catch (const std::exception& e) { error_text(error,capacity,e.what()); return HD_TEXTURE_LOOKUP_ERROR; } + catch (...) { error_text(error,capacity,"texture dump finalization allocation failed"); return HD_TEXTURE_LOOKUP_ERROR; } +} void duck_texture_pack_reset_dump(DuckTexturePack* p) { if (!p) return; - p->dumped.clear(); + duck_texture_pack_flush_dumps(p,nullptr,0); + p->dump_history = std::make_shared(); p->dump_limit_reported = false; + p->lifecycle_error.clear(); std::lock_guard lock(p->worker.mutex); p->worker.dump_error.clear(); } diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index 12af28cf..a1ce10c2 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -1469,6 +1469,8 @@ static const char *TEX_FS = " if(native==0) discard; stp=(native>>15)&1;\n" " } else if(v_hd_mode==2){\n" " if(hd.a<128.0/255.0) discard; stp=0;\n" + " } else if(v_hd_mode==4){\n" + " if(hd.a==0.0) discard; stp=hd.a<1.0 ? 1 : 0;\n" " } else {\n" " stp=hd.a<=242.0/255.0 ? 1 : 0;\n" " if(all(equal(hd,vec4(0.0))) || (stp==0 && all(equal(hd.rgb,vec3(0.0))))) discard;\n" diff --git a/runtime/src/gpu_hd_textures.cpp b/runtime/src/gpu_hd_textures.cpp index 190b70ac..cb58846f 100644 --- a/runtime/src/gpu_hd_textures.cpp +++ b/runtime/src/gpu_hd_textures.cpp @@ -4,16 +4,30 @@ #include "duckstation_texture_pack.h" #include +#include +#include #include #include #include +#include #include #include #include #include +#include namespace { constexpr size_t kVramWords = 1024u * 512u; +constexpr size_t kCompositionBudget = 64u * 1024u * 1024u; +constexpr size_t kCompositionCount = 16; +constexpr uint32_t kCompositionDimension = 8192; +constexpr size_t kMaxParts = 64; +struct Composition { + std::vector signature; + std::vector rgba; + uint32_t width = 0, height = 0; + uint64_t cache_key = 0, last_use = 0; +}; struct Session { std::string root; HdTexturePack* beetle = nullptr; @@ -21,17 +35,30 @@ struct Session { bool replacements = false; bool dump = false; bool dump_failed = false; + bool copy_pending = false; + std::vector> compositions; + size_t composition_bytes = 0; + uint64_t composition_clock = 0; + std::string dump_error; + std::deque diagnostics; GpuHdTextureDiag diag{}; ~Session() { hd_texture_pack_destroy(beetle); duck_texture_pack_destroy(duck); } }; struct Lease { HdTexturePixels beetle{}; DuckTexturePixels duck{}; + std::shared_ptr composition; ~Lease() { hd_texture_pixels_release(&beetle); duck_texture_pixels_release(&duck); } }; std::unique_ptr session; const uint16_t* native_vram = nullptr; uint64_t generation = 1; +uint64_t composition_serial = 0; + +struct DecodedParts { + std::array pixels{}; + ~DecodedParts() { for (auto& pixel : pixels) duck_texture_pixels_release(&pixel); } +}; void fail(char* error, size_t capacity, const char* message) { if (error && capacity) std::snprintf(error, capacity, "%s", message); @@ -75,16 +102,136 @@ bool make_query(uint16_t texpage, uint16_t cx, uint16_t cy, const int limits[4], query.vram = native_vram; query.vram_word_count = kVramWords; return true; } +uint8_t replacement_class(const uint8_t* pixel, bool semi) { + if (!semi) return pixel[3] >= 128 ? 255 : 0; + if (!(pixel[0] | pixel[1] | pixel[2] | pixel[3])) return 0; + if (pixel[3] <= 242) return 128; + return (pixel[0] | pixel[1] | pixel[2]) ? 255 : 0; +} +void replacement_pixel(const DuckTexturePixels& image, bool semi, bool linear, + double x, double y, uint8_t* output) { + const int nx = std::clamp(static_cast(std::floor(x)), 0, static_cast(image.width) - 1); + const int ny = std::clamp(static_cast(std::floor(y)), 0, static_cast(image.height) - 1); + const uint8_t* nearest = image.rgba + static_cast(ny) * image.stride + nx * 4; + const uint8_t alpha = replacement_class(nearest, semi); + output[3] = alpha; + if (!alpha) { output[0] = output[1] = output[2] = 0; return; } + if (!linear) { std::memcpy(output, nearest, 3); return; } + /* Scaling filters colors within the nearest PSX class. Alpha class is + * discrete: mixing cutout/STP/opaque would alter the primitive's blend. */ + x -= 0.5; y -= 0.5; + const int bx = static_cast(std::floor(x)), by = static_cast(std::floor(y)); + const double fx = x - bx, fy = y - by; + double sum[3]{}, weight = 0; + for (int dy = 0; dy < 2; ++dy) for (int dx = 0; dx < 2; ++dx) { + const int px = std::clamp(bx + dx, 0, static_cast(image.width) - 1); + const int py = std::clamp(by + dy, 0, static_cast(image.height) - 1); + const uint8_t* pixel = image.rgba + static_cast(py) * image.stride + px * 4; + if (replacement_class(pixel, semi) != alpha) continue; + const double w = (dx ? fx : 1 - fx) * (dy ? fy : 1 - fy); + for (int channel = 0; channel < 3; ++channel) sum[channel] += pixel[channel] * w; + weight += w; + } + for (int channel = 0; channel < 3; ++channel) + output[channel] = weight > 0 ? static_cast(std::clamp(sum[channel] / weight + 0.5, 0.0, 255.0)) : nearest[channel]; +} +void native_pixel(const HdTextureDrawQuery& query, const std::array& palette, + int u, int v, uint8_t* output) { + const unsigned per_word = 4u >> query.depth; + uint16_t word = query.vram[(query.page_y + v) * 1024u + query.page_x + u / per_word]; + if (query.depth == HD_TEXTURE_DEPTH_4BPP) word = palette[(word >> ((u & 3) * 4)) & 15]; + else if (query.depth == HD_TEXTURE_DEPTH_8BPP) word = palette[(word >> ((u & 1) * 8)) & 255]; + output[0] = static_cast(((word & 31u) * 255u + 15u) / 31u); + output[1] = static_cast((((word >> 5) & 31u) * 255u + 15u) / 31u); + output[2] = static_cast((((word >> 10) & 31u) * 255u + 15u) / 31u); + output[3] = !word ? 0 : (word & 0x8000u) ? 128 : 255; +} +std::shared_ptr compose(const HdTextureDrawQuery& query, + const std::array& matches, const DecodedParts& decoded, size_t count) { + const unsigned per_word = 4u >> query.depth; + const unsigned first_word = query.u_first / per_word, last_word = query.u_last / per_word; + if (query.page_x + last_word >= 1024 || query.page_y + query.v_last >= 512) return {}; + const uint32_t native_width = query.u_last - query.u_first + 1; + const uint32_t native_height = query.v_last - query.v_first + 1; + std::array palette{}; + const size_t palette_size = query.depth == HD_TEXTURE_DEPTH_4BPP ? 16 : query.depth == HD_TEXTURE_DEPTH_8BPP ? 256 : 0; + for (size_t i = 0; i < palette_size; ++i) + palette[i] = query.vram[(query.clut_y & 511u) * 1024u + ((query.clut_x + i) & 1023u)]; + std::vector signature{ + static_cast(query.page_x) | (static_cast(query.page_y) << 16) | (static_cast(query.depth) << 32), + static_cast(query.u_first) | (static_cast(query.v_first) << 8) | + (static_cast(query.u_last) << 16) | (static_cast(query.v_last) << 24), + duck_texture_hash_rect(query.vram, query.vram_word_count, query.page_x + first_word, + query.page_y + query.v_first, last_word - first_word + 1, native_height), + duck_texture_hash_words_le(palette.data(), palette_size)}; + double scale_x = 1, scale_y = 1; + const bool linear = duck_texture_pack_linear_filter(session->duck) != 0; + signature.push_back(linear); + for (size_t i = 0; i < count; ++i) { + const auto& pixel = decoded.pixels[i]; + if (!pixel.rgba) continue; + const auto& match = matches[i]; + scale_x = std::max(scale_x, static_cast(pixel.width) / match.source_width); + scale_y = std::max(scale_y, static_cast(pixel.height) / match.source_height); + signature.insert(signature.end(), {match.entry_id, static_cast(match.origin_u), + static_cast(match.origin_v), match.source_width, match.source_height, + static_cast(match.clip_u), static_cast(match.clip_v), + match.clip_width, match.clip_height, pixel.width, pixel.height, match.key.semitransparent}); + } + for (auto& cached : session->compositions) + if (cached->signature == signature) { cached->last_use = ++session->composition_clock; return cached; } + const double wanted_width = std::ceil(native_width * scale_x); + const double wanted_height = std::ceil(native_height * scale_y); + if (wanted_width > kCompositionDimension || wanted_height > kCompositionDimension) return {}; + const uint32_t width = static_cast(wanted_width), height = static_cast(wanted_height); + const size_t bytes = static_cast(width) * height * 4u; + if (bytes > kCompositionBudget) return {}; + while (!session->compositions.empty() && (session->compositions.size() >= kCompositionCount || + session->composition_bytes + bytes > kCompositionBudget)) { + auto oldest = std::min_element(session->compositions.begin(), session->compositions.end(), + [](const auto& a, const auto& b) { return a->last_use < b->last_use; }); + session->composition_bytes -= (*oldest)->rgba.size(); session->compositions.erase(oldest); + } + auto result = std::make_shared(); + result->signature = std::move(signature); result->width = width; result->height = height; + result->rgba.resize(bytes); + for (uint32_t y = 0; y < height; ++y) for (uint32_t x = 0; x < width; ++x) + native_pixel(query, palette, query.u_first + (static_cast(x) * 2 + 1) * native_width / (width * 2u), + query.v_first + (static_cast(y) * 2 + 1) * native_height / (height * 2u), + result->rgba.data() + (static_cast(y) * width + x) * 4); + for (size_t i = 0; i < count; ++i) { + const auto& image = decoded.pixels[i]; const auto& match = matches[i]; + if (!image.rgba) continue; + const int x0 = std::clamp(static_cast(std::floor((match.clip_u - query.u_first) * static_cast(width) / native_width)), 0, static_cast(width)); + const int y0 = std::clamp(static_cast(std::floor((match.clip_v - query.v_first) * static_cast(height) / native_height)), 0, static_cast(height)); + const int x1 = std::clamp(static_cast(std::ceil((match.clip_u + match.clip_width - query.u_first) * static_cast(width) / native_width)), 0, static_cast(width)); + const int y1 = std::clamp(static_cast(std::ceil((match.clip_v + match.clip_height - query.v_first) * static_cast(height) / native_height)), 0, static_cast(height)); + for (int y = y0; y < y1; ++y) for (int x = x0; x < x1; ++x) { + const double u = query.u_first + (x + 0.5) * native_width / width; + const double v = query.v_first + (y + 0.5) * native_height / height; + if (u < match.clip_u || u >= match.clip_u + match.clip_width || + v < match.clip_v || v >= match.clip_v + match.clip_height) continue; + replacement_pixel(image, match.key.semitransparent != 0, linear, + (u - match.origin_u) * image.width / match.source_width, + (v - match.origin_v) * image.height / match.source_height, + result->rgba.data() + (static_cast(y) * width + x) * 4); + } + } + result->cache_key = (UINT64_C(1) << 63) | ++composition_serial; + result->last_use = ++session->composition_clock; + session->compositions.push_back(result); session->composition_bytes += bytes; + return result; +} void dump_query(const HdTextureDrawQuery& query, int semi) { if (!session->dump || session->dump_failed) return; char error[256]{}; const int status = duck_texture_pack_dump_draw(session->duck, &query, semi, error, sizeof(error)); if (status == HD_TEXTURE_LOOKUP_FOUND) ++session->diag.dumped_textures; if (status < 0) { - std::fprintf(stderr, "psxrecomp: texture dump stopped: %s\n", error); /* Keep the configured raster authority until a safe session boundary. * Tearing it down inside a pre-draw query would lose command order. */ session->dump_failed = true; + session->dump_error = error; } } } @@ -113,7 +260,14 @@ extern "C" int gpu_hd_textures_configure(const char* root, int replacements, std::filesystem::create_directories(directory / "dumps"); } if (!duck_texture_pack_create(root, &next->duck, error, capacity)) return 0; - if (session && session->root == next->root) { + if (session) { + char dump_error[256]{}; + if (duck_texture_pack_flush_dumps(session->duck, dump_error, sizeof(dump_error)) < 0) { + session->dump_error = dump_error; + fail(error, capacity, dump_error); return 0; + } + } + if (session && (session->replacements || session->dump) && session->root == next->root) { if (!duck_texture_pack_copy_tracking(next->duck, session->duck)) { fail(error, capacity, "Could not preserve texture-upload tracking during reload."); return 0; } @@ -163,6 +317,13 @@ extern "C" int gpu_hd_textures_reload(char* error, size_t capacity) { } extern "C" void gpu_hd_textures_set_dump_enabled(int enabled) { if (!session || session->dump == (enabled != 0)) return; + if (enabled && !session->replacements) gpu_hd_textures_reset_tracking(); + if (!enabled) { + char error[256]{}; + if (duck_texture_pack_flush_dumps(session->duck, error, sizeof(error)) < 0) { + session->dump_error = error; session->dump_failed = true; return; + } + } gl_renderer_set_hd_texture_mode(session->replacements || enabled); session->dump = enabled != 0; session->dump_failed = false; @@ -173,21 +334,55 @@ extern "C" int gpu_hd_textures_active(void) { return session && (session->replac extern "C" void gpu_hd_textures_get_diag(GpuHdTextureDiag* out) { if (!out) return; *out = {}; - if (!session) { out->root = ""; return; } + if (!session) { out->root = ""; out->diagnostic = ""; return; } *out = session->diag; out->root = session->root.c_str(); out->active = gpu_hd_textures_active(); out->replacements = session->replacements; out->dump = session->dump && !session->dump_failed; out->format = session->beetle ? 1 : 2; - if (session->beetle) { HdTexturePackInfo info{}; hd_texture_pack_get_info(session->beetle, &info); out->replacement_count = info.unique_key_count; } - else { DuckTexturePackInfo info{}; duck_texture_pack_get_info(session->duck, &info); out->replacement_count = info.replacement_count; } + DuckTexturePackInfo info{}; duck_texture_pack_get_info(session->duck, &info); + out->dumped_textures = info.queued_dump_count; out->pending_dump_sources = info.pending_dump_sources; + if (session->beetle) { HdTexturePackInfo beetle_info{}; hd_texture_pack_get_info(session->beetle, &beetle_info); out->replacement_count = beetle_info.unique_key_count; } + else out->replacement_count = info.replacement_count; + std::string diagnostic = info.diagnostic ? info.diagnostic : ""; + if (!session->dump_error.empty()) { + if (!diagnostic.empty()) diagnostic += "; "; + diagnostic += session->dump_error; + } + if (session->diagnostics.empty() || session->diagnostics.back() != diagnostic) + session->diagnostics.push_back(std::move(diagnostic)); + out->diagnostic = session->diagnostics.back().c_str(); } extern "C" void gpu_hd_textures_note_applied(void) { if (session) ++session->diag.applied_draws; } extern "C" void gpu_hd_textures_set_vram(const uint16_t* vram) { native_vram = vram; gpu_hd_textures_reset_tracking(); } extern "C" void gpu_hd_textures_reset_tracking(void) { if (!session) return; + gl_renderer_clear_hd_texture_cache(); + session->compositions.clear(); session->composition_bytes = 0; + session->copy_pending = false; hd_texture_pack_reset_tracking(session->beetle); duck_texture_pack_reset_tracking(session->duck); } +extern "C" void gpu_hd_textures_begin_upload(int x, int y, int width, int height) { + /* The GP0 header arrives before native payload writes. Complete uploads + * establish their fresh provenance only in track_upload(), after writing. */ + gpu_hd_textures_invalidate(x, y, width, height); +} +extern "C" void gpu_hd_textures_begin_copy(int sx, int sy, int dx, int dy, int width, int height) { + if (!session || !native_vram) return; + session->copy_pending = false; + if (width < 1 || width > 1024 || height < 1 || height > 512) { gpu_hd_textures_reset_tracking(); return; } + hd_texture_pack_invalidate(session->beetle, dx & 1023, dy & 511, width, height); + session->copy_pending = duck_texture_pack_begin_copy(session->duck, sx & 1023, sy & 511, + dx & 1023, dy & 511, width, height, native_vram, kVramWords) > 0; + if (!session->copy_pending) + duck_texture_pack_invalidate(session->duck, dx & 1023, dy & 511, width, height); +} +extern "C" void gpu_hd_textures_end_copy(void) { + if (!session || !session->copy_pending) return; + session->copy_pending = false; + if (duck_texture_pack_end_copy(session->duck, native_vram, kVramWords) < 0) + gpu_hd_textures_reset_tracking(); +} extern "C" void gpu_hd_textures_invalidate(int x, int y, int width, int height) { if (!session || width < 1 || height < 1) return; if (width > 1024 || height > 512) { gpu_hd_textures_reset_tracking(); return; } @@ -242,17 +437,38 @@ extern "C" int gpu_hd_textures_acquire_draw(uint16_t tp, uint16_t cx, uint16_t c image->cache_key = (static_cast(match.entry.texture_hash) << 32) | match.entry.palette_hash; image->alpha_mode = 1; } else { - DuckTextureMatch match{}; - if (duck_texture_pack_match_draw(session->duck, &query, semi, &match) != HD_TEXTURE_LOOKUP_FOUND) return 0; - ++session->diag.matched_draws; - duck_texture_pack_request_decode(session->duck, match.entry_id); - if (duck_texture_pack_acquire_decoded(session->duck, match.entry_id, &lease->duck) != HD_TEXTURE_LOOKUP_FOUND) return 0; - image->rgba = lease->duck.rgba; image->width = lease->duck.width; - image->height = lease->duck.height; image->stride = lease->duck.stride; - image->source_width = match.source_width; image->source_height = match.source_height; - image->origin_u = match.origin_u; image->origin_v = match.origin_v; - image->cache_key = match.entry_id; - image->alpha_mode = match.key.semitransparent ? 3 : 2; + try { + std::array matches{}; size_t count = 0; + if (duck_texture_pack_match_parts(session->duck, &query, semi, matches.data(), matches.size(), &count) != HD_TEXTURE_LOOKUP_FOUND || !count) return 0; + ++session->diag.matched_draws; + DecodedParts decoded; size_t ready = 0; + for (size_t i = 0; i < count; ++i) { + duck_texture_pack_request_decode(session->duck, matches[i].entry_id); + if (duck_texture_pack_acquire_decoded(session->duck, matches[i].entry_id, &decoded.pixels[i]) == HD_TEXTURE_LOOKUP_FOUND) ++ready; + } + if (!ready) return 0; + const auto& match = matches[0]; + if (count == 1 && match.clip_u <= query.u_first && match.clip_v <= query.v_first && + match.clip_u + match.clip_width > query.u_last && match.clip_v + match.clip_height > query.v_last) { + lease->duck = decoded.pixels[0]; decoded.pixels[0] = {}; + image->rgba = lease->duck.rgba; image->width = lease->duck.width; + image->height = lease->duck.height; image->stride = lease->duck.stride; + image->source_width = match.source_width; image->source_height = match.source_height; + image->origin_u = match.origin_u; image->origin_v = match.origin_v; + image->cache_key = match.entry_id; + image->alpha_mode = match.key.semitransparent ? 3 : 2; + } else { + lease->composition = compose(query, matches, decoded, count); + if (!lease->composition) return 0; + const auto& composed = *lease->composition; + image->rgba = composed.rgba.data(); image->width = composed.width; + image->height = composed.height; image->stride = composed.width * 4u; + image->source_width = query.u_last - query.u_first + 1; + image->source_height = query.v_last - query.v_first + 1; + image->origin_u = query.u_first; image->origin_v = query.v_first; + image->cache_key = composed.cache_key; image->alpha_mode = 4; + } + } catch (const std::exception&) { return 0; } } image->generation = generation; image->lease = lease.release(); ++session->diag.ready_draws; diff --git a/runtime/src/gpu_render.c b/runtime/src/gpu_render.c index 7a04d105..1cbf16fd 100644 --- a/runtime/src/gpu_render.c +++ b/runtime/src/gpu_render.c @@ -158,7 +158,14 @@ void gr_set_perspective_triangle(int enabled, float q0, float q1, float q2) { g_b->set_perspective_triangle(enabled, q0, q1, q2); } void gr_fill_rect(int x, int y, int w, int h, uint16_t c) { g_b->fill_rect(x, y, w, h, c); gpu_hd_textures_invalidate(x, y, w, h); } -void gr_copy_rect(int sx, int sy, int dx, int dy, int w, int h) { g_b->copy_rect(sx, sy, dx, dy, w, h); gpu_hd_textures_invalidate(dx, dy, w, h); } +void gr_copy_rect(int sx, int sy, int dx, int dy, int w, int h) { + if (!gpu_hd_textures_active()) { g_b->copy_rect(sx, sy, dx, dy, w, h); return; } + if (g_effective == GR_BACKEND_VULKAN) g_b->vram_read(0,0); + gpu_hd_textures_begin_copy(sx,sy,dx,dy,w,h); + g_b->copy_rect(sx, sy, dx, dy, w, h); + if (g_effective == GR_BACKEND_VULKAN) g_b->vram_read(0,0); + gpu_hd_textures_end_copy(); +} void gr_draw_flat_triangle(int x0, int y0, int x1, int y1, int x2, int y2, uint16_t c) { g_b->draw_flat_triangle(x0, y0, x1, y1, x2, y2, c); hd_invalidate_triangle(x0, y0, x1, y1, x2, y2); @@ -214,7 +221,7 @@ int gr_render_display(uint32_t *o, int p, int dx, int dy, int dw, int dh) { int gr_render_display_hires(uint32_t *o, int p, int dx, int dy, int dw, int dh) { return g_b->render_display_hires(o, p, dx, dy, dw, dh); } -void gr_vram_upload_begin(int x, int y, int w, int h) { gpu_hd_textures_invalidate(x,y,w,h); } +void gr_vram_upload_begin(int x, int y, int w, int h) { gpu_hd_textures_begin_upload(x,y,w,h); } void gr_vram_write(int x, int y, uint16_t pixel) { g_b->vram_write(x, y, pixel); gpu_hd_textures_invalidate(x,y,1,1); } uint16_t gr_vram_read(int x, int y) { return g_b->vram_read(x, y); } void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *d) { g_b->vram_transfer_in(x, y, w, h, d); gpu_hd_textures_track_upload(x,y,w,h,d); } diff --git a/runtime/src/psx_window_icon.cpp b/runtime/src/psx_window_icon.cpp index 513cf0b9..3e2e5acf 100644 --- a/runtime/src/psx_window_icon.cpp +++ b/runtime/src/psx_window_icon.cpp @@ -17,6 +17,7 @@ #define STB_IMAGE_IMPLEMENTATION #define STBI_ONLY_PNG +#define STBI_ONLY_JPEG #define STBI_NO_STDIO #include "../third_party/stb_image.h" diff --git a/runtime/src/texture_image_decode.cpp b/runtime/src/texture_image_decode.cpp new file mode 100644 index 00000000..2ccadb01 --- /dev/null +++ b/runtime/src/texture_image_decode.cpp @@ -0,0 +1,107 @@ +#include "texture_image_decode.h" + +#include "../third_party/stb_image.h" +#include + +#include +#include +#include +#include +#include + +namespace { + +bool fail(std::string& error, const char* message) { + error = message; + return false; +} + +bool probe_impl(const uint8_t* encoded, size_t encoded_size, + size_t decoded_budget, TextureImageProbe& probe, + std::string& error) { + if (!encoded || !encoded_size) return fail(error, "texture image is empty"); + if (encoded_size > TEXTURE_IMAGE_MAX_ENCODED_BYTES) + return fail(error, "texture image exceeds 64 MiB encoded limit"); + int width = 0, height = 0, channels = 0; + static const uint8_t png_magic[] = {137, 80, 78, 71, 13, 10, 26, 10}; + if (encoded_size >= sizeof(png_magic) && + std::memcmp(encoded, png_magic, sizeof(png_magic)) == 0) { + probe.format = TextureImageFormat::Png; + if (!stbi_info_from_memory(encoded, static_cast(encoded_size), + &width, &height, &channels)) + return fail(error, "malformed PNG texture image header"); + } else if (encoded_size >= 2 && encoded[0] == 0xff && encoded[1] == 0xd8) { + probe.format = TextureImageFormat::Jpeg; + if (!stbi_info_from_memory(encoded, static_cast(encoded_size), + &width, &height, &channels)) + return fail(error, "malformed JPEG texture image header"); + } else if (encoded_size >= 12 && std::memcmp(encoded, "RIFF", 4) == 0 && + std::memcmp(encoded + 8, "WEBP", 4) == 0) { + probe.format = TextureImageFormat::WebP; + WebPBitstreamFeatures features{}; + if (WebPGetFeatures(encoded, encoded_size, &features) != VP8_STATUS_OK) + return fail(error, "malformed WebP texture image header"); + if (features.has_animation) + return fail(error, "animated WebP texture images are unsupported"); + width = features.width; + height = features.height; + } else { + return fail(error, "texture image must be PNG, JPEG, or WebP"); + } + if (width <= 0 || height <= 0 || width > static_cast(TEXTURE_IMAGE_MAX_DIMENSION) || + height > static_cast(TEXTURE_IMAGE_MAX_DIMENSION)) + return fail(error, "texture image dimensions exceed 8192 pixels per side"); + const uint64_t rgba_bytes = static_cast(width) * height * 4u; + if (rgba_bytes > decoded_budget || rgba_bytes > TEXTURE_IMAGE_DEFAULT_DECODED_BUDGET) + return fail(error, "texture image exceeds decoded RGBA budget (maximum 64 MiB)"); + probe.width = static_cast(width); + probe.height = static_cast(height); + return true; +} + +} // namespace + +bool texture_image_probe(const uint8_t* encoded, size_t encoded_size, + size_t decoded_budget, TextureImageProbe& probe, + std::string& error) { + probe = {}; + error.clear(); + TextureImageProbe checked; + if (!probe_impl(encoded, encoded_size, decoded_budget, checked, error)) return false; + probe = checked; + return true; +} + +bool texture_image_decode(const uint8_t* encoded, size_t encoded_size, + size_t decoded_budget, DecodedTextureImage& image, + std::string& error) { + image = {}; + error.clear(); + TextureImageProbe probe; + if (!probe_impl(encoded, encoded_size, decoded_budget, probe, error)) return false; + const size_t rgba_bytes = static_cast(probe.width) * probe.height * 4u; + try { + DecodedTextureImage decoded; + decoded.width = probe.width; + decoded.height = probe.height; + if (probe.format == TextureImageFormat::WebP) { + decoded.rgba.resize(rgba_bytes); + if (!WebPDecodeRGBAInto(encoded, encoded_size, decoded.rgba.data(), + decoded.rgba.size(), static_cast(probe.width * 4u))) + return fail(error, "malformed WebP texture image payload"); + } else { + int width = 0, height = 0, channels = 0; + std::unique_ptr pixels( + stbi_load_from_memory(encoded, static_cast(encoded_size), + &width, &height, &channels, 4), &stbi_image_free); + if (!pixels) return fail(error, "malformed PNG/JPEG texture image payload"); + if (width != static_cast(probe.width) || height != static_cast(probe.height)) + return fail(error, "texture dimensions changed during decode"); + decoded.rgba.assign(pixels.get(), pixels.get() + rgba_bytes); + } + image = std::move(decoded); + return true; + } catch (const std::bad_alloc&) { + return fail(error, "texture image allocation failed"); + } +} diff --git a/runtime/tests/fixtures/texture_images/README.txt b/runtime/tests/fixtures/texture_images/README.txt new file mode 100644 index 00000000..930b1192 --- /dev/null +++ b/runtime/tests/fixtures/texture_images/README.txt @@ -0,0 +1,4 @@ +Synthetic fixtures generated with Pillow 12.3.0, no game artwork. +PNG and lossless WebP: four literal RGBA pixels (255,0,17,0), (3,127,241,1), (63,19,211,128), (23,47,71,254). +JPEG: uniform RGB (30,80,120), quality100, no subsampling. Animated WebP adds a solid RGBA (1,2,3,255) frame to exercise explicit rejection. +Progressive JPEG and lossy WebP: same uniform RGB, quality100. diff --git a/runtime/tests/fixtures/texture_images/animated.webp b/runtime/tests/fixtures/texture_images/animated.webp new file mode 100644 index 0000000000000000000000000000000000000000..4063821b95e0c633f1c065dd064735f5597c1cfc GIT binary patch literal 172 zcmWIYbaPw6z`zjh>J$(bU=hIuWD5Z?GY~T{IQn_|vVmC;p0Aq=oCi^n!oa`;)aRoE z6xRm|3CJr5D6~|VD$a9gShVBmf3{*4rk0GOd@>v?9LFO08+NVVyYPwNUw%hFUpJ5* g2!NZ#05VGiCO1eh5G8JGka znFSgDA7PLOx|0=RHUkteGBLBTvaxe;a&ZF{Y!zT&Vq|7!Vqs=wWnlrTt_8|7un4jW zDH=Mm2?r*!D;0_uHBMZ}q3pErplHy=4=Tnkx|JhscGpMnOVgprDf$6l~v6xt!?ccon4bAPnkMx`iz;g7A;<~ zblLJ1D_3pWyk+aQ?K^hvI&}ER(PPI?oIG{u@|COCuHU$M>*1rvPo6$|{^I4UkDoq& z`TFhqkDtFl{$gZch6fqMV`%;Y1_C1!3kx#~JIG&*Oyxk#EXcyDXviky7|5PjD6C}E z$RXl1apA^;oXW;QA4HRiE^>*fm^@Vd2=W@(XT*7|i7cPNJ%;etEe0NDMquPI3o_U< U{Lx?kj`>H82EziT2K)av0rO(NKmY&$ literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/texture_images/progressive.jpg b/runtime/tests/fixtures/texture_images/progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de73b46c08ca49d22886fcfbdf6483d6b3fca12d GIT binary patch literal 524 zcmbu4O$x$5429pDpzT611G=uBWEMg}@8!kBL-@1t1kxMCj72G8+eI^*N%&sgi(Tvi zJ=FUeBtU<#J9geptGr~|23Ok&v4lc}5g}3{?Tqz1D!NtXu?xC0{}*<{LWrQEe%$e? zP2uE9f5DA|rOP=BRR-dJK*+=)Abf#di+O4-dI6?w^~nwKD-_F-OgfgS1qLh~xV`{u CJ35a5 literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/texture_images/rgba.png b/runtime/tests/fixtures/texture_images/rgba.png new file mode 100644 index 0000000000000000000000000000000000000000..1e6d76491e3603aacbab1f5c5be81c07ff761496 GIT binary patch literal 82 zcmeAS@N?(olHy`uVBq!ia0vp^EI`c2!3HFM+VGzRQj(r7jv*Cul7F06IB_86|HO-N gKZ=@n?E7ucpcBu!)pGr`)j&lIp00i_>zopr07>&1I{*Lx literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/texture_images/rgba.webp b/runtime/tests/fixtures/texture_images/rgba.webp new file mode 100644 index 0000000000000000000000000000000000000000..d0f24538b5a61391c3342c7663e2f48189d7b937 GIT binary patch literal 68 zcmWIYbaS&|U|7j+!Ra XjW-~H#x5=X-Cwm)+HV$T2oII9&@ literal 0 HcmV?d00001 diff --git a/runtime/tests/run_gl_texture_filter.py b/runtime/tests/run_gl_texture_filter.py index a58ff7ce..fd407225 100644 --- a/runtime/tests/run_gl_texture_filter.py +++ b/runtime/tests/run_gl_texture_filter.py @@ -16,6 +16,8 @@ def main(): parser.add_argument('--fixture',type=Path) parser.add_argument('--hd-pack', action='store_true') parser.add_argument('--cxx') + parser.add_argument('--webp-include') + parser.add_argument('--webp-decoder') parser.add_argument('--argument', action='append', default=[]) args=parser.parse_args() fw=Path(__file__).resolve().parents[2] @@ -44,11 +46,15 @@ def main(): obj=out/(name+'.o'); run([args.cc,*flags,'-c',source,'-o',obj]); objects.append(obj) if args.hd_pack: if not args.cxx: parser.error('--hd-pack requires --cxx') + if not args.webp_include or not args.webp_decoder: + parser.error('--hd-pack requires --webp-include and --webp-decoder') cppflags=[arg if arg!='-std=gnu11' else '-std=c++17' for arg in flags] - for name in ('gpu_hd_textures','hd_texture_pack','duckstation_texture_pack'): + cppflags+=['-I',args.webp_include] + for name in ('gpu_hd_textures','hd_texture_pack','duckstation_texture_pack','texture_image_decode'): obj=out/(name+'.o') run([args.cxx,*cppflags,'-c',fw/'runtime/src'/(name+'.cpp'),'-o',obj]) objects.append(obj) + objects.append(args.webp_decoder) exe=out/('filter.exe' if platform.system()=='Windows' else 'filter') libraries=['-lm'] if platform.system()=='Windows': diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp index b5126650..b8d7ebe7 100644 --- a/runtime/tests/test_duckstation_texture_pack.cpp +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -4,12 +4,14 @@ #include "../src/png_write.h" #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -55,6 +57,33 @@ void png(const fs::path& path, unsigned w = 4, unsigned h = 4) { check(file != nullptr,"open original test PNG"); if (file) { check(png_write_rgba(file,pixels.data(),w,h) == 1,"encode original test PNG"); std::fclose(file); } } +void config(const fs::path& root, const char* contents) { + fs::create_directories(root); + std::ofstream file(root/"config.yaml"); file << contents; +} +std::vector dumps(const fs::path& root) { + std::vector result; + if (fs::exists(root/"dumps")) for (const auto& file : fs::directory_iterator(root/"dumps")) + if (file.path().extension() == ".png") result.push_back(file.path()); + std::sort(result.begin(),result.end()); return result; +} +std::vector read_png(const fs::path& file, unsigned w, unsigned h) { + std::ifstream input(file,std::ios::binary); + std::vector encoded((std::istreambuf_iterator(input)),{}), result; + int width = 0, height = 0, channels = 0; + auto* pixels = encoded.empty() ? nullptr : stbi_load_from_memory(encoded.data(),static_cast(encoded.size()),&width,&height,&channels,4); + check(pixels && width == int(w) && height == int(h),"dump PNG has expected dimensions"); + if (pixels) { result.assign(pixels,pixels+size_t(width)*height*4); stbi_image_free(pixels); } + return result; +} +void flush(DuckTexturePack* pack) { + char error[512]{}; + check(duck_texture_pack_flush_dumps(pack,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,error); +} +std::string filename(const DuckTextureKey& key) { + char name[192]{}; check(duck_texture_format_name(&key,name,sizeof(name)),"format fixture name"); + return std::string(name)+".png"; +} DuckTexturePixels decoded(DuckTexturePack* pack, uint64_t id) { DuckTexturePixels pixels{}; check(duck_texture_pack_request_decode(pack,id) >= 0,"queue PNG decode"); @@ -112,6 +141,9 @@ void names() { check(duck_texture_parse_name("texpage-STC16-0123456789ABCDEF-256x256-8-16-32x64.PNG",&key) == 1 && key.kind == DUCK_TEXTURE_PAGE && key.semitransparent && key.depth == 2 && !key.palette_hash, "direct ST filename omits palette fields"); + for (const char* extension : {"jpg","jpeg","webp"}) + check(duck_texture_parse_name((std::string("texupload-C16-0123456789ABCDEF-4x2-0-0-4x2.")+extension).c_str(),&key), + "JPEG and WebP canonical payload extensions accepted"); const char* invalid[] = { "texpage-P4-12345678-0123456789ABCDEF-64x256-0-0-4x1-P0-15.png", "texpage-P4-0123456789ABCDEF-0123456789ABCDEF-64x256-0-0-3x1-P0-15.png", @@ -220,13 +252,15 @@ void st_alias_duplicates(const fs::path& root) { png(root/"alias"/"replacements"/"art"/"sample.png"); fs::create_directories(root/"alias"); std::ofstream config(root/"alias"/"config.yaml"); - config << "MaxVRAMWriteCoalesceWidth: 1\nAliases:\n " << std::string(ordinary).substr(0,std::strlen(ordinary)-4) + config << "MaxVRAMWriteCoalesceWidth: 1\nBogusOption: true\nAliases:\n " << std::string(ordinary).substr(0,std::strlen(ordinary)-4) << ": |\n art/sample.png\n"; config.close(); pack = create(root/"alias"); check(duck_texture_pack_match_draw(pack.get(),&q,1,&match) == HD_TEXTURE_LOOKUP_FOUND && !match.key.semitransparent, "bounded literal alias and opposite-convention fallback"); DuckTexturePackInfo info{}; duck_texture_pack_get_info(pack.get(),&info); - check(info.ignored_count && std::strstr(info.diagnostic,"MaxVRAMWriteCoalesceWidth"),"unsupported config option explicitly diagnosed"); + DuckTextureConfig settings{}; duck_texture_pack_get_config(pack.get(),&settings); + check(settings.max_vram_write_coalesce_width == 1,"exact root-level coalesce option recognized"); + check(info.ignored_count && std::strstr(info.diagnostic,"BogusOption"),"unknown config option explicitly diagnosed"); png(root/"duplicate"/"replacements"/"a"/ordinary); png(root/"duplicate"/"replacements"/"b"/ordinary); pack = create(root/"duplicate"); check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_AMBIGUOUS,"recursive duplicate keys fall back deterministically"); @@ -263,6 +297,7 @@ void failed_payloads(const fs::path& root) { } void dumping(const fs::path& root) { + config(root,"DumpTexturePages: true\nDumpTextureWidthThreshold: 1\nDumpTextureHeightThreshold: 1\nReducePaletteRange: false\n"); auto pack = create(root); std::vector vram(1024*512); vram[0] = 0x3210; vram[400*1024+1] = 0x001f; vram[400*1024+2] = 0x8000; vram[400*1024+3] = 0xffff; auto q = query(vram,0,0,0,0,3,0,0,0,400); char error[512]{}; @@ -275,13 +310,14 @@ void dumping(const fs::path& root) { std::ifstream encoded_file(file,std::ios::binary); std::vector encoded((std::istreambuf_iterator(encoded_file)),{}); auto* image = encoded.empty() ? nullptr : stbi_load_from_memory(encoded.data(),static_cast(encoded.size()),&width,&height,&channels,4); - constexpr uint8_t expected[] = {0,0,0,0,255,0,0,255,0,0,0,128,255,255,255,128}; + constexpr uint8_t expected[] = {0,0,0,0,255,0,0,255,0,0,0,143,255,255,255,143}; check(image && width == 4 && height == 1 && !std::memcmp(image,expected,sizeof(expected)),"dump decodes native indices/CLUT and ST alpha"); if (image) stbi_image_free(image); std::ofstream owner(file,std::ios::binary|std::ios::trunc); owner << "owner edit"; owner.close(); pack = create(root); duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)); pack.reset(); std::ifstream edited(file,std::ios::binary); std::string payload((std::istreambuf_iterator(edited)),{}); check(payload == "owner edit","existing owner dump is never overwritten"); + config(root/"upload","DumpTextureWidthThreshold: 1\nDumpTextureHeightThreshold: 1\nReducePaletteRange: false\n"); pack = create(root/"upload"); duck_texture_pack_track_upload(pack.get(),0,0,1,1,vram.data(),vram.size()); check(duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"tracked source emits texupload dump"); @@ -291,17 +327,251 @@ void dumping(const fs::path& root) { /* PNG fixtures remain independent of the VRAM pointer used by replacement * matching, so dumping never reads already-replaced pixels. */ } + +void official_binary_oracle(const fs::path& root) { + /* Literal oracle from the official DuckStation 697599c47 binary, driven + * by a separately authored synthetic PS-X EXE. No source-derived expected + * filename or hash is used here. Both normal/semi uses must aggregate. */ + std::vector vram(1024*512), source(10*56); + constexpr uint16_t pattern[]{0x3210,0x7654,0xba98,0xfedc}; + for (size_t i = 0; i < source.size(); ++i) source[i] = pattern[i%4]; + place(vram,640,0,10,source); + for (unsigned i = 1; i < 16; ++i) { + vram[256*1024+i] = static_cast((i*1537)&0x7fff); + vram[257*1024+i] = static_cast(((i*293)&0x7fff)|0x8000); + } + auto pack = create(root); char error[512]{}; + check(duck_texture_pack_track_upload(pack.get(),640,0,10,56,vram.data(),vram.size()),"oracle immutable upload tracked"); + auto a = query(vram,640,0,0,0,31,0,31,0,256); + check(duck_texture_pack_dump_draw(pack.get(),&a,0,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"oracle first ordinary use recorded"); + a.u_last = 35; + check(duck_texture_pack_dump_draw(pack.get(),&a,1,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"oracle nested semi use expands same record"); + auto b = query(vram,640,0,0,0,39,32,55,0,257); + check(duck_texture_pack_dump_draw(pack.get(),&b,0,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"oracle second palette crop recorded"); + DuckTexturePackInfo info{}; duck_texture_pack_get_info(pack.get(),&info); + check(info.pending_dump_sources == 1 && info.pending_palette_records == 2 && info.queued_dump_count == 0, + "authoring observes one source/two palettes and writes nothing before retirement"); + check(dumps(root).empty(),"oracle draws do not emit per-draw files"); + place(vram,640,0,10,std::vector(560)); + check(duck_texture_pack_track_upload(pack.get(),640,0,10,56,vram.data(),vram.size()),"oracle full overwrite retires original snapshot"); + flush(pack.get()); duck_texture_pack_get_info(pack.get(),&info); + check(info.queued_dump_count == 2 && info.pending_palette_records == 0,"oracle retirement queues exactly two identities"); + const char* names[]{ + "texupload-STP4-18DDF9B731440D20-0DAE5F73C88719B1-10x56-0-0-36x32-P0-15.png", + "texupload-P4-18DDF9B731440D20-AF506869B80F5CE7-10x56-0-32-40x24-P0-15.png"}; + check(dumps(root).size() == 2,"official binary golden fixture produces only its two PNGs"); + for (unsigned variant = 0; variant < 2; ++variant) { + const unsigned width = variant ? 40 : 36, height = variant ? 24 : 32, y0 = variant ? 32 : 0; + const auto pixels = read_png(root/"dumps"/names[variant],width,height); + bool correct = pixels.size() == size_t(width)*height*4; + for (unsigned y = 0; correct && y < height; ++y) for (unsigned x = 0; x < width; ++x) { + const unsigned index = (pattern[((y+y0)*10+x/4)%4]>>((x%4)*4))&15; + const uint16_t color = index ? static_cast(variant ? ((index*293)&0x7fff)|0x8000 : (index*1537)&0x7fff) : 0; + const size_t at = (size_t(y)*width+x)*4; + for (unsigned c = 0; c < 3; ++c) correct &= pixels[at+c] == (((color>>(5*c))&31)*255+15)/31; + correct &= pixels[at+3] == (color ? 255 : 0); + } + check(correct,"official binary golden crop retains original pixels and captured palette"); + } +} + +void defaults_and_options(const fs::path& root) { + auto pack = create(root/"defaults"); DuckTextureConfig c{}; + duck_texture_pack_get_config(pack.get(),&c); + check(!c.dump_texture_pages && !c.dump_full_texture_pages && !c.dump_c16_textures && c.reduce_palette_range && + c.texture_dump_width_threshold == 16 && c.texture_dump_height_threshold == 16 && + !c.max_vram_write_splits && !c.max_vram_write_coalesce_width && !c.max_vram_write_coalesce_height && + !c.convert_copies_to_writes && !c.replacement_scale_linear_filter && c.dump_vram_write_force_alpha_channel && + c.vram_write_dump_width_threshold == 128 && c.vram_write_dump_height_threshold == 128, + "pinned native authoring defaults are exact"); + std::vector vram(1024*512); char error[512]{}; + auto q = query(vram,0,0,0,0,63,0,31,0,400); + check(duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)) == HD_TEXTURE_LOOKUP_NONE,"default no upload means no page fallback"); + duck_texture_pack_track_upload(pack.get(),0,0,4,15,vram.data(),vram.size()); q.u_last = 15; q.v_last = 14; + duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); flush(pack.get()); + check(dumps(root/"defaults").empty(),"height below minimum suppresses dump even when width meets threshold"); + duck_texture_pack_track_upload(pack.get(),0,0,16,16,vram.data(),vram.size()); q.depth = 2; q.v_last = 15; + check(duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)) == HD_TEXTURE_LOOKUP_NONE,"C16 dumps disabled by default"); + flush(pack.get()); check(dumps(root/"defaults").empty(),"default C16 has no delayed output"); + + config(root/"options","DumpTexturePages: true\nDumpFullTexturePages: true\nDumpTextureForceAlphaChannel: true\n" + "DumpC16Textures: true\nReducePaletteRange: false\nDumpTextureWidthThreshold: 1\nDumpTextureHeightThreshold: 1\n" + "MaxVRAMWriteSplits: 3\nMaxVRAMWriteCoalesceWidth: 9\nMaxVRAMWriteCoalesceHeight: 11\nConvertCopiesToWrites: true\n" + "ReplacementScaleLinearFilter: true\nDumpVRAMWriteForceAlphaChannel: false\nDumpVRAMWriteWidthThreshold: 3\nDumpVRAMWriteHeightThreshold: 5\n" + "Options:\n DumpTexturePages: false\nUnknownOption: true\n"); + pack = create(root/"options"); duck_texture_pack_get_config(pack.get(),&c); + check(c.dump_texture_pages && c.dump_full_texture_pages && c.dump_texture_force_alpha_channel && c.dump_c16_textures && + !c.reduce_palette_range && c.max_vram_write_splits == 3 && c.max_vram_write_coalesce_width == 9 && + c.max_vram_write_coalesce_height == 11 && c.convert_copies_to_writes && duck_texture_pack_linear_filter(pack.get()) && + !c.dump_vram_write_force_alpha_channel && c.vram_write_dump_width_threshold == 3 && c.vram_write_dump_height_threshold == 5, + "exact root-level scalar options parsed without nested Options override"); + DuckTexturePackInfo info{}; duck_texture_pack_get_info(pack.get(),&info); + check(std::strstr(info.diagnostic,"Options") && std::strstr(info.diagnostic,"UnknownOption"),"unsupported nested and unknown options diagnosed"); + q = query(vram,0,0,2,0,0,0,0); + duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)); flush(pack.get()); + const auto files = dumps(root/"options"); check(files.size() == 1,"full page force-alpha emits one page"); + if (!files.empty()) { + DuckTextureKey key{}; duck_texture_parse_name(files[0].filename().u8string().c_str(),&key); + check(key.width == 256 && key.height == 256 && !key.semitransparent,"fullpage and force-alpha filename conventions"); + const auto pixels = read_png(files[0],256,256); + check(pixels.size() == 256*256*4 && pixels[0] == 0 && pixels[3] == 255,"force-alpha preserves opaque native zero"); + } + config(root/"zero","MaxHashCacheVRAMUsageMB: 0\n"); pack = create(root/"zero"); + check(!duck_texture_pack_track_upload(pack.get(),0,0,16,16,vram.data(),vram.size()),"zero snapshot budget falls back without blocking"); + duck_texture_pack_get_info(pack.get(),&info); check(std::strstr(info.diagnostic,"budget"),"snapshot budget fallback diagnosed"); + flush(pack.get()); +} + +void immutable_palettes_and_pages(const fs::path& root) { + std::vector vram(1024*512); + place(vram,0,0,8,std::vector(8*16,0x3210)); vram[400*1024+1] = 0x001f; + auto pack = create(root/"upload"); auto q = query(vram,0,0,0,0,15,0,15,0,400); char error[512]{}; + duck_texture_pack_track_upload(pack.get(),0,0,8,16,vram.data(),vram.size()); + duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); q.u_last = 31; + duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)); + vram[400*1024+1] = 0x83e0; duck_texture_pack_invalidate(pack.get(),1,400,1,1); + duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)); + /* Overwrite both source and CLUT before retirement; neither may be read + * by the writer instead of the saved original words/palettes. */ + place(vram,0,0,8,std::vector(8*16)); vram[400*1024+1] = 0x7c00; + duck_texture_pack_invalidate(pack.get(),0,0,8,16); flush(pack.get()); + const auto files = dumps(root/"upload"); check(files.size() == 2,"palette versions keep separate lifetime records"); + bool old_red = false, saved_green = false; + for (const auto& file : files) { + DuckTextureKey key{}; duck_texture_parse_name(file.filename().u8string().c_str(),&key); + check(key.width == 32 && key.height == 16 && key.semitransparent && key.palette_min == 0 && key.palette_max == 3, + "nested crops union, ordinary+semi aggregate, palette bounds reduced"); + const auto pixels = read_png(file,32,16); + if (pixels.size() >= 8) { old_red |= pixels[4] == 255 && pixels[5] == 0 && pixels[7] == 255; + saved_green |= pixels[4] == 0 && pixels[5] == 255 && pixels[7] == 143; } + } + check(old_red && saved_green,"captured palette versions retain original colors and ST flags"); + + config(root/"page","DumpTexturePages: true\nReducePaletteRange: false\n"); + vram.assign(vram.size(),0); place(vram,0,0,8,std::vector(8*16,0x1111)); + vram[400*1024+1] = 0x001f; vram[401*1024+1] = 0x03e0; + pack = create(root/"page"); q = query(vram,0,0,0,0,15,0,15,0,400); + duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); auto b = q; b.clut_y = 401; + duck_texture_pack_dump_draw(pack.get(),&b,0,error,sizeof(error)); + vram[401*1024+1] = 0x7c00; duck_texture_pack_invalidate(pack.get(),1,401,1,1); + q.u_last = 31; duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); flush(pack.get()); + const auto pagefiles = dumps(root/"page"); check(pagefiles.size() == 2,"changing CLUT B does not fragment CLUT A lifetime"); + bool a_union = false, b_original = false; + for (const auto& file : pagefiles) { + DuckTextureKey key{}; duck_texture_parse_name(file.filename().u8string().c_str(),&key); + const auto pixels = read_png(file,key.width,key.height); + if (!pixels.empty()) { a_union |= key.width == 32 && pixels[0] == 255; + b_original |= key.width == 16 && pixels[1] == 255 && pixels[2] == 0; } + } + check(a_union && b_original,"page authoring residency is isolated per CLUT and snapshots old palettes"); +} + +void split_coalesce_copy_parts(const fs::path& root) { + std::vector vram(1024*512); place(vram,0,0,4,std::vector(4*16,0x3210)); + vram[400*1024+1] = 0x1f; vram[400*1024+2] = 0x3e0; vram[400*1024+3] = 0x7c00; + DuckTextureKey key{}; key.depth = 0; key.source_width_words = 4; key.source_height = 16; key.width = 16; key.height = 16; + key.source_hash = duck_texture_hash_rect(vram.data(),vram.size(),0,0,4,16); + key.palette_hash = duck_texture_hash_words_le(vram.data()+400*1024,4); key.palette_max = 3; + config(root/"split","MaxVRAMWriteSplits: 2\n"); png(root/"split"/"replacements"/filename(key),16,16); + auto pack = create(root/"split"); duck_texture_pack_track_upload(pack.get(),0,0,4,16,vram.data(),vram.size()); + auto q = query(vram,0,0,0,0,15,0,15,0,400); char error[512]{}; + duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); + vram[1024] = 0; duck_texture_pack_invalidate(pack.get(),0,1,1,1); + std::array parts{}; size_t count = 0; + check(duck_texture_pack_match_parts(pack.get(),&q,0,parts.data(),parts.size(),&count) == HD_TEXTURE_LOOKUP_FOUND && count == 3, + "partial overwrite splits original identity into three surviving clips"); + unsigned area = 0; for (size_t i = 0; i < count; ++i) area += unsigned(parts[i].clip_width)*parts[i].clip_height; + check(area == 252,"split clips exclude exactly four overwritten native texels"); + check(duck_texture_pack_match_parts(pack.get(),&q,0,parts.data(),1,&count) == HD_TEXTURE_LOOKUP_ERROR && count == 0, + "insufficient multi-part output capacity never truncates composition"); + auto cut = q; cut.u_last = 3; cut.v_first = cut.v_last = 1; + DuckTextureMatch match{}; check(duck_texture_pack_match(pack.get(),&cut,&match) == HD_TEXTURE_LOOKUP_NONE,"overwritten region has native fallback"); + flush(pack.get()); const auto files = dumps(root/"split"); check(files.size() == 1,"split preserves pending original upload observation"); + if (!files.empty()) { const auto pixels = read_png(files[0],16,16); check(pixels.size() >= 68 && pixels[68] == 255,"split dump uses immutable preoverwrite words"); } + + config(root/"coalesce","MaxVRAMWriteCoalesceHeight: 8\n"); + vram.assign(vram.size(),0); place(vram,0,0,4,std::vector(4*16,0x1111)); vram[400*1024+1] = 31; + pack = create(root/"coalesce"); duck_texture_pack_track_upload(pack.get(),0,0,4,8,vram.data(),vram.size()); + duck_texture_pack_track_upload(pack.get(),0,8,4,8,vram.data(),vram.size()); + q = query(vram,0,0,0,0,15,0,15,0,400); duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); flush(pack.get()); + const auto coalesced = dumps(root/"coalesce"); check(coalesced.size() == 1,"unused adjacent writes coalesce before authoring use"); + if (!coalesced.empty()) { DuckTextureKey k{}; duck_texture_parse_name(coalesced[0].filename().u8string().c_str(),&k); + check(k.source_width_words == 4 && k.source_height == 16,"coalesced original extent retained in filename"); } + + config(root/"copy","ConvertCopiesToWrites: true\nDumpC16Textures: true\nDumpTextureWidthThreshold: 1\nDumpTextureHeightThreshold: 1\n"); + vram.assign(vram.size(),0); place(vram,64,0,4,words(64)); vram[0] = 0x8001; + const uint64_t oldhash = duck_texture_hash_rect(vram.data(),vram.size(),64,0,4,16); + auto changed = vram; changed[64+1024] = vram[0]; + DuckTextureKey updated{}; updated.depth = 2; updated.source_width_words = 4; updated.source_height = 16; + updated.width = 4; updated.height = 16; updated.source_hash = duck_texture_hash_rect(changed.data(),changed.size(),64,0,4,16); + png(root/"copy"/"replacements"/filename(updated)); + pack = create(root/"copy"); duck_texture_pack_track_upload(pack.get(),64,0,4,16,vram.data(),vram.size()); + q = query(vram,64,0,2,0,3,0,15); duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)); + check(duck_texture_pack_begin_copy(pack.get(),0,0,64,1,1,1,vram.data(),vram.size()),"copy begins before native write and retires old observation"); + vram[64+1024] = 0x8001; + check(duck_texture_pack_end_copy(pack.get(),vram.data(),vram.size()),"copy commit records postmask native destination"); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND && match.key.source_hash == updated.source_hash && updated.source_hash != oldhash, + "copy-to-write rehashes whole ORIGINAL destination extent"); + flush(pack.get()); const auto copyfiles = dumps(root/"copy"); check(copyfiles.size() == 1,"copy flush preserves old destination authored texture"); + + pack = create(root/"copy_default"); duck_texture_pack_track_upload(pack.get(),64,0,4,16,vram.data(),vram.size()); + duck_texture_pack_begin_copy(pack.get(),0,0,64,1,1,1,vram.data(),vram.size()); + duck_texture_pack_end_copy(pack.get(),vram.data(),vram.size()); + DuckTexturePackInfo info{}; duck_texture_pack_get_info(pack.get(),&info); + check(info.snapshot_bytes == 0,"default copy drops destination identity rather than cloning source provenance"); + + config(root/"parts","MaxVRAMWriteSplits: 2\n"); + vram.assign(vram.size(),0); place(vram,0,0,8,std::vector(8*16,0x3210)); + key.source_width_words = 8; key.source_hash = duck_texture_hash_rect(vram.data(),vram.size(),0,0,8,16); + key.palette_hash = duck_texture_hash_words_le(vram.data()+400*1024,4); key.width = 16; + png(root/"parts"/"replacements"/filename(key)); key.semitransparent = 1; + png(root/"parts"/"replacements"/filename(key)); key.offset_x = 16; + png(root/"parts"/"replacements"/filename(key)); + pack = create(root/"parts"); duck_texture_pack_track_upload(pack.get(),0,0,8,16,vram.data(),vram.size()); + q = query(vram,0,0,0,0,31,0,15,0,400); + check(duck_texture_pack_match_parts(pack.get(),&q,0,parts.data(),parts.size(),&count) == HD_TEXTURE_LOOKUP_FOUND && count == 2, + "adjacent replacement rectangles compose across one source"); + bool left_ordinary = false, right_st = false; + for (size_t i = 0; i < count; ++i) { left_ordinary |= parts[i].clip_u == 0 && !parts[i].key.semitransparent; + right_st |= parts[i].clip_u == 16 && parts[i].key.semitransparent; } + check(left_ordinary && right_st,"same-coverage ST preference does not remove another rectangle's only ST asset"); + check(duck_texture_pack_match_draw(pack.get(),&q,0,&match) == HD_TEXTURE_LOOKUP_NONE,"single-cover API preserves native fallback for composed draw"); +} + +void copied_pack_ownership(const fs::path& root) { + std::vector vram(1024*512); place(vram,0,0,8,std::vector(8*16,0x3210)); + vram[400*1024+1] = 31; + auto original = create(root/"same"); duck_texture_pack_track_upload(original.get(),0,0,8,16,vram.data(),vram.size()); + auto q = query(vram,0,0,0,0,15,0,15,0,400); char error[512]{}; + duck_texture_pack_dump_draw(original.get(),&q,0,error,sizeof(error)); + { + auto copied = create(root/"same"); check(duck_texture_pack_copy_tracking(copied.get(),original.get()),"same-root tracking clone with pending observations"); + DuckTexturePackInfo info{}; duck_texture_pack_get_info(copied.get(),&info); + check(info.pending_dump_sources == 1 && info.pending_palette_records == 1,"cloned pack owns independent palette record count"); + } /* Copy destroyed FIRST: original still has valid owned observations. */ + q.u_last = 31; + check(duck_texture_pack_dump_draw(original.get(),&q,1,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND, + "original remains valid and expands union after copy destruction"); + flush(original.get()); check(dumps(root/"same").size() == 2,"copy destruction does not steal original larger crop"); + duck_texture_pack_dump_draw(original.get(),&q,1,error,sizeof(error)); + auto elsewhere = create(root/"different"); check(duck_texture_pack_copy_tracking(elsewhere.get(),original.get()),"different-root clone retains provenance"); + original.reset(); flush(elsewhere.get()); + check(dumps(root/"different").size() == 1,"different root does not share filename deduplication suppression"); +} } // namespace -int main() { +int main(int argc, char** argv) { const fs::path root = fs::temp_directory_path()/fs::u8path("psx-duck-texture-\xCE\xA9-"+ std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); try { fixed_hash_vectors(); names(); page_and_decode(root/"page"); upload(root/"upload"); reload_tracking(root/"reload"); palettes(root/"palettes"); st_alias_duplicates(root/"keys"); failed_payloads(root/"invalid"); dumping(root/"dump"); + official_binary_oracle(root/"official-oracle"); defaults_and_options(root/"defaults"); + immutable_palettes_and_pages(root/"snapshots"); split_coalesce_copy_parts(root/"lifecycle"); copied_pack_ownership(root/"ownership"); /* Every artifact is created beneath this fresh, explicitly named test * root. No user-provided paths participate in recursive cleanup. */ - fs::remove_all(root); + if (argc > 1 && !std::strcmp(argv[1],"--keep-artifacts")) std::printf("ARTIFACT_ROOT: %s\n",root.u8string().c_str()); + else fs::remove_all(root); } catch (const std::exception& e) { check(false,e.what()); } if (failures) { std::fprintf(stderr,"test_duckstation_texture_pack: %d failure(s)\n",failures); return 1; } std::puts("PASS: DuckStation format, independent hashes, native matching, decode and dumps"); diff --git a/runtime/tests/test_gl_hd_texture_pack.c b/runtime/tests/test_gl_hd_texture_pack.c index d5b65cf0..77d52b1b 100644 --- a/runtime/tests/test_gl_hd_texture_pack.c +++ b/runtime/tests/test_gl_hd_texture_pack.c @@ -2,6 +2,7 @@ #define PSX_TEST_HD_TEXTURE_PACK 1 #define STB_IMAGE_IMPLEMENTATION #define STBI_ONLY_PNG +#define STBI_ONLY_JPEG #include "../third_party/stb_image.h" #define main scale_fixture_main #include "test_gl_scale_invariance.c" @@ -29,6 +30,60 @@ static uint16_t source_words[4*4]; static uint16_t reference[1024*512]; static uint8_t pixels[64*64*4]; static const uint16_t texture_page=8u|(2u<<7); +static uint16_t part_words[16]; +static uint16_t part_palette[16]; +static const uint16_t part_p4_words[2]={0x1010,0x1132}; + +static void part_png(const char* root,const DuckTextureKey* key,int width,int height,int pattern) { + char stem[256],path[2048]; + check(duck_texture_format_name(key,stem,sizeof(stem)),"partial fixture filename"); + snprintf(path,sizeof(path),"%s/replacements/%s.png",root,stem); + uint8_t rgba[8*8*4]; + for(int y=0;y200 && image.rgba[8*4+3]==128; + gpu_hd_textures_release_image(&image);if(!ready) SDL_Delay(1); + } + if(!ready) { + GpuHdTextureImage image={0}; + if(gpu_hd_textures_acquire_draw(tp,0,0,bounds,0,1,&image)) + fprintf(stderr,"composition mode=%u size=%ux%u pixel8=%u,%u,%u,%u\n",image.alpha_mode,image.width,image.height, + image.rgba[32],image.rgba[33],image.rgba[34],image.rgba[35]); + gpu_hd_textures_release_image(&image); + } + check(ready,"adjacent mixed-density replacements decode and compose"); + gr_fill_rect(0,0,16,16,0x001f);gr_set_semi_transparency(1,1); + gr_draw_textured_rect(2,2,8,2,0,0,0,0,tp);capture(); + check(sample(8,8)[0]>200 && sample(8,8)[1]<20,"first partial region displays its replacement"); + check(sample(9,8)[0]==0 && sample(9,8)[1]==0 && sample(9,8)[2]==0, + "composed non-ST alpha128 keeps occupied black"); + check(sample(10,8)[0]>200 && sample(10,8)[1]<20 && sample(10,8)[2]<20, + "replacement cutout exposes destination rather than native base"); + check(sample(16,8)[0]>200 && sample(16,8)[1]<20 && sample(16,8)[2]>200, + "ST colored alpha0 blends once without native-base double blend"); + check(sample(18,8)[0]<20 && sample(18,8)[2]>200,"composed ST alpha243 remains opaque"); + check(sample(20,8)[0]>200 && sample(20,8)[2]<20,"composed all-zero ST cutout retains destination"); + check(sample(24,8)[0]>200 && sample(24,8)[1]<20,"native transparent hole remains cutout"); + check(sample(32,8)[0]<20 && sample(32,8)[2]>200,"uncovered native opaque texel retains native color"); + uint16_t output[8];gr_vram_transfer_out(2,2,8,1,output); + check(output[0]==0x03e0 && output[2]==0x03ff && output[6]==0x7c00, + "composition does not change guest native draw or STP behavior"); + GpuHdTextureImage first={0},again={0}; + check(gpu_hd_textures_acquire_draw(tp,0,0,bounds,0,1,&first),"composed cache acquire"); + check(gpu_hd_textures_acquire_draw(tp,0,0,bounds,0,1,&again) && first.cache_key==again.cache_key, + "unchanged native content reuses composition cache"); + const uint64_t old_key=first.cache_key; + gpu_hd_textures_release_image(&first);gpu_hd_textures_release_image(&again); + gr_vram_write(775,0,0x7c00); + check(gpu_hd_textures_acquire_draw(tp,0,0,bounds,0,1,&again) && again.cache_key!=old_key && + again.rgba[28*4+2]>200 && again.rgba[28*4+1]<20, + "native hole write refreshes composed content without touching replacements"); + gpu_hd_textures_release_image(&again); + + /* Reduced palette key uses entries 0..1; native holes use other colors. + * Updating an unused replacement-key color must still refresh the holes. */ + const uint16_t p4_tp=13; + const int p4_bounds[4]={0,16,7,16}; + state();gr_vram_transfer_in(0,300,16,1,part_palette); + gr_vram_transfer_in(832,16,2,1,part_p4_words); + ready=0; + for(int i=0;i<2000 && !ready;++i) { + if(gpu_hd_textures_acquire_draw(p4_tp,0,300,p4_bounds,0,0,&again)) + ready=again.alpha_mode==4 && again.rgba[0]>200 && again.rgba[1]<20; + gpu_hd_textures_release_image(&again);if(!ready) SDL_Delay(1); + } + check(ready,"word-aligned P4 partial replacement composes"); + check(gpu_hd_textures_acquire_draw(p4_tp,0,300,p4_bounds,0,0,&first) && first.rgba[8*4+1]>200 && first.rgba[8*4+3]==128, + "native P4 hole decodes current CLUT and native STP"); + const uint64_t palette_key=first.cache_key;gpu_hd_textures_release_image(&first); + gr_vram_write(2,300,0x7c00); + check(gpu_hd_textures_acquire_draw(p4_tp,0,300,p4_bounds,0,0,&again) && again.cache_key!=palette_key && + again.rgba[8*4+2]>200 && again.rgba[8*4+1]<20 && again.rgba[8*4+3]==255, + "CLUT write outside replacement palette range refreshes native holes"); + gpu_hd_textures_release_image(&again); + gr_fill_rect(0,0,16,16,0x001f);gr_draw_textured_rect(2,8,8,1,0,16,0,300,p4_tp);capture(); + check(sample(8,32)[0]>200 && sample(24,32)[2]>200 && sample(28,32)[2]>200, + "P4 replacement and updated native palette holes render together"); + check(vram[8*1024+2]==0x03e0 && vram[8*1024+6]==0x7c00, + "P4 replacement presentation leaves native VRAM intact"); + + /* Adding another lower-density image exercises scale filtering. The + * native guest still sees blue; the replacement alternates red and blue. */ + DuckTextureKey key={0};key.source_hash=duck_texture_hash_words_le(part_words,16); + key.source_width_words=8;key.source_height=2;key.offset_x=6;key.width=2;key.height=2; + key.depth=HD_TEXTURE_DEPTH_16BPP;key.kind=DUCK_TEXTURE_UPLOAD; + part_png(root,&key,4,4,3);char error[256]={0}; + for(int linear=0;linear<2;++linear) { + pack_config(root,linear);check(gpu_hd_textures_reload(error,sizeof(error)),"mixed-density filtering reload"); + state();gr_vram_transfer_in(768,0,8,2,part_words);ready=0; + for(int i=0;i<2000 && !ready;++i) { + if(gpu_hd_textures_acquire_draw(tp,0,0,bounds,0,0,&again)) + ready=again.alpha_mode==4 && again.width==32 && again.rgba[24*4]>200; + gpu_hd_textures_release_image(&again);if(!ready) SDL_Delay(1); + } + check(ready,"mixed-density scaling fixture ready"); + gr_fill_rect(0,0,16,16,0x03e0);gr_draw_textured_rect(2,2,8,2,0,0,0,0,tp);capture(); + const uint8_t* scaled=sample(33,8); + check(linear ? scaled[0]>170 && scaled[0]<220 && scaled[2]>40 && scaled[2]<90 : + scaled[0]>240 && scaled[2]<20,"root scale-filter option changes lower-density replacement colors"); + check(vram[2*1024+8]==0x7c00,"replacement scaling filter never changes native VRAM"); + } +} + int main(int argc,char** argv) { if(argc==2 && !strcmp(argv[1],"--native-baseline")) { char* native_args[]={"hd-baseline","4","twin","1"}; @@ -114,7 +267,7 @@ int main(int argc,char** argv) { } if(argc!=2) return 2; for(int i=0;i<16;++i) source_words[i]=i%4==3?0:i%4==2?0x8000:i%4==1?0x83e0:0x03e0; - pack_png(argv[1],0); pack_png(argv[1],1); + pack_png(argv[1],0); pack_png(argv[1],1);pack_parts(argv[1]);pack_config(argv[1],0); gr_set_backend(GR_BACKEND_SOFTWARE); gr_init(reference); sw_set_faithful_authority(1); native_scene(); sw_set_faithful_authority(0); if(SDL_Init(SDL_INIT_VIDEO)!=0) return 2; @@ -134,6 +287,12 @@ int main(int argc,char** argv) { check(gpu_hd_textures_reload(error,sizeof(error)),"live pack reload succeeds"); /* No new upload: a reload must retain identities of resident textures. */ wait_ready(0); wait_ready(1); + state();gr_copy_rect(512,0,540,0,4,4); + const int copied_bounds[4]={28,0,31,3};GpuHdTextureImage copied={0}; + check(!gpu_hd_textures_acquire_draw(texture_page,0,0,copied_bounds,0,0,&copied), + "default VRAM copy does not invent source upload provenance at destination"); + gpu_hd_textures_release_image(&copied); + gr_fill_rect(540,0,4,4,0); native_scene(); gl_renderer_sync_cpu(); check(memcmp(vram,reference,sizeof(vram))==0,"HD native VRAM matches faithful software across all operations"); uint16_t readback[64*64]; @@ -168,6 +327,7 @@ int main(int argc,char** argv) { check(sample(17,24)[0]==0 && sample(17,24)[1]>200 && sample(17,24)[2]==0, "ST black alpha127 remains occupied semitransparent black"); gr_set_semi_transparency(0,0); + composition_scene(argv[1]);state();wait_ready(0);wait_ready(1); /* A queued draw must consume its source before a later source overwrite. * The next draw falls back because upload identity has been invalidated. */ @@ -184,8 +344,9 @@ int main(int argc,char** argv) { check(readback[0]==0x03e0 && readback[1]==0x03e0 && readback[2]==0x03e0 && readback[3]==0x03e0, "self-overlap keeps sequential native texture reads"); const int bounds[4]={0,0,3,3}; GpuHdTextureImage lease={0}; - check(!gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), - "self-overlap invalidates upload only after its draw"); + check(gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease) && lease.alpha_mode==4 && + lease.rgba[4*4+1]>200 && lease.rgba[0]>200, + "self-overlap preserves surviving replacement regions with new native holes"); gpu_hd_textures_release_image(&lease); gr_vram_transfer_in(512,0,4,4,source_words); check(gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), diff --git a/runtime/tests/test_texture_image_decode.cpp b/runtime/tests/test_texture_image_decode.cpp new file mode 100644 index 00000000..abfa9e69 --- /dev/null +++ b/runtime/tests/test_texture_image_decode.cpp @@ -0,0 +1,60 @@ +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#define STBI_ONLY_JPEG +#define STBI_NO_STDIO +#include "../third_party/stb_image.h" +#include "texture_image_decode.h" + +#include +#include +#include + +int main(int argc, char** argv) { + if (argc != 2) return 2; + int failures = 0; + const auto check = [&](bool okay, const char* message) { + if (!okay) { std::fprintf(stderr, "FAIL: %s\n", message); ++failures; } + }; + const auto read = [&](const char* name) { + std::ifstream input(std::string(argv[1]) + "/" + name, std::ios::binary); + return std::vector(std::istreambuf_iterator(input), {}); + }; + const uint8_t expected[] = {255, 0, 17, 0, 3, 127, 241, 1, + 63, 19, 211, 128, 23, 47, 71, 254}; + std::string error; + TextureImageProbe probe; + DecodedTextureImage image; + for (const char* name : {"rgba.png", "rgba.webp"}) { + const auto bytes = read(name); + check(texture_image_probe(bytes.data(), bytes.size(), 16, probe, error), "bounded probe succeeds"); + check(probe.width == 4 && probe.height == 1, "PNG/WebP dimensions"); + check(texture_image_decode(bytes.data(), bytes.size(), 16, image, error), "PNG/WebP decode succeeds"); + check(image.rgba == std::vector(expected, expected + sizeof(expected)), "raw RGBA including alpha retained"); + check(!texture_image_decode(bytes.data(), bytes.size(), 15, image, error) && image.rgba.empty() && !error.empty(), "budget failure clears output"); + check(!texture_image_decode(bytes.data(), bytes.size() / 2, 16, image, error), "truncated payload rejected"); + } + for (const char* name : {"opaque.jpg", "progressive.jpg", "opaque-lossy.webp"}) { + const auto bytes = read(name); + check(texture_image_decode(bytes.data(), bytes.size(), 1024, image, error), "lossy image decode succeeds"); + check(image.width == 16 && image.height == 16 && image.rgba.size() == 1024, "lossy image dimensions"); + for (size_t i = 0; i + 3 < image.rgba.size(); i += 4) { + check(image.rgba[i] >= 26 && image.rgba[i] <= 34 && image.rgba[i+1] >= 76 && + image.rgba[i+1] <= 84 && image.rgba[i+2] >= 116 && image.rgba[i+2] <= 124 && + image.rgba[i+3] == 255, "lossy RGB tolerance and opaque alpha"); + } + } + auto png = read("rgba.png"); + png[16] = 0; png[17] = 0; png[18] = 0x20; png[19] = 1; + check(!texture_image_probe(png.data(), png.size(), TEXTURE_IMAGE_DEFAULT_DECODED_BUDGET, probe, error), "8193-pixel dimension rejected before decode"); + png[18] = 0x10; png[19] = 0; png[22] = 0x10; png[23] = 1; + check(!texture_image_probe(png.data(), png.size(), TEXTURE_IMAGE_DEFAULT_DECODED_BUDGET, probe, error), "oversized RGBA rejected before decode"); + const uint8_t junk[] = {0, 1, 2, 3}; + check(!texture_image_probe(junk, sizeof(junk), 64, probe, error), "unsupported bytes rejected"); + check(!texture_image_probe(junk, TEXTURE_IMAGE_MAX_ENCODED_BYTES + 1, 64, probe, error), "encoded bound checked before input read"); + const auto animation = read("animated.webp"); + check(!texture_image_probe(animation.data(), animation.size(), 64, probe, error) && error.find("animated") != std::string::npos, "animated WebP explicitly rejected"); + check(!texture_image_probe(nullptr, 0, 64, probe, error), "empty input rejected"); + if (failures) return 1; + std::puts("PASS: PNG/JPEG/WebP raw RGBA, dimensions, budgets, malformed and animated input"); + return 0; +} diff --git a/runtime/texture_image_dependency.cmake b/runtime/texture_image_dependency.cmake new file mode 100644 index 00000000..6c9cb619 --- /dev/null +++ b/runtime/texture_image_dependency.cmake @@ -0,0 +1,37 @@ +include_guard(GLOBAL) +include(FetchContent) +include("${CMAKE_CURRENT_LIST_DIR}/../cmake/psx_dependency_archive.cmake") + +function(psxrecomp_add_texture_image_decoder) + if(TARGET psx_texture_image_decode) + return() + endif() + psxrecomp_dependency_source_dir(psx_libwebp ENV PSX_LIBWEBP_SOURCE_DIR OUT _webp_src) + psxrecomp_dependency_archive(psx_libwebp SOURCE_DIR "${_webp_src}" + OUT_URL _webp_url OUT_HASH _webp_hash) + set(_webp_timestamp_args "") + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) + list(APPEND _webp_timestamp_args DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + # Populate with MakeAvailable, then explicitly exclude unused encoder/tool + # targets from ALL. Only the static webpdecoder target is linked below. + FetchContent_Declare(psx_libwebp URL "${_webp_url}" URL_HASH "${_webp_hash}" + SOURCE_SUBDIR _psx_no_automatic_subdirectory ${_webp_timestamp_args}) + FetchContent_MakeAvailable(psx_libwebp) + set(BUILD_SHARED_LIBS OFF) + set(WEBP_LINK_STATIC ON) + set(WEBP_USE_THREAD OFF) + foreach(_option IN ITEMS ANIM_UTILS CWEBP DWEBP GIF2WEBP IMG2WEBP VWEBP + WEBPINFO LIBWEBPMUX WEBPMUX EXTRAS WEBP_JS FUZZTEST) + set(WEBP_BUILD_${_option} OFF) + endforeach() + add_subdirectory("${psx_libwebp_SOURCE_DIR}" "${psx_libwebp_BINARY_DIR}" EXCLUDE_FROM_ALL) + add_library(psx_texture_image_decode STATIC "${PSXRECOMP_ROOT}/runtime/src/texture_image_decode.cpp") + target_include_directories(psx_texture_image_decode PUBLIC "${PSXRECOMP_ROOT}/runtime/include") + target_include_directories(psx_texture_image_decode PRIVATE "${psx_libwebp_SOURCE_DIR}/src") + target_compile_features(psx_texture_image_decode PUBLIC cxx_std_17) + target_link_libraries(psx_texture_image_decode PUBLIC webpdecoder) + set_property(TARGET psx_texture_image_decode PROPERTY PSX_WEBP_INCLUDE_DIR "${psx_libwebp_SOURCE_DIR}/src") +endfunction() + +psxrecomp_add_texture_image_decoder() diff --git a/third_party/deps.manifest b/third_party/deps.manifest index e12e97d4..7d9386b9 100644 --- a/third_party/deps.manifest +++ b/third_party/deps.manifest @@ -25,3 +25,4 @@ SDL3 SDL3-3.4.10.tar.gz 12b3428 psx_zlib zlib-1.3.1.tar.gz 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz libjuice libjuice-v1.7.2.tar.gz 75159867c4a5a689a6559e11aa0d30c9eba12ce73a4ae3d898b521467e1f635d https://github.com/paullouisageneau/libjuice/archive/refs/tags/v1.7.2.tar.gz psx_openxr_sdk OpenXR-SDK-b76b80adaf65ac3ad6cc1ce61974fb29a5d02352.tar.gz fb8962fe83ca8a5987ebb3d717aa297ea355540449fa230b25307824eab800f4 https://github.com/KhronosGroup/OpenXR-SDK/archive/b76b80adaf65ac3ad6cc1ce61974fb29a5d02352.tar.gz +psx_libwebp libwebp-4fa21912338357f89e4fd51cf2368325b59e9bd9.tar.gz 923f3382a47a2af185c3240c954cf004428b237bd7317413a95146d01eb4b94b https://codeload.github.com/webmproject/libwebp/tar.gz/4fa21912338357f89e4fd51cf2368325b59e9bd9