diff --git a/THIRD_PARTY_ATTRIBUTION.md b/THIRD_PARTY_ATTRIBUTION.md index 01cc8d09..1d71d291 100644 --- a/THIRD_PARTY_ATTRIBUTION.md +++ b/THIRD_PARTY_ATTRIBUTION.md @@ -54,6 +54,7 @@ constrains this repository's own terms; the obligation is to carry the notice. | --- | --- | --- | --- | | [toml11](https://github.com/ToruNiina/toml11) by Toru Niina | MIT | recompiler **and** runtime | `recompiler/lib/toml11/LICENSE`, shipped as `runtime/licenses/toml11-NOTICES.txt` | | [stb_image](http://nothings.org/stb) by Sean Barrett | MIT **or** public domain (Unlicense), at your option | runtime | notice in `runtime/third_party/stb_image.h`, shipped as `runtime/licenses/stb_image-NOTICES.txt` | +| [xxHash 0.8.3](https://github.com/Cyan4973/xxHash/tree/v0.8.3) by Yann Collet | BSD-2-Clause | runtime texture-pack interoperability | `runtime/third_party/xxhash.LICENSE`, shipped as `runtime/licenses/xxhash-NOTICES.txt` | | [rabbitizer](https://github.com/Decompollaborate/rabbitizer) by Decompollaborate | MIT | recompiler only | `recompiler/lib/rabbitizer/LICENSE` | | [ELFIO](https://github.com/serge1/ELFIO) by Serge Lamikhov-Center | MIT | recompiler only | `recompiler/lib/ELFIO/LICENSE.txt` | | [{fmt}](https://github.com/fmtlib/fmt) by Victor Zverovich | MIT | recompiler only | `recompiler/lib/fmt/LICENSE.rst` | @@ -65,6 +66,15 @@ runtime dependency lands. The recompiler-only entries are developer tooling and are not in the shipped package; if that ever changes, their notices have to ship too. +The unmodified xxHash header comes directly from upstream tag `v0.8.3`, +`xxhash.h` (SHA-256 +`17973c0dc49d9854ca26caa191f0e12f7a424b68858d9a78de3860d959d85e4b`). +Its BSD notice remains in the header and the separate notice files. The +DuckStation-compatible filename parser, matching, PNG dumping and cache are +independent psxrecomp implementations; no DuckStation implementation is +included. Format observations and the upstream reference pin are recorded in +[DUCKSTATION_TEXTURE_FORMAT.md](docs/DUCKSTATION_TEXTURE_FORMAT.md). + ## TinyCC (TCC) — toolchain-free overlay compiler shipped to players [TinyCC](https://bellard.org/tcc/) by Fabrice Bellard and contributors, licensed diff --git a/docs/DUCKSTATION_TEXTURE_FORMAT.md b/docs/DUCKSTATION_TEXTURE_FORMAT.md new file mode 100644 index 00000000..92e287b0 --- /dev/null +++ b/docs/DUCKSTATION_TEXTURE_FORMAT.md @@ -0,0 +1,186 @@ +# DuckStation texture-pack interoperability + +psxrecomp implements a bounded subset of modern DuckStation texture packs. +The parser, matcher, cache and dumper are independently written here. The +reference version is upstream commit +`697599c47a646a6cfcc4d246018adf4904d55001`, observed on 2026-10-06. +This pin matters for palette-range and alpha behavior. + +## Provenance + +At the reference pin, DuckStation's +[license](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/LICENSE) +is CC-BY-NC-ND-4.0; its texture-cache and shader files carry the same SPDX +identifier. This project includes no DuckStation implementation, translated +implementation, patch, or binary. Its own code retains the project's +PolyForm Noncommercial terms. Reading format behavior does not provide a +license to incorporate DuckStation's source. Pack artwork has its own rights; +users supply it separately. + +The hash implementation comes directly from +[xxHash v0.8.3](https://github.com/Cyan4973/xxHash/tree/v0.8.3), under +[BSD-2-Clause](https://github.com/Cyan4973/xxHash/blob/v0.8.3/LICENSE). +The unmodified `runtime/third_party/xxhash.h` has SHA-256 +`17973c0dc49d9854ca26caa191f0e12f7a424b68858d9a78de3860d959d85e4b`. +Its license ships as `runtime/licenses/xxhash-NOTICES.txt`. +The [v0.8.0 release](https://github.com/Cyan4973/xxHash/releases/tag/v0.8.0) +made XXH3 output stable across future versions. Thus the upstream v0.8.3 +implementation supplies the unseeded XXH3 behavior used by DuckStation's +vendored v0.8.0 without importing any emulator code. + +## Supported files + +Choose a game/package folder containing `replacements/` and `dumps/`, or choose +`replacements/` itself. Replacement PNGs can live in subdirectories. See the +[player workflow](HD_TEXTURE_PACKS.md) for dumping, editing and reloading. +DuckStation documents its serial-based folders and authoring process in its +[texture replacement wiki](https://github.com/stenzek/duckstation/wiki/Texture-Replacement). + +Paletted filename stems follow this grammar: + +```text +----x---x-P- +kind = texupload | texpage +mode = P4 | P8 | STP4 | STP8 +``` + +Direct stems omit the palette fields: + +```text +---x---x +mode = C16 | STC16 +``` + +Append `.png`. Hash fields contain exactly 16 hexadecimal digits; either case +is accepted. Dimensions, offsets and palette endpoints are decimal. Source +width is in native VRAM words. Rectangle width and X offset are expanded +texels: four per word for P4, two for P8, one for C16. Height and Y offset are +rows. A page's source size is 64x256, 128x256 or 256x256 words respectively. +These definitions follow the +[upstream filename documentation](https://github.com/stenzek/duckstation/wiki/Texture-Replacement). + +The loader validates nonzero sizes, native VRAM bounds, word-aligned horizontal +rectangles, page bounds and palette endpoints. The replacement PNG's actual +size controls its independent X/Y scale; integer and equal-axis scale factors +are not required. The renderer applies the declared native rectangle's origin +and extent when mapping UVs. + +## Hash compatibility + +Both texture and palette identities use unseeded XXH3-64 over packed native +PSX words. Words are serialized low byte first. Rectangles concatenate each +row's words without stride padding. `texupload` hashes the complete tracked +post-write source rectangle; `texpage` hashes its declared subrectangle. +Dimensions and palette data do not enter the texture hash. + +A full CLUT hashes 16 P4 or 256 P8 words, including bit 15. A P8 CLUT extending +beyond VRAM's right edge hashes only the remaining words, without wrapping. +For reduced `Pmin-max` ranges, the reference implementation hashes the **first +`max-min+1` palette words**, without advancing by `min`; a range whose declared +maximum crosses the edge does not match. It does not revalidate used indices +against the declared range during lookup. psxrecomp preserves this behavior +for interoperability. Changing a palette entry outside that prefix can leave +a replacement matched; use full ranges when authoring new packs here. These +are observations of the pinned +[texture-cache behavior](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_hw_texture_cache.cpp). + +## Alpha conventions + +The decoder preserves raw RGBA bytes. Ordinary P4/P8/C16 replacements treat +alpha below 128 as cutout and alpha at least 128 as occupied, including black. +ST names encode PSX transparency classification: alpha at most 242 marks +STP, while alpha at least 243 marks an opaque texel. Exact RGBA zero is cutout; +ST black with alpha at least 243 also becomes PSX transparent zero. Alpha zero +with nonzero RGB in an ST image remains a semitransparent texel. Actual blending +still uses the primitive's PSX transparency mode. These rules follow the pinned +[replacement merge shader](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_hw_shadergen.cpp). + +When both ordinary and ST records match, psxrecomp prefers the primitive's +convention; if only the other convention exists it can still supply the image. +Multiple matching records with the same preferred convention fall back to +native rendering. This deterministic policy avoids dependence on directory +enumeration or unordered-map order. + +The dumper writes rounded 5-bit-to-8-bit native RGB, full CLUT ranges, alpha +zero for native color zero, alpha 255 for occupied ordinary texels, and alpha +128 for occupied STP texels in ST dumps. The pinned upstream dump produces +143 in that last case because of an incidental mask expression; both values +have the same replacement classification. Dumps here are compatible authoring +inputs, not a promise of byte-identical upstream dump PNGs. Native conversion +was checked against the pinned +[GPU helpers](https://github.com/stenzek/duckstation/blob/697599c47a646a6cfcc4d246018adf4904d55001/src/core/gpu_helpers.h). + +## Current compatibility limits + +The module accepts a single replacement covering the complete draw query. +It supports ordinary unchanged uploads and nonwrapping page subrectangles. +Every touched upload loses its identity after a partial overwrite; it does +not reconstruct split or coalesced upload histories. A same-session pack reload +can copy intact upload identities into the newly indexed pack. Loading a +savestate clears them; page matching remains available, while upload matching +needs subsequent guest uploads. The module's rectangle and query caches must +be invalidated on every native VRAM write. + +The following cases use native rendering or report an unsupported-file/option +diagnostic: + +- Wrapped UV intervals, horizontally wrapped pages and wrapped uploads. +- Draws requiring composition from several replacement images. +- `vram-write-` XXH3-128 images, JPEG/WebP payloads and legacy unnamed layouts. +- Copy-to-upload conversion, upload coalescing and split-upload identities. +- Other `config.yaml` behavior options, including replacement bilinear filtering. + +`config.yaml` supports only a bounded `Aliases:` mapping with literal or quoted +filename keys and relative PNG paths. A single-line literal/folded block value +is accepted. Escaped strings, multiline values, anchors and general YAML +features are outside the subset. Absolute paths and `..` traversal are +rejected. A direct canonical replacement takes precedence over its alias. +Unknown top-level options are reported by name and are not applied. + +## Bounds and lifecycle + +The loader hard-fails above 262144 candidate files or entries, scans at most +1048576 directory entries and limits recursive nesting to 16 levels. Duplicate +identities are marked ambiguous. Matching indexes upload hashes and page +geometry, with 256 rectangle hashes and 256 positive/negative query results. +Native page/CLUT writes invalidate affected cached queries. A query requiring +over 4096 candidate page geometries or more than 4 MiB of new hashing reports +an error and renders natively. + +One low-priority background worker performs PNG decoding and dumping. The +decode queue is bounded to 32, decoded metadata to 512 entries, and decoded +pixels to a default 64 MiB budget. Encoded files are limited to 64 MiB and each +image dimension to 8192 before decoding. Pixel leases survive eviction and +pack destruction. Decoding failure leaves native rendering active. + +Dumps use word-aligned draw rectangles, choose `texupload` when an intact +tracked source contains the draw, and otherwise choose `texpage`. The queue +holds at most eight snapshots. A bounded set deduplicates 8192 dump identities +per session; reaching it reports a diagnostic until dump tracking is reset. +PNG writing finishes on pack destruction. A temporary file is published +atomically without replacing an existing final file. Write failures remove +only the created temporary, print an error even on shutdown, and are returned +by the next dump call. All snapshots decode native VRAM rather than replacement +pixels. + +## Independent validation + +`runtime/tests/test_duckstation_texture_pack.cpp` contains fixed XXH3 vectors +generated with the separate Python xxhash 3.6.0 native wheel. Input word `i` +is `((i * 7919) ^ 0xa51c) & 0xffff`, serialized little endian. Examples: + +| Word count | XXH3-64 | +| --- | --- | +| 0 | `2D06800538D394C2` | +| 4 | `3631D2A04EC85311` | +| 8 | `000CA76CB9B329DC` | +| 16 | `9CBCDEA9305D7528` | +| 256 | `0005D0B83A005F4E` | +| 8192 | `C68A00DCCA652920` | + +The test also checks literal filenames, strided native rectangles, crop +coordinates, CLUT edges and reduced-prefix behavior, positive/negative cache +invalidation, reload identity copying, duplicate ambiguity, alias diagnostics, +raw alpha thresholds, Unicode folders, background decoding, native dump pixels, +worker shutdown and preservation of existing edited files. Renderer fixtures +exercise the alpha classification and native-VRAM authority separately. diff --git a/docs/HD_TEXTURE_PACKS.md b/docs/HD_TEXTURE_PACKS.md new file mode 100644 index 00000000..ac05b39f --- /dev/null +++ b/docs/HD_TEXTURE_PACKS.md @@ -0,0 +1,60 @@ +# HD texture packs + +The **HD Texture Packs** mod loads compatible DuckStation texture PNGs and can +dump original textures encountered during gameplay. It is disabled by default. +Use **OpenGL** to display replacements. Software and Vulkan can collect dumps +but continue to display the original artwork. Tomba USA (`SCUS-94236`) is the +first integrated game target; game projects can supply their own manifest for +the same `psx.hd-textures` plugin. + +## Install a pack + +1. Open the launcher's **Mods** page and enable **HD Texture Packs**. +2. Under **Texture pack folder**, click **Open folder**. On first use the + launcher creates `mods/texture-packs/` beside the executable, with + empty `replacements` and `dumps` directories. +3. Copy the pack's replacement PNGs into `replacements`, preserving their + filenames. Subdirectories are supported. If the download already contains + `replacements`, copy that directory's contents into this one. +4. Leave **Load replacements** enabled and **Dump textures** disabled. +5. Select **OpenGL** in Settings, then click **Play**. + +**Change folder** selects another pack root. Empty folders are valid; their +`replacements` and `dumps` directories are created when the pack is configured. +You can also select a pack's `replacements` directory directly, with `dumps` +beside it. Keep editable packs outside `mods/bundled`, which builds regenerate. + +## Dump and edit textures + +1. Enable **HD Texture Packs**, disable **Load replacements**, and enable + **Dump textures**. +2. Click **Play** and visit the scenes you want to capture. Original texture + PNGs appear under `dumps` as the game uses them. +3. Exit to the launcher. Copy selected PNGs into `replacements`, then edit or + enlarge them while keeping the filenames, aspect ratio, and an integer + size multiplier. +4. Enable **Load replacements**, disable **Dump textures**, and click **Play** + using OpenGL to check the artwork. + +The switches apply on **Play**. Restart after adding or editing pack files so +the runtime scans them again. Disabling the mod restores the original artwork. +The plugin does not write replacement pixels into native VRAM or change the +save namespace. + +## Compatibility and diagnostics + +Pack artwork is supplied separately. Matching depends on native game pixels +and palettes; a different region or revision may not match. PNG is supported, +with a maximum of 8192 pixels per side, 64 MiB encoded, and 64 MiB decoded RGBA. +Unsupported identities, configuration features, or images fall back to the +original artwork. After loading a savestate, upload-based replacements need +fresh texture uploads from the game before they can match again; page-based +matching remains available. See [DuckStation texture format notes](DUCKSTATION_TEXTURE_FORMAT.md) +for the exact supported filenames, hashing, alpha behavior, and limitations. + +Development builds expose the [TCP `hd_textures` command](TCP_COMMANDS.md). +It reports the selected root, switches, backend support, pack size, and matched, +ready, and applied draw counts. Optional integer fields `replacements: 0|1`, +`dump: 0|1`, and `reload: 1` control an already configured pack during a test +session. Use `present_shot` to capture the composed OpenGL window; native VRAM +screenshots retain the original artwork. diff --git a/docs/TCP_COMMANDS.md b/docs/TCP_COMMANDS.md index 30216a88..6090a1d8 100644 --- a/docs/TCP_COMMANDS.md +++ b/docs/TCP_COMMANDS.md @@ -55,6 +55,7 @@ Columns: **N** = native, **D** = DuckStation oracle. | `screenshot_wide_hires` | ✓ | | `path`, `base_x` | The displayed band of the native-wide surface at internal resolution (`wide_w×S` by `height×S`). `present_shot` is capped at the window; this checks a widescreen + internal-resolution combination at full size | | `present_shot` | ✓ | | `path` | PNG of the **composed present surface** — the frame after the backend fits the display buffer to the window, so it carries the presented aspect. ⚠ every other capture resolves the display buffer *before* that fit: on a 508×256 display in a 4:3 window they answer 508×256 while the player sees 640×480. Use this one for anything aspect-shaped (widescreen, letterbox), where a pre-fit buffer would hide the very stage the change touches. Staged and fulfilled on the next present, so the ack means *queued* — poll `present_shot_seq`. Unavailable headless and on the Vulkan backend (its swapchain has no readback hook) | | `present_shot_seq` | ✓ | | — | Completion counter for `present_shot`, plus `wrote` (1 = that completion produced a PNG). Sample before staging, poll until `seq` moves. Advances on success *and* failure, so the poll always terminates | +| `hd_textures` | ✓ | | optional `replacements`, `dump` (0 or 1), `reload` (1) | HD pack root, switches, backend support, replacement count, matched/ready/applied draw counts, and queued dump count. Optional controls update the configured host pack on the emulation thread; no configured root is an error. Reload retains unchanged upload identities. OpenGL displays replacements; software/Vulkan retain original artwork. Pair with `present_shot` for visual evidence and native VRAM probes for architectural data | | `gl_interp` | ✓ | | — | OpenGL frame-rate presenter ([FRAME_RATE.md](FRAME_RATE.md)): enabled/suspended, host and target Hz, swaps, `source` (`vblank`/`flip`), `flip_period`, `captures` (new source frames) and `duplicates` (VBlanks that re-presented the same frame) | | `render_pass_stats` | ✓ | | — | Render passes ([RENDER_PASSES.md](RENDER_PASSES.md)): plans, phases wanted/planned (shedding), passes, rollbacks (`nesting_repairs`: watchdog aborts whose skipped frame exits the restore undid), dropped device stores by class, `verify_mismatch` under `PSX_RENDER_PASS_VERIFY=1`, host-time split per pass, smoothed pass cost (`cost_us`; `cost_rewarms`: estimates no pass had run on for a while, measured again), presents made from pass images (`late_presents`: held past the frame's planned end because the next flip was late; `expired`: frames whose images stopped showing after several frame lengths without a flip), pass image textures allocated (`image_textures`, `image_bytes`), `status` (`psx_mod_render_pass_status`: 0 ready, 1 no presenter, 2 backend, 3 disabled, 4 session, 5 fast-forward, 6 busy), `backups_reused` (passes that reused the previous pass's VRAM backup), `spans` / `span_failures` and the last failure `span_fail` (`psx_mod_run_guest_span`: reason, exit PC, `$ra`, PC after a call that did not return) | | `window_size` | ✓ | | `w`, `h` | Resize the game window as a player would (Fit / adaptive widescreen follow it): window-shaped checks in `--hidden-window` runs. Pair with `present_shot` | diff --git a/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml b/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml new file mode 100644 index 00000000..6132b392 --- /dev/null +++ b/mods/builtin/packages/psx.enhancement.hd-textures/1.0.0/manifest.toml @@ -0,0 +1,46 @@ +format_version = 5 +id = "psx.enhancement.hd-textures" +version = "1.0.0" +name = "HD Texture Packs" +author = "psxrecomp" +description = "Load compatible DuckStation PNG texture packs with OpenGL or dump original textures during gameplay. No artwork is bundled." +resolver = "declarative" +save_compatibility = "shared" + +[[target]] +game_id = "*" + +[[feature]] +id = "hd-textures" +name = "HD Texture Packs" +description = "Disabled by default. Open the pack folder to install PNG artwork in replacements, or enable Dump textures to collect originals in dumps. Options apply on Play. OpenGL displays replacements; other backends retain original artwork." +group = "Visual" +default_enabled = false + +[[resource]] +feature = "hd-textures" +id = "pack" +label = "Texture pack folder" +description = "The launcher creates mods/texture-packs/ beside the executable on first use. Open folder to add replacement PNGs or read captured originals; Change folder selects another compatible pack or empty directory." +format = "directory" +required = true + +[[option]] +feature = "hd-textures" +id = "replacements" +label = "Load replacements" +description = "Display matching PNG artwork from replacements in OpenGL. Unsupported or missing replacements retain the original artwork." +type = "boolean" +default = "true" + +[[option]] +feature = "hd-textures" +id = "dump" +label = "Dump textures" +description = "Write original texture PNGs under dumps as they are encountered while playing. Disable after collecting the areas you need." +type = "boolean" +default = "false" + +[[plugin]] +feature = "hd-textures" +id = "psx.hd-textures" diff --git a/overlay_toolchain/include/gpu_render.h b/overlay_toolchain/include/gpu_render.h index 7c46f922..3ef70221 100644 --- a/overlay_toolchain/include/gpu_render.h +++ b/overlay_toolchain/include/gpu_render.h @@ -102,6 +102,7 @@ int gr_render_display_hires(uint32_t *out_pixels, int out_pitch, int disp_x, int disp_y, int disp_w, int disp_h); /* VRAM transfers */ +void gr_vram_upload_begin(int x, int y, int w, int h); void gr_vram_write(int x, int y, uint16_t pixel); uint16_t gr_vram_read(int x, int y); void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *data); diff --git a/overlay_toolchain/include/mod_plugins.h b/overlay_toolchain/include/mod_plugins.h index 0046a14d..6f0f8cb6 100644 --- a/overlay_toolchain/include/mod_plugins.h +++ b/overlay_toolchain/include/mod_plugins.h @@ -270,6 +270,9 @@ void psx_mod_set_world_scene_predicate(PSXModWorldScenePredicate predicate); */ int psx_mod_option_value(const char* package_id, const char* feature_id, const char* option_id, char* out, uint32_t out_size); +/* The committed option for the trusted callback's owning package/feature. */ +int psx_mod_current_option_value(const char* option_id, + char* out, uint32_t out_size); /* * Read the committed owner-selected path for a resource declared by the * package feature whose trusted plugin is currently running. Returns 0 when @@ -278,6 +281,12 @@ int psx_mod_option_value(const char* package_id, const char* feature_id, */ int psx_mod_current_resource_path(const char* resource_id, char* out, uint32_t out_size); +/* Configure an HD pack from the callback's selected directory resource. + * Activation/emulation-thread only. Replacements require OpenGL. */ +int psx_mod_set_hd_texture_pack(const char* resource_id, + int replacements_enabled, int dump_enabled); +int psx_mod_set_hd_texture_dump(int enabled); +int psx_mod_reload_hd_texture_pack(void); /* Read-only canonical media verified by the engine for this plugin's owning * package/feature. The pointer lives until the committed plan is replaced or * cleared. Available only during that plugin's callbacks; returns 0 for an diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index eacbe427..5223da59 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -201,6 +201,15 @@ if(BUILD_TESTING) --sdl-library "$" --output "${CMAKE_CURRENT_BINARY_DIR}/gl-filter-results") set_tests_properties(gl_texture_filter_test PROPERTIES LABELS "gpu;opengl;hardware" TIMEOUT 180) + add_test(NAME gl_hd_texture_pack_test + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_gl_texture_filter.py" + --hd-pack --fixture "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_gl_hd_texture_pack.c" + --cc "${CMAKE_C_COMPILER}" --cxx "${CMAKE_CXX_COMPILER}" + --sdl-include "$" + --sdl-library "$" + --output "${CMAKE_CURRENT_BINARY_DIR}/gl-hd-texture-results") + set_tests_properties(gl_hd_texture_pack_test PROPERTIES LABELS "gpu;opengl;hardware" TIMEOUT 180) endif() add_executable(launcher_device_roundtrip_test tests/test_launcher_device_roundtrip.cpp) @@ -394,6 +403,12 @@ if(BUILD_TESTING) find_package(Threads REQUIRED) target_link_libraries(hd_texture_pack_test PRIVATE Threads::Threads) add_test(NAME hd_texture_pack_test COMMAND hd_texture_pack_test) + add_executable(duckstation_texture_pack_test + tests/test_duckstation_texture_pack.cpp + src/duckstation_texture_pack.cpp) + target_include_directories(duckstation_texture_pack_test PRIVATE include) + target_link_libraries(duckstation_texture_pack_test PRIVATE Threads::Threads) + add_test(NAME duckstation_texture_pack_test COMMAND duckstation_texture_pack_test) # Registered since before this change, but it did NOT LINK on origin/master: # sio.c had grown psx_read_word / psx_get_cycle_count / i_mask / diff --git a/runtime/include/duckstation_texture_pack.h b/runtime/include/duckstation_texture_pack.h new file mode 100644 index 00000000..cb23a9c4 --- /dev/null +++ b/runtime/include/duckstation_texture_pack.h @@ -0,0 +1,139 @@ +#ifndef PSXRECOMP_DUCKSTATION_TEXTURE_PACK_H +#define PSXRECOMP_DUCKSTATION_TEXTURE_PACK_H + +#include "hd_texture_pack.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct DuckTexturePack DuckTexturePack; + +enum DuckTextureKind { DUCK_TEXTURE_UPLOAD = 0, DUCK_TEXTURE_PAGE = 1 }; + +/* Independent representation of the modern filename wire format. Source + * width is in VRAM words; offsets and rectangle sizes are expanded texels. + * Hashes are unseeded XXH3-64 over explicitly little-endian native words. */ +typedef struct DuckTextureKey { + uint64_t source_hash; + uint64_t palette_hash; + uint16_t source_width_words; + uint16_t source_height; + uint16_t offset_x; + uint16_t offset_y; + uint16_t width; + uint16_t height; + uint8_t kind; + uint8_t depth; /* enum HdTextureDepth */ + uint8_t semitransparent; /* STP4 / STP8 / STC16 alpha convention */ + uint8_t palette_min; + uint8_t palette_max; +} DuckTextureKey; + +typedef struct DuckTexturePackInfo { + const char* root; + const char* diagnostic; /* unsupported config/formats and malformed names */ + size_t replacement_count; + size_t ambiguous_count; + size_t ignored_count; +} DuckTexturePackInfo; + +typedef struct DuckTextureMatch { + DuckTextureKey key; + uint64_t entry_id; + const char* replacement_path; + /* Origin of replacement rectangle relative to current texture page, in + * texels. May be negative for uploads spanning more than one page. */ + int32_t origin_u; + int32_t origin_v; + uint16_t source_width; + uint16_t source_height; +} DuckTextureMatch; + +typedef struct DuckTexturePixels { + const uint8_t* rgba; + uint32_t width; + uint32_t height; + uint32_t stride; + void* lease; +} DuckTexturePixels; + +/* Accepts a game/package folder containing replacements/ and dumps/, or the + * replacements directory itself. Missing folders are valid empty packs, so + * a new pack can start by dumping. Recursive PNG scan; no game assets bundled. + * Non-PNG payloads, unsupported options and invalid names are diagnosed. */ +int duck_texture_pack_create(const char* root, DuckTexturePack** out_pack, + char* error, size_t error_capacity); +void duck_texture_pack_destroy(DuckTexturePack* pack); +void duck_texture_pack_get_info(const DuckTexturePack* pack, + DuckTexturePackInfo* out_info); + +int duck_texture_parse_name(const char* filename, DuckTextureKey* out_key); +int duck_texture_format_name(const DuckTextureKey* key, char* output, + size_t capacity); /* stem only, without .png */ +uint64_t duck_texture_hash_words_le(const uint16_t* words, size_t count); +/* Native word rectangle must not wrap. Zero on invalid input (also a valid + * hash value); use valid geometry rather than zero as a success indication. */ +uint64_t duck_texture_hash_rect(const uint16_t* vram, size_t count, + uint16_t x, uint16_t y, + uint16_t width_words, uint16_t height); + +/* Track post-write native VRAM, after PSX set/check-mask semantics. Wrapped + * uploads are invalidated but deliberately not tracked. Copies, coalescing, + * and split-upload identities are outside this initial compatibility subset. + * Any touched upload is dropped, so partial overwrites fail conservatively. */ +int duck_texture_pack_track_upload(DuckTexturePack* pack, + uint16_t x, uint16_t y, + uint16_t width_words, uint16_t height, + const uint16_t* vram, size_t count); +void duck_texture_pack_invalidate(DuckTexturePack* pack, + uint16_t x, uint16_t y, + uint16_t width_words, uint16_t height); +void duck_texture_pack_reset_tracking(DuckTexturePack* pack); +/* Same-session reload only: copy immutable upload identities while the caller + * keeps native VRAM unchanged. Lookup, image and dump caches stay independent. + * This is not a savestate restoration API. Strong replacement on success. */ +int duck_texture_pack_copy_tracking(DuckTexturePack* destination, + const DuckTexturePack* source); + +/* Only one replacement covering the entire inclusive query is accepted. + * UV wrapping, edge-wrapped pages and multi-image composition fall back. + * Uses HD_TEXTURE_LOOKUP_* statuses. Query geometry is shared with Beetle. */ +int duck_texture_pack_match(DuckTexturePack* pack, + const HdTextureDrawQuery* query, + DuckTextureMatch* out_match); +/* Prefer ST names on semitransparent primitives and ordinary names otherwise; + * when only the other convention exists it remains usable. The filename's + * convention always determines how the renderer interprets its alpha. */ +int duck_texture_pack_match_draw(DuckTexturePack* pack, + const HdTextureDrawQuery* query, + int semitransparent, DuckTextureMatch* out_match); + +/* One bounded background worker; no PNG decoding in the render thread. + * Pixels retain raw DuckStation alpha bytes. The renderer interprets the + * key.semitransparent flag; this is not conventional opacity for ST names. + * Release the lease even after pack destruction. */ +void duck_texture_pack_set_decode_budget(DuckTexturePack* pack, size_t bytes); +int duck_texture_pack_request_decode(DuckTexturePack* pack, uint64_t entry_id); +int duck_texture_pack_acquire_decoded(DuckTexturePack* pack, uint64_t entry_id, + DuckTexturePixels* out_pixels); +void duck_texture_pixels_release(DuckTexturePixels* pixels); + +/* Dumps the word-aligned draw rectangle. Uses texupload when exactly one + * unchanged tracked upload covers it; otherwise texpage. Full palette range + * is emitted; transparent native zero stays zero, native STP uses alpha 128 + * for ST names. Existing files are never overwritten. Writes native decoded + * pixels only, never already-replaced pixels. Returns FOUND for a new dump, + * NONE for an existing/skipped dump, ERROR on invalid input or write failure. + * FOUND means queued; bounded background writes finish on pack destruction. + * Any asynchronous write failure is reported by the next dump call. */ +int duck_texture_pack_dump_draw(DuckTexturePack* pack, + const HdTextureDrawQuery* query, + int semitransparent, + char* error, size_t error_capacity); +void duck_texture_pack_reset_dump(DuckTexturePack* pack); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/runtime/include/gpu_gl_renderer.h b/runtime/include/gpu_gl_renderer.h index 16ed6d18..aca7f5b7 100644 --- a/runtime/include/gpu_gl_renderer.h +++ b/runtime/include/gpu_gl_renderer.h @@ -25,6 +25,10 @@ int gl_renderer_select_texture_bank(uint16_t id); /* Retain texture indices while sampling current guest CLUTs (fades/animation). */ int gl_renderer_select_texture_bank_live_clut(uint16_t id); int gl_renderer_texture_banks_supported(void); +/* HD texture presentation uses an independent native SW VRAM authority. + * Both entry points drain pending draws before changing or deleting textures. */ +void gl_renderer_set_hd_texture_mode(int enabled); +void gl_renderer_clear_hd_texture_cache(void); /* Set the GL swap interval / vsync mode (1=vsync, 0=immediate, -1=adaptive). * Safe before or after context creation; applies live when a context exists. */ diff --git a/runtime/include/gpu_hd_textures.h b/runtime/include/gpu_hd_textures.h new file mode 100644 index 00000000..825910b0 --- /dev/null +++ b/runtime/include/gpu_hd_textures.h @@ -0,0 +1,69 @@ +#ifndef PSXRECOMP_GPU_HD_TEXTURES_H +#define PSXRECOMP_GPU_HD_TEXTURES_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Emulation-thread configuration. Roots are external user-owned directories; + * a successful replacement session selects native CPU VRAM authority on GL. + * Reconfiguration retains the previous session if opening the new root fails. + * Empty root with both options off clears the current session. */ +int gpu_hd_textures_configure(const char* root, int replacements_enabled, + int dump_enabled, char* error, size_t capacity); +void gpu_hd_textures_shutdown(void); +int gpu_hd_textures_reload(char* error, size_t capacity); +void gpu_hd_textures_set_dump_enabled(int enabled); +int gpu_hd_textures_replacements_enabled(void); +int gpu_hd_textures_dump_enabled(void); +int gpu_hd_textures_active(void); +typedef struct GpuHdTextureDiag { + const char* root; + int active, replacements, dump, format; + uint64_t draw_queries, matched_draws, ready_draws, applied_draws, dumped_textures; + size_t replacement_count; +} GpuHdTextureDiag; +void gpu_hd_textures_get_diag(GpuHdTextureDiag* out); +void gpu_hd_textures_note_applied(void); +/* Non-GL backends can observe native draws for dumping; replacements remain + * an OpenGL presentation feature. */ +void gpu_hd_textures_observe_draw(uint16_t texpage, uint16_t clut_x, + uint16_t clut_y, const int limits[4], + uint32_t texture_window, int semitransparent); + +/* Native VRAM and residency hooks. A full restage resets identity rather than + * pretending a savestate is an original texture upload. */ +void gpu_hd_textures_set_vram(const uint16_t* vram); +void gpu_hd_textures_track_upload(int x, int y, int width_words, int height, + const uint16_t* words); +void gpu_hd_textures_invalidate(int x, int y, int width_words, int height); +void gpu_hd_textures_reset_tracking(void); + +typedef struct GpuHdTextureImage { + const uint8_t* rgba; + uint32_t width, height, stride; + uint64_t cache_key; + uint64_t generation; + int32_t origin_u, origin_v; + uint32_t source_width, source_height; + /* 1: Beetle (native STP), 2: Duck non-ST, 3: Duck ST alpha encoding. */ + int alpha_mode; + void* lease; +} GpuHdTextureImage; + +/* Inclusive page-UV bounds, before native raster writes the destination. + * Texture-window masks are applied to the queried footprint. Pending decode, + * ambiguity and unsupported geometry all return 0 for native sampling. */ +int gpu_hd_textures_acquire_draw(uint16_t texpage, uint16_t clut_x, + uint16_t clut_y, const int limits[4], + uint32_t texture_window, int semitransparent, + GpuHdTextureImage* image); +void gpu_hd_textures_release_image(GpuHdTextureImage* image); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/runtime/include/gpu_render.h b/runtime/include/gpu_render.h index 7c46f922..4dbad651 100644 --- a/runtime/include/gpu_render.h +++ b/runtime/include/gpu_render.h @@ -102,6 +102,9 @@ int gr_render_display_hires(uint32_t *out_pixels, int out_pitch, int disp_x, int disp_y, int disp_w, int disp_h); /* VRAM transfers */ +/* A0 header invalidation precedes payload writes; an interrupted transfer + * cannot leave a prior immutable texture identity resident. */ +void gr_vram_upload_begin(int x, int y, int w, int h); void gr_vram_write(int x, int y, uint16_t pixel); uint16_t gr_vram_read(int x, int y); void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *data); diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index 228b735b..27a67378 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -328,6 +328,11 @@ void psx_mod_set_world_scene_predicate(PSXModWorldScenePredicate predicate); */ int psx_mod_option_value(const char* package_id, const char* feature_id, const char* option_id, char* out, uint32_t out_size); +/* Read an option of the package/feature whose trusted callback is running. + * Shared framework plugins use this without hard-coding a title package id. + * The same committed-plan and buffer rules as psx_mod_option_value apply. */ +int psx_mod_current_option_value(const char* option_id, + char* out, uint32_t out_size); /* * Read the committed owner-selected path for a resource declared by the * package feature whose trusted plugin is currently running. Returns 0 when @@ -336,6 +341,15 @@ int psx_mod_option_value(const char* package_id, const char* feature_id, */ int psx_mod_current_resource_path(const char* resource_id, char* out, uint32_t out_size); +/* Configure the host HD texture pack from this callback's directory resource. + * Replacements are currently OpenGL-only; dumping also works on software and + * Vulkan. Configuration is host state, never guest RAM or savestate data. + * Activation/emulation-thread only. Failure is reported to the runtime log. */ +int psx_mod_set_hd_texture_pack(const char* resource_id, + int replacements_enabled, int dump_enabled); +/* Emulation-thread controls for an already configured pack. */ +int psx_mod_set_hd_texture_dump(int enabled); +int psx_mod_reload_hd_texture_pack(void); /* Read-only canonical media verified by the engine for this plugin's owning * package/feature. The pointer lives until the committed plan is replaced or * cleared. Available only during that plugin's callbacks; returns 0 for an diff --git a/runtime/licenses/xxhash-NOTICES.txt b/runtime/licenses/xxhash-NOTICES.txt new file mode 100644 index 00000000..9016ed16 --- /dev/null +++ b/runtime/licenses/xxhash-NOTICES.txt @@ -0,0 +1,32 @@ +xxHash Library +Copyright (c) 2012-2021 Yann Collet +All rights reserved. + +Vendored header: xxHash v0.8.3, xxhash.h +Copyright (C) 2012-2023 Yann Collet +Source: https://github.com/Cyan4973/xxHash/blob/v0.8.3/xxhash.h +SHA-256: 17973c0dc49d9854ca26caa191f0e12f7a424b68858d9a78de3860d959d85e4b +The root LICENSE and the header both grant the BSD 2-Clause terms below. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index a065f3c8..56cf540a 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -365,6 +365,9 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/gpu_vram_dirty.c ${PSXRECOMP_ROOT}/runtime/src/gpu_render.c ${PSXRECOMP_ROOT}/runtime/src/gpu_gl_renderer.c + ${PSXRECOMP_ROOT}/runtime/src/gpu_hd_textures.cpp + ${PSXRECOMP_ROOT}/runtime/src/hd_texture_pack.cpp + ${PSXRECOMP_ROOT}/runtime/src/duckstation_texture_pack.cpp ${PSXRECOMP_ROOT}/runtime/src/psx_openxr.c ${PSXRECOMP_ROOT}/runtime/src/mod_controller_source.c ${PSXRECOMP_ROOT}/runtime/src/pad_external_input.c @@ -458,6 +461,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/mod_builtin_pgxp.c ${PSXRECOMP_ROOT}/runtime/src/mod_builtin_bezel.c ${PSXRECOMP_ROOT}/runtime/src/mod_builtin_ram.c + ${PSXRECOMP_ROOT}/runtime/src/mod_builtin_hd_textures.c ${PSXRECOMP_ROOT}/runtime/src/mod_packages.cpp ${PSXRECOMP_ROOT}/runtime/src/mod_media.cpp ${PSXRECOMP_ROOT}/runtime/src/mod_runtime.cpp diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 6f64b711..9d4b592f 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -35,6 +35,7 @@ #endif #include "dma.h" #include "gpu.h" +#include "gpu_hd_textures.h" #include "gpu_render.h" /* gr_scale + gr_render_display_hires (screenshot_hires) */ #include "present_ring.h" #include "gpu_timeline.h" @@ -9838,6 +9839,56 @@ encode:; * surface was actually allocated at (after the driver/memory clamp), the * driver limit, the hr surface size, and the window's point and pixel sizes * (the HiDPI drawable). Everything a preset needs to be verified against. */ +static void handle_hd_textures(int id, const char *json) +{ + GpuHdTextureDiag info; + gpu_hd_textures_get_diag(&info); + /* Development validation controls run at the emulation-thread boundary. + * The normal owner workflow remains the Mods controls applied on Play. */ + const int replacements = json_get_int(json, "replacements", -1); + const int dump = json_get_int(json, "dump", -1); + const int reload = json_get_int(json, "reload", 0); + if (replacements != -1 || dump != -1 || reload) { + if ((replacements < -1 || replacements > 1) || (dump < -1 || dump > 1)) { + send_err(id, "replacements and dump must be 0 or 1"); return; + } + if (!info.root || !info.root[0]) { + send_err(id, "no texture pack is configured; enable the mod and click Play"); return; + } + char error[512] = ""; + if (replacements != -1) { + if (!gpu_hd_textures_configure(info.root, replacements, + dump == -1 ? info.dump : dump, error, sizeof(error))) { + send_err(id, error); return; + } + } else { + if (dump != -1) gpu_hd_textures_set_dump_enabled(dump); + if (reload && !gpu_hd_textures_reload(error, sizeof(error))) { + send_err(id, error); return; + } + } + gpu_hd_textures_get_diag(&info); + } + char root_json[8192]; + json_escape_string(root_json, sizeof(root_json), info.root ? info.root : ""); + const int backend = gr_backend(); + send_fmt("{\"id\":%d,\"ok\":true,\"root\":\"%s\",\"active\":%s," + "\"replacements\":%s,\"dump\":%s,\"format\":%d," + "\"backend\":\"%s\",\"replacement_backend_supported\":%s," + "\"replacement_count\":%llu,\"draw_queries\":%llu," + "\"matched_draws\":%llu,\"ready_draws\":%llu," + "\"applied_draws\":%llu,\"dumped_textures\":%llu}", + id, root_json, info.active ? "true" : "false", + info.replacements ? "true" : "false", info.dump ? "true" : "false", info.format, + backend == GR_BACKEND_OPENGL ? "opengl" : + backend == GR_BACKEND_VULKAN ? "vulkan" : "software", + backend == GR_BACKEND_OPENGL ? "true" : "false", + (unsigned long long)info.replacement_count, + (unsigned long long)info.draw_queries, (unsigned long long)info.matched_draws, + (unsigned long long)info.ready_draws, (unsigned long long)info.applied_draws, + (unsigned long long)info.dumped_textures); +} + static void handle_video_info(int id, const char *json) { (void)json; @@ -15405,6 +15456,7 @@ static const CmdEntry s_commands[] = { { "dump_buffer", handle_dump_buffer }, { "wide_full", handle_wide_full }, { "video_info", handle_video_info }, + { "hd_textures", handle_hd_textures }, { "screenshot_wide_hires", handle_screenshot_wide_hires }, { "wide_shot", handle_wide_shot }, { "gpu_opcodes", handle_gpu_opcodes }, diff --git a/runtime/src/duckstation_texture_pack.cpp b/runtime/src/duckstation_texture_pack.cpp new file mode 100644 index 00000000..6dfbfed6 --- /dev/null +++ b/runtime/src/duckstation_texture_pack.cpp @@ -0,0 +1,751 @@ +#include "duckstation_texture_pack.h" +#include "../third_party/stb_image.h" +#define XXH_INLINE_ALL +#include "../third_party/xxhash.h" +#include "png_write.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#else +#include +#include +#endif + +namespace fs = std::filesystem; +namespace { +constexpr size_t kWords = 1024u * 512u; +constexpr size_t kMaxEntries = 262144; +constexpr size_t kMaxUploads = 8192; +constexpr size_t kMaxRectCache = 256; +constexpr size_t kMaxDumps = 8192; +constexpr size_t kMaxDumpQueue = 8; +constexpr size_t kMaxDecodeQueue = 32; +constexpr size_t kDefaultBudget = 64u * 1024u * 1024u; +constexpr size_t kMaxEncoded = 64u * 1024u * 1024u; +constexpr uint32_t kMaxImageDimension = 8192; + +void error_text(char* dest, size_t capacity, const std::string& text) { + if (dest && capacity) std::snprintf(dest, capacity, "%s", text.c_str()); +} +std::string lower(std::string s) { + for (char& c : s) if (c >= 'A' && c <= 'Z') c += 'a' - 'A'; + return s; +} +std::string trim(const std::string& s) { + const auto a = s.find_first_not_of(" \t\r\n"); + if (a == std::string::npos) return {}; + return s.substr(a, s.find_last_not_of(" \t\r\n") - a + 1); +} +unsigned expansion(unsigned depth) { return depth == 0 ? 4 : depth == 1 ? 2 : 1; } +struct Rect { unsigned x, y, w, h; }; +bool intersects(Rect a, Rect b) { + return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h; +} +bool contains(Rect a, Rect b) { + return b.x >= a.x && b.y >= a.y && b.x + b.w <= a.x + a.w && b.y + b.h <= a.y + a.h; +} +bool valid_rect(Rect r) { return r.w && r.h && r.x + r.w <= 1024 && r.y + r.h <= 512; } + +template bool number(const std::string& text, T& value, int base = 10) { + if (text.empty()) return false; + const auto result = std::from_chars(text.data(), text.data() + text.size(), value, base); + return result.ec == std::errc{} && result.ptr == text.data() + text.size(); +} +bool dimensions(const std::string& s, uint16_t& w, uint16_t& h) { + const size_t at = s.find('x'); + return at != std::string::npos && number(s.substr(0, at), w) && + number(s.substr(at + 1), h) && w && h; +} +bool valid_key(const DuckTextureKey& k) { + if (k.kind > DUCK_TEXTURE_PAGE || k.depth > 2 || k.semitransparent > 1 || + !k.source_width_words || k.source_width_words > 1024 || !k.source_height || + k.source_height > 512 || !k.width || !k.height) return false; + const unsigned ppw = expansion(k.depth); + if (k.offset_x % ppw || k.width % ppw || + unsigned(k.offset_x) + k.width > unsigned(k.source_width_words) * ppw || + unsigned(k.offset_y) + k.height > k.source_height) return false; + if (k.kind == DUCK_TEXTURE_PAGE && (k.source_width_words != 256 / ppw || + k.source_height != 256 || unsigned(k.offset_x) + k.width > 256 || + unsigned(k.offset_y) + k.height > 256)) return false; + if (k.depth < 2 && (k.palette_min > k.palette_max || + k.palette_max >= (k.depth == 0 ? 16 : 256))) return false; + return k.depth < 2 || (!k.palette_hash && !k.palette_min && !k.palette_max); +} +std::string stem(const DuckTextureKey& k) { + char result[192]; + if (!duck_texture_format_name(&k, result, sizeof(result))) return {}; + return result; +} +bool query_rect(const HdTextureDrawQuery* q, Rect& r) { + if (!q || !q->vram || q->vram_word_count < kWords || q->depth > 2 || + q->u_first > q->u_last || q->v_first > q->v_last || + q->page_x >= 1024 || q->page_x % 64 || q->page_y > 256 || q->page_y % 256) + return false; + const unsigned ppw = expansion(q->depth); + /* A page that wraps in X is outside the initial safe subset. */ + if (q->page_x + 256 / ppw > 1024) return false; + r = {q->page_x + q->u_first / ppw, unsigned(q->page_y) + q->v_first, + unsigned(q->u_last / ppw - q->u_first / ppw + 1), + unsigned(q->v_last - q->v_first + 1)}; + return valid_rect(r); +} + +uint64_t palette_hash(const HdTextureDrawQuery& q, unsigned first, unsigned last, bool& valid) { + valid = false; + if (q.depth == 2) { valid = true; return 0; } + const unsigned full = q.depth == 0 ? 16 : 256; + if (first > last || last >= full || q.clut_x >= 1024 || q.clut_y >= 512 || q.clut_x % 16) + return 0; + unsigned count = last - first + 1; + if (first == 0 && last == full - 1) count = std::min(full, 1024u - q.clut_x); + else if (q.clut_x + last >= 1024) return 0; + /* Compatibility with the pinned upstream behavior: range length is used, + * but min does not offset the hashed palette pointer. */ + valid = true; + return duck_texture_hash_words_le(q.vram + q.clut_y * 1024 + q.clut_x, count); +} + +struct Entry { DuckTextureKey key{}; std::string path; bool ambiguous = false; }; +struct Upload { Rect rect; uint64_t hash; }; +struct RectHash { Rect rect; uint64_t hash; const uint16_t* vram; }; +struct PageGroup { + Rect texels; + std::unordered_map> hashes; +}; +struct MatchCache { + HdTextureDrawQuery query; + int st, status; + DuckTextureMatch result; +}; +struct Image { std::vector rgba; uint32_t w = 0, h = 0; }; +struct Lease { std::shared_ptr image; }; +enum class DecodeStatus { Queued, Loading, Ready, Failed }; +struct Decode { + DecodeStatus status = DecodeStatus::Queued; + std::string path; + std::shared_ptr image; + uint64_t use = 0; +}; +struct Dump { fs::path path; std::shared_ptr image; }; +/* png_write.h's C-compatible CRC table is mutable during initialization. + * Different pack workers therefore serialize this TU's calls to the writer. */ +std::mutex png_writer_mutex; +std::atomic dump_sequence{static_cast(std::chrono::steady_clock::now().time_since_epoch().count())}; + +FILE* create_exclusive(const fs::path& path) { +#ifdef _WIN32 + HANDLE handle = CreateFileW(path.c_str(),GENERIC_WRITE,0,nullptr,CREATE_NEW,FILE_ATTRIBUTE_NORMAL,nullptr); + if (handle == INVALID_HANDLE_VALUE) return nullptr; + const int fd = _open_osfhandle(reinterpret_cast(handle),_O_BINARY | _O_WRONLY); + if (fd < 0) { CloseHandle(handle); return nullptr; } + FILE* file = _fdopen(fd,"wb"); + if (!file) _close(fd); +#else + const int fd = ::open(path.c_str(),O_WRONLY | O_CREAT | O_EXCL,0600); + if (fd < 0) return nullptr; + FILE* file = ::fdopen(fd,"wb"); + if (!file) ::close(fd); +#endif + return file; +} + +bool publish_exclusive(const fs::path& temp, const fs::path& target) { +#ifdef _WIN32 + if (MoveFileExW(temp.c_str(),target.c_str(),0)) return true; + const DWORD code = GetLastError(); + if (code != ERROR_ALREADY_EXISTS && code != ERROR_FILE_EXISTS) throw std::runtime_error("cannot publish texture dump PNG"); +#else + if (::link(temp.c_str(),target.c_str()) == 0) return true; + if (errno != EEXIST) throw std::runtime_error("cannot publish texture dump PNG"); +#endif + return false; +} + +struct Worker { + std::mutex mutex; + std::condition_variable wake; + std::unordered_map decoded; + std::deque decode_queue; + std::deque dump_queue; + std::thread thread; + size_t budget = kDefaultBudget, used = 0; + uint64_t tick = 0; + bool stop = false; + std::string dump_error; + ~Worker() { + { std::lock_guard lock(mutex); stop = true; } + wake.notify_all(); + if (thread.joinable()) thread.join(); + } + void evict(size_t incoming) { + while (used + incoming > budget) { + auto candidate = decoded.end(); + for (auto it = decoded.begin(); it != decoded.end(); ++it) + if (it->second.status == DecodeStatus::Ready && + (candidate == decoded.end() || it->second.use < candidate->second.use)) candidate = it; + if (candidate == decoded.end()) break; + used -= candidate->second.image->rgba.size(); + decoded.erase(candidate); + } + } +}; + +void worker_main(Worker* w) { +#ifdef _WIN32 + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); +#endif + for (;;) { + uint64_t id = 0; std::string path; Dump dump; size_t budget = 0; + { + std::unique_lock lock(w->mutex); + w->wake.wait(lock, [&] { return w->stop || !w->decode_queue.empty() || !w->dump_queue.empty(); }); + /* Finish queued dumps on shutdown so the final frame is durable. */ + if (w->stop && w->dump_queue.empty()) return; + if (!w->stop && !w->decode_queue.empty()) { + id = w->decode_queue.front(); w->decode_queue.pop_front(); + auto it = w->decoded.find(id); + if (it == w->decoded.end()) continue; + it->second.status = DecodeStatus::Loading; + path = it->second.path; budget = w->budget; + } else { dump = std::move(w->dump_queue.front()); w->dump_queue.pop_front(); } + } + if (id) { + std::shared_ptr image; + try { + std::ifstream input(fs::u8path(path), std::ios::binary | std::ios::ate); + const auto length = input ? input.tellg() : std::streampos(-1); + if (length > 0 && static_cast(length) <= kMaxEncoded) { + std::vector encoded(static_cast(length)); + input.seekg(0); + int width = 0, height = 0, channels = 0; + constexpr uint8_t png_signature[] = {137,80,78,71,13,10,26,10}; + if (input.read(reinterpret_cast(encoded.data()), length) && + encoded.size() >= sizeof(png_signature) && + !std::memcmp(encoded.data(),png_signature,sizeof(png_signature)) && + stbi_info_from_memory(encoded.data(), static_cast(encoded.size()), &width, &height, &channels) && + width > 0 && height > 0 && width <= int(kMaxImageDimension) && height <= int(kMaxImageDimension) && + uint64_t(width) * height * 4 <= budget) { + stbi_uc* pixels = stbi_load_from_memory(encoded.data(), static_cast(encoded.size()), + &width, &height, &channels, 4); + if (pixels) { + std::unique_ptr owned(pixels, &stbi_image_free); + image = std::make_shared(); image->w = width; image->h = height; + image->rgba.assign(pixels, pixels + size_t(width) * height * 4); + } + } + } + } catch (...) { image.reset(); } + std::lock_guard lock(w->mutex); + auto it = w->decoded.find(id); + if (it == w->decoded.end()) continue; + if (image) w->evict(image->rgba.size()); + if (!image || w->used + image->rgba.size() > w->budget) { it->second.status = DecodeStatus::Failed; continue; } + w->used += image->rgba.size(); it->second.image = std::move(image); + it->second.status = DecodeStatus::Ready; it->second.use = ++w->tick; + } else { + bool created = false; + fs::path temporary = dump.path; + temporary += ".partial-" + std::to_string(++dump_sequence); + try { + std::error_code ec; + fs::create_directories(dump.path.parent_path(), ec); + if (ec) throw std::runtime_error("cannot create texture dump directory: " + ec.message()); + if (fs::exists(dump.path,ec) && !ec) continue; + FILE* file = create_exclusive(temporary); + if (!file) throw std::runtime_error("cannot create temporary texture dump PNG"); + created = true; + bool success = false; + try { + std::lock_guard write_lock(png_writer_mutex); + success = png_write_rgba(file,dump.image->rgba.data(),dump.image->w,dump.image->h) != 0; + } catch (...) { std::fclose(file); throw; } + if (std::fclose(file) != 0 || !success) throw std::runtime_error("cannot write texture dump PNG"); + publish_exclusive(temporary,dump.path); + fs::remove(temporary,ec); + } catch (const std::exception& e) { + if (created) { std::error_code ignored; fs::remove(temporary,ignored); } + std::lock_guard lock(w->mutex); w->dump_error = e.what(); + std::fprintf(stderr,"[texture-pack] dump failed: %s\n",e.what()); + } catch (...) { + if (created) { std::error_code ignored; fs::remove(temporary,ignored); } + std::lock_guard lock(w->mutex); w->dump_error = "texture dump allocation failed"; + std::fprintf(stderr,"[texture-pack] dump failed: allocation failed\n"); + } + } + } +} +} // namespace + +struct DuckTexturePack { + std::string root, diagnostic; + fs::path replacements, dumps; + std::vector entries; + std::array>, 3> uploads_by_hash; + std::array, 3> page_groups; + std::unordered_map by_name; + std::vector uploads; + std::vector rect_cache; + std::vector match_cache; + std::unordered_set dumped; + size_t ignored = 0, ambiguous = 0; + bool geometry_limit_reported = false, dump_limit_reported = false; + Worker worker; +}; + +namespace { +void note(DuckTexturePack& p, const std::string& s) { + ++p.ignored; + if (p.diagnostic.size() < 2048) p.diagnostic += (p.diagnostic.empty() ? "" : "; ") + s; +} +void add_entry(DuckTexturePack& p, const DuckTextureKey& key, const fs::path& path) { + const std::string name = stem(key); + auto previous = p.by_name.find(name); + if (previous != p.by_name.end()) { + auto& entry = p.entries[previous->second]; + if (!entry.ambiguous) { entry.ambiguous = true; ++p.ambiguous; } + return; + } + if (p.entries.size() >= kMaxEntries) throw std::runtime_error("texture replacement entry limit (262144) exceeded"); + p.by_name.emplace(name, p.entries.size()); + if (key.kind == DUCK_TEXTURE_UPLOAD) p.uploads_by_hash[key.depth][key.source_hash].push_back(p.entries.size()); + else { + const uint64_t geometry = (uint64_t(key.offset_x) << 48) | (uint64_t(key.offset_y) << 32) | + (uint64_t(key.width) << 16) | key.height; + PageGroup& group = p.page_groups[key.depth][geometry]; + group.texels = {key.offset_x, key.offset_y, key.width, key.height}; + group.hashes[key.source_hash].push_back(p.entries.size()); + } + p.entries.push_back({key, path.u8string(), false}); +} +uint64_t cached_hash(DuckTexturePack& p, const HdTextureDrawQuery& q, Rect r) { + for (const auto& saved : p.rect_cache) + if (saved.vram == q.vram && saved.rect.x == r.x && saved.rect.y == r.y && saved.rect.w == r.w && saved.rect.h == r.h) return saved.hash; + const uint64_t value = duck_texture_hash_rect(q.vram, q.vram_word_count, r.x, r.y, r.w, r.h); + if (p.rect_cache.size() >= kMaxRectCache) p.rect_cache.erase(p.rect_cache.begin()); + p.rect_cache.push_back({r, value, q.vram}); + return value; +} +std::string unquote(std::string s) { + s = trim(s); + if (s.size() > 1 && ((s.front() == '\'' && s.back() == '\'') || (s.front() == '"' && s.back() == '"'))) + return s.substr(1, s.size() - 2); + const auto comment = s.find(" #"); + return comment == std::string::npos ? s : trim(s.substr(0, comment)); +} +void load_config(DuckTexturePack& p, const fs::path& path) { + std::error_code ec; + if (!fs::exists(path, ec)) return; + if (fs::file_size(path, ec) > 1024 * 1024 || ec) { note(p, "config.yaml exceeds 1 MiB"); return; } + std::ifstream file(path); std::string line; bool aliases = false; + while (std::getline(file, line)) { + const std::string text = trim(line); + if (text.empty() || text.front() == '#') continue; + if (line.size() > 4096) { note(p, "oversized config.yaml line"); continue; } + const auto colon = text.find(':'); + if (colon == std::string::npos) { note(p, "unsupported config.yaml syntax"); continue; } + const std::string key = unquote(text.substr(0, colon)); + std::string value = unquote(text.substr(colon + 1)); + const bool indented = line.front() == ' ' || line.front() == '\t'; + if (!indented) { aliases = key == "Aliases"; if (!aliases) note(p, "config.yaml option not applied: " + key); continue; } + if (!aliases) continue; + DuckTextureKey parsed{}; + if (!duck_texture_parse_name(key.c_str(), &parsed)) { note(p, "invalid alias key"); continue; } + if (value == "|" || value == "|-" || value == ">" || value == ">-") { + if (!std::getline(file, line) || line.find_first_not_of(" \t") < 4) { note(p, "unsupported alias block"); continue; } + value = trim(line); + } + fs::path relative = fs::u8path(value); + bool unsafe = relative.empty() || relative.is_absolute() || relative.has_root_name(); + for (const auto& part : relative) if (part == "..") unsafe = true; + if (unsafe || lower(relative.extension().string()) != ".png") { note(p, "unsupported alias path"); continue; } + const fs::path target = p.replacements / relative; + if (!fs::is_regular_file(target, ec) || ec) { note(p, "alias target missing"); ec.clear(); continue; } + const auto old = p.by_name.find(stem(parsed)); + /* A direct canonical filename takes precedence over an alias. */ + if (old == p.by_name.end()) add_entry(p, parsed, target); + } +} +} // namespace + +extern "C" { +int duck_texture_parse_name(const char* filename, DuckTextureKey* out) { + if (!filename || !out) return 0; + try { + std::string text(filename); + if (text.size() > 192 || text.find_first_of("/\\") != std::string::npos) return 0; + const auto dot = text.find('.'); + if (dot != std::string::npos) { + if (lower(text.substr(dot)) != ".png") return 0; + text.resize(dot); + } + std::vector parts; + size_t begin = 0; + for (size_t at = 0; at <= text.size(); ++at) if (at == text.size() || text[at] == '-') { + parts.push_back(text.substr(begin, at - begin)); begin = at + 1; + } + if (parts.size() != 7 && parts.size() != 10) return 0; + DuckTextureKey k{}; + if (parts[0] == "texupload") k.kind = DUCK_TEXTURE_UPLOAD; + else if (parts[0] == "texpage") k.kind = DUCK_TEXTURE_PAGE; + else return 0; + std::string mode = parts[1]; + if (mode.size() > 2 && mode.substr(0, 2) == "ST") { k.semitransparent = 1; mode.erase(0, 2); } + if (mode == "P4") k.depth = 0; else if (mode == "P8") k.depth = 1; else if (mode == "C16") k.depth = 2; else return 0; + if (parts[2].size() != 16 || !number(parts[2], k.source_hash, 16)) return 0; + size_t field = 3; + if (k.depth < 2) { + if (parts.size() != 10 || parts[3].size() != 16 || !number(parts[3], k.palette_hash, 16)) return 0; + ++field; + } else if (parts.size() != 7) return 0; + if (!dimensions(parts[field++], k.source_width_words, k.source_height) || + !number(parts[field++], k.offset_x) || !number(parts[field++], k.offset_y) || + !dimensions(parts[field++], k.width, k.height)) return 0; + if (k.depth < 2) { + unsigned a = 0, b = 0; + if (parts[field].empty() || parts[field][0] != 'P' || !number(parts[field].substr(1), a) || + !number(parts[field + 1], b) || a > 255 || b > 255) return 0; + k.palette_min = static_cast(a); k.palette_max = static_cast(b); + } + if (!valid_key(k)) return 0; + *out = k; return 1; + } catch (...) { return 0; } +} + +int duck_texture_format_name(const DuckTextureKey* k, char* out, size_t capacity) { + if (!k || !out || !capacity || !valid_key(*k)) return 0; + const char* kind = k->kind == DUCK_TEXTURE_UPLOAD ? "texupload" : "texpage"; + const char* mode = k->depth == 0 ? "P4" : k->depth == 1 ? "P8" : "C16"; + const char* st = k->semitransparent ? "ST" : ""; + int count; + if (k->depth < 2) count = std::snprintf(out, capacity, + "%s-%s%s-%016llX-%016llX-%ux%u-%u-%u-%ux%u-P%u-%u", kind, st, mode, + static_cast(k->source_hash), static_cast(k->palette_hash), + k->source_width_words, k->source_height, k->offset_x, k->offset_y, k->width, k->height, k->palette_min, k->palette_max); + else count = std::snprintf(out, capacity, "%s-%s%s-%016llX-%ux%u-%u-%u-%ux%u", kind, st, mode, + static_cast(k->source_hash), k->source_width_words, k->source_height, + k->offset_x, k->offset_y, k->width, k->height); + return count >= 0 && static_cast(count) < capacity; +} + +uint64_t duck_texture_hash_words_le(const uint16_t* words, size_t count) { + if ((!words && count) || count > kWords) return 0; + XXH3_state_t state; + XXH3_64bits_reset(&state); + std::array bytes{}; + while (count) { + const size_t chunk = std::min(count, bytes.size() / 2); + for (size_t i = 0; i < chunk; ++i) { bytes[2 * i] = words[i] & 255; bytes[2 * i + 1] = words[i] >> 8; } + XXH3_64bits_update(&state, bytes.data(), chunk * 2); + words += chunk; count -= chunk; + } + return XXH3_64bits_digest(&state); +} +uint64_t duck_texture_hash_rect(const uint16_t* vram, size_t count, uint16_t x, uint16_t y, + uint16_t width, uint16_t height) { + if (!vram || count < kWords || !valid_rect({x, y, width, height})) return 0; + XXH3_state_t state; XXH3_64bits_reset(&state); + std::array bytes{}; + for (unsigned row = 0; row < height; ++row) { + const uint16_t* src = vram + (y + row) * 1024 + x; + for (unsigned i = 0; i < width; ++i) { bytes[2 * i] = src[i] & 255; bytes[2 * i + 1] = src[i] >> 8; } + XXH3_64bits_update(&state, bytes.data(), size_t(width) * 2); + } + return XXH3_64bits_digest(&state); +} + +int duck_texture_pack_create(const char* root, DuckTexturePack** out, char* error, size_t capacity) { + if (out) *out = nullptr; + if (!out || !root || !root[0]) { error_text(error, capacity, "texture pack root is unset"); return 0; } + try { + auto p = std::make_unique(); + std::error_code ec; fs::path input = fs::absolute(fs::u8path(root), ec).lexically_normal(); + if (ec || (fs::exists(input, ec) && !fs::is_directory(input, ec))) { + error_text(error, capacity, "texture pack root is not a directory"); return 0; + } + if (lower(input.filename().u8string()) == "replacements") { p->replacements = input; input = input.parent_path(); } + else p->replacements = input / "replacements"; + p->root = input.u8string(); p->dumps = input / "dumps"; + std::vector paths; + if (fs::is_directory(p->replacements, ec)) { + size_t scanned = 0; + for (fs::recursive_directory_iterator it(p->replacements, fs::directory_options::skip_permission_denied, ec), end; + !ec && it != end; it.increment(ec)) { + if (++scanned > kMaxEntries * 4) throw std::runtime_error("texture replacement directory scan limit exceeded"); + if (it.depth() >= 16) { if (it->is_directory(ec)) it.disable_recursion_pending(); note(*p, "replacement nesting exceeds 16 levels"); } + if (it->is_regular_file(ec)) { + const std::string name = it->path().filename().u8string(); + if (name.rfind("texupload-", 0) != 0 && name.rfind("texpage-", 0) != 0 && name.rfind("vram-write-", 0) != 0) continue; + if (lower(it->path().extension().string()) != ".png") { note(*p, "replacement format outside PNG subset"); continue; } + if (paths.size() >= kMaxEntries) throw std::runtime_error("texture replacement file limit (262144) exceeded"); + paths.push_back(it->path()); + } + } + if (ec) { error_text(error, capacity, "cannot scan replacement directory: " + ec.message()); return 0; } + } else if (ec && ec != std::errc::no_such_file_or_directory) { + error_text(error, capacity, "cannot inspect replacement directory: " + ec.message()); return 0; + } + std::sort(paths.begin(), paths.end()); + for (const auto& path : paths) { + DuckTextureKey k{}; + if (duck_texture_parse_name(path.filename().u8string().c_str(), &k)) add_entry(*p, k, path); + else note(*p, "invalid or unsupported replacement filename"); + } + load_config(*p, input / "config.yaml"); + *out = p.release(); return 1; + } catch (const std::exception& e) { error_text(error, capacity, e.what()); return 0; } + catch (...) { error_text(error, capacity, "texture pack allocation failed"); return 0; } +} +void duck_texture_pack_destroy(DuckTexturePack* p) { delete p; } +void duck_texture_pack_get_info(const DuckTexturePack* p, DuckTexturePackInfo* out) { + if (!out) return; + *out = {}; + if (p) { out->root = p->root.c_str(); out->diagnostic = p->diagnostic.c_str(); out->replacement_count = p->entries.size(); + out->ambiguous_count = p->ambiguous; out->ignored_count = p->ignored; } +} +void duck_texture_pack_invalidate(DuckTexturePack* p, uint16_t x, uint16_t y, uint16_t w, uint16_t h) { + if (!p || !w || !h) return; + if (w > 1024 || h > 512) { duck_texture_pack_reset_tracking(p); return; } + const unsigned px = x % 1024, py = y % 512; + const unsigned w0 = std::min(unsigned(w), 1024 - px), h0 = std::min(unsigned(h), 512 - py); + const std::array regions{{{px,py,w0,h0},{0,py,unsigned(w)-w0,h0}, + {px,0,w0,unsigned(h)-h0},{0,0,unsigned(w)-w0,unsigned(h)-h0}}}; + for (const auto& r : regions) if (r.w && r.h) { + const size_t old_uploads = p->uploads.size(); + p->uploads.erase(std::remove_if(p->uploads.begin(), p->uploads.end(), [&](const Upload& u){ return intersects(u.rect,r); }), p->uploads.end()); + p->rect_cache.erase(std::remove_if(p->rect_cache.begin(), p->rect_cache.end(), [&](const RectHash& v){ return intersects(v.rect,r); }), p->rect_cache.end()); + if (p->uploads.size() != old_uploads) p->match_cache.clear(); + else p->match_cache.erase(std::remove_if(p->match_cache.begin(), p->match_cache.end(), [&](const MatchCache& c) { + const auto& q = c.query; + const Rect page{q.page_x,q.page_y,256 / expansion(q.depth),256}; + const Rect clut{q.clut_x,q.clut_y,std::min(q.depth == 0 ? 16u : 256u,1024u-q.clut_x),1}; + return intersects(page,r) || (q.depth < 2 && intersects(clut,r)); + }), p->match_cache.end()); + } +} +void duck_texture_pack_reset_tracking(DuckTexturePack* p) { if (p) { p->uploads.clear(); p->rect_cache.clear(); p->match_cache.clear(); } } +int duck_texture_pack_copy_tracking(DuckTexturePack* destination, const DuckTexturePack* source) { + if (!destination || !source) return 0; + try { + auto uploads = source->uploads; + destination->uploads.swap(uploads); + destination->rect_cache.clear(); + destination->match_cache.clear(); + return 1; + } catch (...) { return 0; } +} +int duck_texture_pack_track_upload(DuckTexturePack* p, uint16_t x, uint16_t y, uint16_t w, uint16_t h, + const uint16_t* vram, size_t count) { + if (!p) return 0; + duck_texture_pack_invalidate(p, x, y, w, h); + if (!vram || count < kWords || !valid_rect({x,y,w,h})) return 0; + try { + if (p->uploads.size() >= kMaxUploads) p->uploads.erase(p->uploads.begin()); + p->uploads.push_back({{x,y,w,h},duck_texture_hash_rect(vram,count,x,y,w,h)}); + p->match_cache.clear(); return 1; + } catch (...) { return 0; } +} + +int duck_texture_pack_match(DuckTexturePack* p, const HdTextureDrawQuery* q, DuckTextureMatch* out) { + return duck_texture_pack_match_draw(p,q,0,out); +} +int duck_texture_pack_match_draw(DuckTexturePack* p, const HdTextureDrawQuery* q, int st, DuckTextureMatch* out) { + if (out) *out = {}; + if (!p || !q || !out) return HD_TEXTURE_LOOKUP_ERROR; + Rect wanted{}; if (!query_rect(q, wanted)) return HD_TEXTURE_LOOKUP_NONE; + try { + for (const auto& c : p->match_cache) { + const auto& a = c.query; + if (c.st == (st != 0) && a.vram == q->vram && a.page_x == q->page_x && a.page_y == q->page_y && + a.depth == q->depth && a.clut_x == q->clut_x && a.clut_y == q->clut_y && + a.u_first == q->u_first && a.u_last == q->u_last && a.v_first == q->v_first && a.v_last == q->v_last) { + *out = c.result; return c.status; + } + } + const unsigned ppw = expansion(q->depth); + size_t winner = size_t(-1); int origin_u = 0, origin_v = 0, rank = 2; + bool ambiguous = false; + const auto consider = [&](size_t index, int u, int v) { + const Entry& e = p->entries[index]; const auto& k = e.key; + const int candidate_rank = k.semitransparent == (st != 0) ? 0 : 1; + if (candidate_rank > rank) return; + bool valid = false; + if (palette_hash(*q,k.palette_min,k.palette_max,valid) != k.palette_hash || !valid) return; + if (candidate_rank < rank) { rank = candidate_rank; winner = size_t(-1); ambiguous = false; } + if (e.ambiguous || winner != size_t(-1)) ambiguous = true; + winner = index; origin_u = u; origin_v = v; + }; + for (const auto& upload : p->uploads) { + if (!contains(upload.rect,wanted)) continue; + const auto found = p->uploads_by_hash[q->depth].find(upload.hash); + if (found == p->uploads_by_hash[q->depth].end()) continue; + for (const size_t index : found->second) { + const auto& k = p->entries[index].key; + if (upload.rect.w != k.source_width_words || upload.rect.h != k.source_height) continue; + Rect rect{upload.rect.x+k.offset_x/ppw,upload.rect.y+k.offset_y,unsigned(k.width)/ppw,k.height}; + if (contains(rect,wanted)) consider(index,(int(upload.rect.x)-q->page_x)*int(ppw)+k.offset_x, + int(upload.rect.y)-q->page_y+k.offset_y); + } + } + size_t groups_checked = 0, hashed_bytes = 0; bool over_budget = false; + for (const auto& item : p->page_groups[q->depth]) { + const auto& group = item.second; const auto& r = group.texels; + Rect rect{q->page_x+r.x/ppw,q->page_y+r.y,r.w/ppw,r.h}; + if (!contains(rect,wanted)) continue; + const bool cached = std::any_of(p->rect_cache.begin(),p->rect_cache.end(),[&](const RectHash& c) { + return c.vram == q->vram && c.rect.x == rect.x && c.rect.y == rect.y && c.rect.w == rect.w && c.rect.h == rect.h; + }); + if (!cached) hashed_bytes += size_t(rect.w) * rect.h * 2; + if (++groups_checked > 4096 || hashed_bytes > 4u*1024u*1024u) { over_budget = true; break; } + const auto found = group.hashes.find(cached_hash(*p,*q,rect)); + if (found != group.hashes.end()) for (const size_t index : found->second) consider(index,r.x,r.y); + } + if (over_budget && !p->geometry_limit_reported) { + note(*p,"page candidate geometry/hash budget exceeded"); + std::fprintf(stderr,"[texture-pack] page candidate geometry/hash budget exceeded; using native texture\n"); + p->geometry_limit_reported = true; + } + int status = over_budget ? HD_TEXTURE_LOOKUP_ERROR : winner == size_t(-1) ? HD_TEXTURE_LOOKUP_NONE : + ambiguous ? HD_TEXTURE_LOOKUP_AMBIGUOUS : HD_TEXTURE_LOOKUP_FOUND; + if (status == HD_TEXTURE_LOOKUP_FOUND) { + const Entry& e = p->entries[winner]; out->key = e.key; out->entry_id = winner + 1; + out->replacement_path = e.path.c_str(); out->origin_u = origin_u; out->origin_v = origin_v; + out->source_width = e.key.width; out->source_height = e.key.height; + } + if (p->match_cache.size() >= 256) p->match_cache.erase(p->match_cache.begin()); + p->match_cache.push_back({*q,st != 0,status,*out}); + return status; + } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } +} +void duck_texture_pack_set_decode_budget(DuckTexturePack* p, size_t bytes) { + if (!p) return; + std::lock_guard lock(p->worker.mutex); + p->worker.budget = bytes; p->worker.evict(0); +} +int duck_texture_pack_request_decode(DuckTexturePack* p, uint64_t id) { + if (!p || !id || id > p->entries.size() || p->entries[id-1].ambiguous) return HD_TEXTURE_LOOKUP_ERROR; + try { + Worker& w = p->worker; std::lock_guard lock(w.mutex); + auto it = w.decoded.find(id); + if (it != w.decoded.end()) { + it->second.use = ++w.tick; + return it->second.status == DecodeStatus::Ready ? HD_TEXTURE_LOOKUP_FOUND : + it->second.status == DecodeStatus::Failed ? HD_TEXTURE_LOOKUP_ERROR : HD_TEXTURE_LOOKUP_NONE; + } + if (!w.budget) return HD_TEXTURE_LOOKUP_ERROR; + if (w.decode_queue.size() >= kMaxDecodeQueue) return HD_TEXTURE_LOOKUP_NONE; + if (w.decoded.size() >= 512) { + auto victim = w.decoded.end(); + for (auto item = w.decoded.begin(); item != w.decoded.end(); ++item) + if ((item->second.status == DecodeStatus::Ready || item->second.status == DecodeStatus::Failed) && + (victim == w.decoded.end() || item->second.use < victim->second.use)) victim = item; + if (victim == w.decoded.end()) return HD_TEXTURE_LOOKUP_NONE; + if (victim->second.image) w.used -= victim->second.image->rgba.size(); + w.decoded.erase(victim); + } + if (!w.thread.joinable()) w.thread = std::thread(worker_main,&w); + w.decoded[id].path = p->entries[id-1].path; w.decode_queue.push_back(id); + w.wake.notify_one(); return HD_TEXTURE_LOOKUP_NONE; + } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } +} +int duck_texture_pack_acquire_decoded(DuckTexturePack* p, uint64_t id, DuckTexturePixels* out) { + if (out) *out = {}; + if (!p || !out) return HD_TEXTURE_LOOKUP_ERROR; + try { + Worker& w = p->worker; std::lock_guard lock(w.mutex); const auto it = w.decoded.find(id); + if (it == w.decoded.end()) return HD_TEXTURE_LOOKUP_NONE; + if (it->second.status != DecodeStatus::Ready) return it->second.status == DecodeStatus::Failed ? HD_TEXTURE_LOOKUP_ERROR : HD_TEXTURE_LOOKUP_NONE; + auto* lease = new Lease{it->second.image}; it->second.use = ++w.tick; + out->lease = lease; out->rgba = lease->image->rgba.data(); out->width = lease->image->w; + out->height = lease->image->h; out->stride = out->width * 4; return HD_TEXTURE_LOOKUP_FOUND; + } catch (...) { return HD_TEXTURE_LOOKUP_ERROR; } +} +void duck_texture_pixels_release(DuckTexturePixels* p) { if (p) { delete static_cast(p->lease); *p = {}; } } + +int duck_texture_pack_dump_draw(DuckTexturePack* p, const HdTextureDrawQuery* q, int st, char* error, size_t capacity) { + if (!p || !q) return HD_TEXTURE_LOOKUP_ERROR; + Rect rect{}; if (!query_rect(q,rect)) return HD_TEXTURE_LOOKUP_NONE; + try { + Worker& w = p->worker; + { std::lock_guard lock(w.mutex); + if (!w.dump_error.empty()) { error_text(error,capacity,w.dump_error); return HD_TEXTURE_LOOKUP_ERROR; } + if (w.dump_queue.size() >= kMaxDumpQueue) return HD_TEXTURE_LOOKUP_NONE; } + const unsigned ppw = expansion(q->depth); const Upload* upload = nullptr; + for (const auto& u : p->uploads) if (contains(u.rect,rect)) { if (upload) { upload = nullptr; break; } upload = &u; } + DuckTextureKey k{}; k.kind = upload ? DUCK_TEXTURE_UPLOAD : DUCK_TEXTURE_PAGE; + k.depth = q->depth; k.semitransparent = st != 0; + k.source_hash = upload ? upload->hash : cached_hash(*p,*q,rect); + k.source_width_words = upload ? upload->rect.w : 256 / ppw; k.source_height = upload ? upload->rect.h : 256; + k.offset_x = (rect.x - (upload ? upload->rect.x : q->page_x)) * ppw; + k.offset_y = rect.y - (upload ? upload->rect.y : q->page_y); k.width = rect.w * ppw; k.height = rect.h; + k.palette_max = q->depth == 0 ? 15 : q->depth == 1 ? 255 : 0; + bool palette_valid = false; k.palette_hash = palette_hash(*q,0,k.palette_max,palette_valid); + if (!palette_valid) return HD_TEXTURE_LOOKUP_NONE; + const std::string name = stem(k); + if (name.empty()) return HD_TEXTURE_LOOKUP_NONE; + if (p->dumped.count(name)) return HD_TEXTURE_LOOKUP_NONE; + if (p->dumped.size() >= kMaxDumps) { + if (!p->dump_limit_reported) { + note(*p,"dump session limit (8192) reached; reset dump tracking to continue"); + std::fprintf(stderr,"[texture-pack] dump session limit (8192) reached; reset dump tracking to continue\n"); + p->dump_limit_reported = true; + } + error_text(error,capacity,"texture dump session limit (8192) reached"); + return HD_TEXTURE_LOOKUP_ERROR; + } + auto image = std::make_shared(); image->w = k.width; image->h = k.height; + image->rgba.resize(size_t(image->w) * image->h * 4); + for (unsigned y = 0; y < image->h; ++y) for (unsigned x = 0; x < image->w; ++x) { + uint16_t color = q->vram[(rect.y+y)*1024 + rect.x + x/ppw]; + if (q->depth < 2) { + const unsigned index = q->depth == 0 ? ((color >> ((x%4)*4)) & 15) : ((color >> ((x%2)*8)) & 255); + color = q->clut_x + index < 1024 ? q->vram[q->clut_y*1024+q->clut_x+index] : 0; + } + uint8_t* pixel = image->rgba.data() + (size_t(y)*image->w+x)*4; + for (unsigned c = 0; c < 3; ++c) { + const unsigned value = (color >> (c*5)) & 31; + pixel[c] = static_cast((value * 255u + 15u) / 31u); + } + pixel[3] = color == 0 ? 0 : (st && (color & 0x8000)) ? 128 : 255; + } + { std::lock_guard lock(w.mutex); + if (w.dump_queue.size() >= kMaxDumpQueue) return HD_TEXTURE_LOOKUP_NONE; + if (!w.thread.joinable()) w.thread = std::thread(worker_main,&w); + w.dump_queue.push_back({p->dumps / (name + ".png"), std::move(image)}); + p->dumped.insert(name); + w.wake.notify_one(); } + return HD_TEXTURE_LOOKUP_FOUND; + } catch (const std::exception& e) { error_text(error,capacity,e.what()); return HD_TEXTURE_LOOKUP_ERROR; } + catch (...) { error_text(error,capacity,"texture dump allocation failed"); return HD_TEXTURE_LOOKUP_ERROR; } +} +void duck_texture_pack_reset_dump(DuckTexturePack* p) { + if (!p) return; + p->dumped.clear(); + p->dump_limit_reported = false; + std::lock_guard lock(p->worker.mutex); + p->worker.dump_error.clear(); +} +} // extern C diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index dbd827d0..f351bfb2 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -5712,6 +5712,7 @@ static void gp0_exec_cpu_to_vram(void) { /* 0 means max dimension */ vram_write_w = (w == 0) ? 0x400 : (uint16_t)w; vram_write_h = (h == 0) ? 0x200 : (uint16_t)h; + gr_vram_upload_begin(vram_write_x, vram_write_y, vram_write_w, vram_write_h); /* A full history retains old uploads; later transfers must not append * to the last slot and overflow its diagnostic word counter. */ diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index 09fca3e0..e9c56bec 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -78,6 +78,7 @@ #include "gl_scale_limits.h" #include "gpu_sw_renderer.h" #include "gpu_gl_renderer.h" +#include "gpu_hd_textures.h" #include "mod_texture_banks.h" #include "frame_interpolation.h" #include "render_pass_plan.h" @@ -371,6 +372,8 @@ static int s_hr_scale = 1; #endif /* Netplay: SW@1× + GPU@s_out_scale dual write; CPU VRAM always authoritative. */ static int s_cpu_auth_dual = 0; +/* Independent of netplay: the replacement surface must never become VRAM. */ +static int s_hd_native_authority = 0; static int s_req_scale = 1; /* requested before context init */ /* Driver limits queried at init: min(GL_MAX_TEXTURE_SIZE, * GL_MAX_RENDERBUFFER_SIZE, GL_MAX_VIEWPORT_DIMS); 0 before init. */ @@ -567,7 +570,7 @@ static GLuint s_tex_prog = 0, s_tex_vao = 0, s_tex_vbo = 0; * fragment shader read the noperspective varying — i.e. bit-identical to the * pre-feature pipeline. twin is the prim's GP0(E2h) texture window, its low 20 * bits as a whole float (mask x, mask y, offset x, offset y; 5 bits each). */ -#define TEXV 21 +#define TEXV 26 static GLuint s_blit_prog = 0, s_blit_vao = 0, s_blit_vbo = 0; static GLuint s_blit_hi_prog = 0; /* windowed hi surface blit */ static GLint s_uBhSrc = -1, s_uBhPass = -1, s_uBhMaskset = -1, s_uBhSrcDiv = -1; @@ -591,6 +594,7 @@ static float s_pq[3]; /* TEX program uniforms. */ static GLint s_uVram = -1, s_uTpage = -1, s_uClut = -1, s_uDepth = -1; +static GLint s_uHdTexture = -1; static GLint s_uPalette = -1; static GLint s_uRaw = -1, s_uSemipass = -1, s_uSemimode = -1; static GLint s_uMaskset = -1, s_uFilter = -1; @@ -1326,6 +1330,8 @@ static const char *TEX_VS = "layout(location=8) in float a_semi;\n" "layout(location=9) in float a_q; /* persp weight; 0 = affine (default) */\n" "layout(location=10) in float a_twin; /* GP0(E2h) bits 0..19 */\n" + "layout(location=11) in vec4 a_hd_source; /* page origin + native extent */\n" + "layout(location=12) in float a_hd_mode;\n" "uniform float u_shift;\n" "uniform float u_xoff; /* native-wide x translation (px); 0 canonical */\n" "uniform float u_xhalf; /* x clip half-extent (px); 512 canonical */\n" @@ -1337,12 +1343,14 @@ static const char *TEX_VS = "flat out ivec2 v_tpage; flat out ivec2 v_clut; flat out int v_depth;\n" "flat out int v_raw; flat out ivec4 v_limits; flat out int v_semi;\n" "flat out int v_twin;\n" + "flat out vec4 v_hd_source; flat out int v_hd_mode;\n" "void main(){ v_uv = a_uv; v_uv_p = a_uv; v_col = a_col;\n" " v_persp = (a_q > 0.0) ? 1 : 0;\n" " v_tpage = ivec2(a_tpage + 0.5); v_clut = ivec2(a_clut + 0.5);\n" " v_depth = int(a_depth + 0.5); v_raw = int(a_raw + 0.5);\n" " v_semi = int(a_semi + 0.5);\n" " v_twin = int(a_twin + 0.5);\n" + " v_hd_source=a_hd_source; v_hd_mode=int(a_hd_mode+0.5);\n" " v_limits = ivec4(floor(a_limits + 0.5));\n" " /* u_shift: align GL's center-sample grid with the PS1 integer grid (see\n" " * GEO_VS) so interpolated uv at a fragment equals the PS1 DDA value. */\n" @@ -1371,6 +1379,8 @@ static const char *TEX_FS = "flat in ivec4 v_limits; /* prim uv sampling bounds (inclusive, post-wrap) */\n" "flat in int v_semi; /* GP0 command has semi-transparency enabled */\n" "flat in int v_twin; /* texture window, GP0(E2h) bits 0..19 */\n" + "flat in vec4 v_hd_source; flat in int v_hd_mode;\n" + "uniform sampler2D u_hd_texture;\n" "uniform usampler2D u_vram;\n" "uniform usampler2D u_palette;\n" "uniform int u_semipass; /* 0=all texels, 1=STP=0 only, 2=STP=1 only */\n" @@ -1427,6 +1437,17 @@ static const char *TEX_FS = " }\n" " return sum;\n" "}\n" + "vec4 hd_texel(vec2 uv){\n" + " ivec2 p=ivec2(floor(uv)); vec2 f=fract(uv);\n" + " p &= ivec2(255);\n" + " ivec4 tw=ivec4(v_twin&31,(v_twin>>5)&31,(v_twin>>10)&31,(v_twin>>15)&31);\n" + " if((tw.x|tw.y)!=0) p=(p & ~(tw.xy*8)) | ((tw.zw & tw.xy)*8);\n" + " else p=clamp(p,v_limits.xy,v_limits.zw);\n" + " vec2 t=(vec2(p)+f-v_hd_source.xy)/v_hd_source.zw;\n" + " ivec2 size=textureSize(u_hd_texture,0);\n" + " ivec2 at=clamp(ivec2(floor(t*vec2(size))),ivec2(0),size-ivec2(1));\n" + " return texelFetch(u_hd_texture,at,0);\n" + "}\n" "void main(){\n" " int stp; vec3 rgb;\n" " /* v_persp is 0 for every prim unless [video] perspective_texturing is on\n" @@ -1434,7 +1455,20 @@ static const char *TEX_FS = " * default is the PS1's affine (noperspective) mapping. */\n" " vec2 uv = (v_persp != 0) ? v_uv_p : v_uv;\n" " vec2 dx=dFdx(uv), dy=dFdy(uv);\n" - " if (u_filter == 0) {\n" + " if (v_hd_mode != 0) {\n" + " vec4 hd=hd_texel(uv);\n" + " rgb=hd.rgb;\n" + " if(v_hd_mode==1){\n" + " if(hd.a==0.0) discard;\n" + " int native=fetch_texel(int(floor(uv.x)),int(floor(uv.y)));\n" + " if(native==0) discard; stp=(native>>15)&1;\n" + " } else if(v_hd_mode==2){\n" + " if(hd.a<128.0/255.0) discard; stp=0;\n" + " } else {\n" + " stp=hd.a<=242.0/255.0 ? 1 : 0;\n" + " if(all(equal(hd,vec4(0.0))) || (stp==0 && all(equal(hd.rgb,vec3(0.0))))) discard;\n" + " }\n" + " } else if (u_filter == 0) {\n" " int raw = fetch_texel(int(floor(uv.x)), int(floor(uv.y)));\n" " if (raw == 0) discard;\n" " rgb = col5(raw);\n" @@ -2266,6 +2300,18 @@ static void pack_flush(void) { rect_clear(&s_pack_dirty); coh_record(GL_COH_PACK, x, y, x + w - 1, y + h - 1); + if (s_hd_native_authority) { + /* All pending draws consumed the previous raw version before this + * refresh. Never encode the HD presentation image into native words. */ + p_glActiveTexture(PSXGL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, s_raw_tex); + glPixelStorei(PSXGL_UNPACK_ROW_LENGTH, VRAM_W); + glTexSubImage2D(GL_TEXTURE_2D, 0, x, y, w, h, PSXGL_RED_INTEGER, + GL_UNSIGNED_SHORT, s_vram + (size_t)y * VRAM_W + x); + glPixelStorei(PSXGL_UNPACK_ROW_LENGTH, 0); + return; + } + p_glBindFramebuffer(PSXGL_FRAMEBUFFER, s_raw_fbo); glViewport(0, 0, VRAM_W, VRAM_H); glEnable(GL_SCISSOR_TEST); @@ -2312,7 +2358,7 @@ static void flush_pack_if_sampling(int tpage_x, int tpage_y, int depth, static int s_depth24_skip_up = 0; static DirtyRect s_d24_skip_fb; /* union of skipped MDEC FB rects (VRAM halfwords) */ static int cpu_vram_authoritative(void) { - return s_cpu_auth_dual || s_depth24_skip_up; + return s_cpu_auth_dual || s_hd_native_authority || s_depth24_skip_up; } static void ensure_cpu(void) { @@ -2575,6 +2621,89 @@ static int s_tb_n = 0; /* verts queued */ static int s_tb_semi = -2; static int s_tb_mask = 0, s_tb_filter = 0; static GLuint s_tb_bank_tex; +static GLuint s_tb_hd_tex; + +#define HD_GL_CACHE_CAP 64 +#define HD_GL_CACHE_BUDGET (128u * 1024u * 1024u) +typedef struct HdGlTexture { + uint64_t generation, key; + uint64_t last_use; + GLuint texture; + size_t bytes; +} HdGlTexture; +static HdGlTexture s_hd_gl_cache[HD_GL_CACHE_CAP]; +static size_t s_hd_gl_cache_bytes; +static uint64_t s_hd_gl_cache_clock; + +void gl_renderer_clear_hd_texture_cache(void) { + flush_flat_batch(); flush_tex_batch(); hiw_flush_queue(); + for (int i = 0; i < HD_GL_CACHE_CAP; ++i) { + if (s_ctx && s_hd_gl_cache[i].texture) + glDeleteTextures(1, &s_hd_gl_cache[i].texture); + } + memset(s_hd_gl_cache, 0, sizeof(s_hd_gl_cache)); + s_hd_gl_cache_bytes = 0; s_tb_hd_tex = 0; +} +static GLuint hd_gl_texture(const GpuHdTextureImage* image) { + if (!image->rgba || !image->width || !image->height || !image->source_width || + !image->source_height || image->stride != (uint64_t)image->width * 4u || + image->width > (uint32_t)s_gl_max_dim || image->height > (uint32_t)s_gl_max_dim) + return 0; + size_t bytes = (size_t)image->stride * image->height; + if (bytes > HD_GL_CACHE_BUDGET) return 0; + int free_slot = -1; + for (int i = 0; i < HD_GL_CACHE_CAP; ++i) { + HdGlTexture* texture = &s_hd_gl_cache[i]; + if (texture->texture && texture->key == image->cache_key && + texture->generation == image->generation) { + texture->last_use = ++s_hd_gl_cache_clock; return texture->texture; + } + if (!texture->texture && free_slot < 0) free_slot = i; + } + while (free_slot < 0 || s_hd_gl_cache_bytes + bytes > HD_GL_CACHE_BUDGET) { + int oldest = -1; + for (int i = 0; i < HD_GL_CACHE_CAP; ++i) + if (s_hd_gl_cache[i].texture && (oldest < 0 || + s_hd_gl_cache[i].last_use < s_hd_gl_cache[oldest].last_use)) oldest = i; + if (oldest < 0) return 0; + flush_flat_batch(); flush_tex_batch(); hiw_flush_queue(); + HdGlTexture* victim = &s_hd_gl_cache[oldest]; + glDeleteTextures(1, &victim->texture); + s_hd_gl_cache_bytes -= victim->bytes; + memset(victim, 0, sizeof(*victim)); free_slot = oldest; + } + /* A newly allocated image cannot affect pending draws, but TextureImage + * changes GL bindings; their flush always binds its own complete state. */ + HdGlTexture* texture = &s_hd_gl_cache[free_slot]; + glGenTextures(1, &texture->texture); + p_glActiveTexture(PSXGL_TEXTURE0 + 2); + glBindTexture(GL_TEXTURE_2D, texture->texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glPixelStorei(PSXGL_UNPACK_ROW_LENGTH, 0); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, image->width, image->height, 0, + GL_RGBA, GL_UNSIGNED_BYTE, image->rgba); + p_glActiveTexture(PSXGL_TEXTURE0); + if (glGetError() != GL_NO_ERROR) { + glDeleteTextures(1, &texture->texture); texture->texture = 0; return 0; + } + texture->key = image->cache_key; texture->generation = image->generation; + texture->last_use = ++s_hd_gl_cache_clock; + texture->bytes = bytes; s_hd_gl_cache_bytes += bytes; + return texture->texture; +} + +static void bind_textured_resources(GLuint source, GLuint palette, GLuint hd) { + p_glActiveTexture(PSXGL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, source); p_glUniform1i(s_uVram, 0); + p_glActiveTexture(PSXGL_TEXTURE0 + 1); + glBindTexture(GL_TEXTURE_2D, palette); p_glUniform1i(s_uPalette, 1); + p_glActiveTexture(PSXGL_TEXTURE0 + 2); + glBindTexture(GL_TEXTURE_2D, hd ? hd : s_up_tex); p_glUniform1i(s_uHdTexture, 2); + p_glActiveTexture(PSXGL_TEXTURE0); +} static int s_tb_twin[4] = {0, 0, 0, 0}; static uint64_t s_batch_total = 0, s_batch_reason[7]; @@ -2729,6 +2858,7 @@ typedef struct { uint8_t check; /* mask-check state at draw time */ uint8_t filter; GLuint tex; /* sampled texture (bank or raw mirror) */ + GLuint palette, hd_tex; int ax0, ay0, ax1, ay1; /* draw area, inclusive */ size_t vfirst; /* float offset into s_hq_v */ int vcount; /* vertices */ @@ -2801,6 +2931,8 @@ static int hiw_enqueue_tex(int nverts, int semi, int mirror, int gate) { c->mask = (uint8_t)s_tb_mask; c->filter = (uint8_t)s_tb_filter; c->tex = s_tb_bank_tex ? s_tb_bank_tex : s_raw_tex; + c->palette = s_tb_bank_tex && !s_tb_bank_live_clut ? s_tb_bank_tex : s_raw_tex; + c->hd_tex = s_tb_hd_tex; if (wq) hiw_wide_set(c, gate); return wq; } @@ -2857,7 +2989,7 @@ static void hiw_replay_wide(void) { p_glUniform1f(s_tex_uXoff, (float)c->wdx); p_glUniform1f(s_tex_uXscale, c->wscale); p_glUniform1f(s_tex_uXcenter, c->wcenter); - glBindTexture(GL_TEXTURE_2D, c->tex); + bind_textured_resources(c->tex, c->palette, c->hd_tex); p_glUniform1i(s_uMaskset, c->mask); p_glUniform1i(s_uFilter, c->filter); p_glBufferData(PSXGL_ARRAY_BUFFER, (ptrdiff_t)((size_t)c->vcount * TEXV * sizeof(float)), @@ -2933,7 +3065,7 @@ static void hiw_flush_queue(void) { p_glBindBuffer(PSXGL_ARRAY_BUFFER, s_tex_vbo); cur = HQ_TEX; } - glBindTexture(GL_TEXTURE_2D, c->tex); + bind_textured_resources(c->tex, c->palette, c->hd_tex); p_glUniform1i(s_uMaskset, c->mask); p_glUniform1i(s_uFilter, c->filter); p_glBufferData(PSXGL_ARRAY_BUFFER, (ptrdiff_t)((size_t)c->vcount * TEXV * sizeof(float)), @@ -2978,13 +3110,8 @@ static void flush_tex_batch(void) { hr_begin(1); p_glUseProgram(s_tex_prog); - p_glActiveTexture(PSXGL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, s_tb_bank_tex ? s_tb_bank_tex : s_raw_tex); - p_glUniform1i(s_uVram, 0); - p_glActiveTexture(PSXGL_TEXTURE0 + 1); - glBindTexture(GL_TEXTURE_2D, s_tb_bank_tex && !s_tb_bank_live_clut ? s_tb_bank_tex : s_raw_tex); - p_glUniform1i(s_uPalette, 1); - p_glActiveTexture(PSXGL_TEXTURE0); + bind_textured_resources(s_tb_bank_tex ? s_tb_bank_tex : s_raw_tex, + s_tb_bank_tex && !s_tb_bank_live_clut ? s_tb_bank_tex : s_raw_tex, s_tb_hd_tex); p_glUniform1i(s_uMaskset, s_tb_mask); p_glUniform1i(s_uFilter, s_tb_filter); p_glBindVertexArray(s_tex_vao); @@ -3386,6 +3513,18 @@ static void gpu_textured_triangle(const int *xs, const int *ys, flush_pack_if_sampling(base_x, base_y, depth, clut_x, clut_y); else if (s_selected_bank_live_clut) flush_pack_if_sampling(-1, 0, depth, clut_x, clut_y); + GpuHdTextureImage hd = {0}; + GLuint hd_tex = 0; + if (s_hd_native_authority && !s_selected_bank_tex) { + uint32_t window = (uint32_t)(s_tw_mask_x | (s_tw_mask_y << 5) | + (s_tw_off_x << 10) | (s_tw_off_y << 15)); + if (gpu_hd_textures_acquire_draw(texpage, clut_x, clut_y, lim, window, + semi >= 0, &hd)) { + hd_tex = hd_gl_texture(&hd); + if (!hd_tex) hd.alpha_mode = 0; + else gpu_hd_textures_note_applied(); + } + } mark_prim_dirty(xs, ys, 3, 1 /* textured */); /* Append to the textured batch. Flush first if this prim's blend/mask/twin/ @@ -3426,7 +3565,7 @@ static void gpu_textured_triangle(const int *xs, const int *ys, !mod_texture_bank_batchable(s_selected_bank_tex != 0, s_mask_check, semi); int reason = -1; if (s_tb_n > 0) { - if (s_tb_bank_tex != s_selected_bank_tex || s_tb_bank_live_clut != s_selected_bank_live_clut) reason = 0; + if (s_tb_bank_tex != s_selected_bank_tex || s_tb_bank_live_clut != s_selected_bank_live_clut || s_tb_hd_tex != hd_tex) reason = 0; else if (isolate) reason = 0; else if (batch_semi != s_tb_semi) reason = 1; else if (s_mask_set != s_tb_mask) reason = 2; @@ -3445,6 +3584,7 @@ static void gpu_textured_triangle(const int *xs, const int *ys, s_tb_semi = batch_semi; s_tb_mask = s_mask_set; s_tb_filter = filter; s_tb_gate = gate; s_tb_bank_tex = s_selected_bank_tex; s_tb_bank_live_clut = s_selected_bank_live_clut; + s_tb_hd_tex = hd_tex; s_tb_twin[0] = twx; s_tb_twin[1] = twy; s_tb_twin[2] = tox; s_tb_twin[3] = toy; } /* The prim's window, exact as a float (20 bits). */ @@ -3464,10 +3604,14 @@ static void gpu_textured_triangle(const int *xs, const int *ys, vp[18] = semi >= 0 ? (float)(semi + 1) : 0.0f; /* a_semi code */ vp[19] = s_pq_valid ? s_pq[i] : 0.0f; /* a_q; 0 = affine */ vp[20] = twin; /* a_twin */ + vp[21] = (float)hd.origin_u; vp[22] = (float)hd.origin_v; + vp[23] = (float)hd.source_width; vp[24] = (float)hd.source_height; + vp[25] = (float)hd.alpha_mode; } s_tb_n += 3; if (isolate) flush_tex_batch(); /* draw this semi prim alone, in submission order */ } + gpu_hd_textures_release_image(&hd); } /* Draw a flat-colored rect (GEO program) DIRECTLY into the active wide surface @@ -3780,8 +3924,17 @@ static int cpu_raster_required(void) { * GPU surface. The CPU mirror stays the authoritative record of the real * frame, so the pass never rasterizes into it. depth24 scanout still * presents from the CPU mirror and keeps its write-through. */ - return !s_raster_ok || s_depth24_skip_up || (s_cpu_auth_dual && !s_pass_active); + return !s_raster_ok || s_depth24_skip_up || + ((s_cpu_auth_dual || s_hd_native_authority) && !s_pass_active); } +/* Bracket each authoritative draw so a later netplay/session policy change + * cannot accidentally disable the faithful floor for HD replacement mode. */ +static int native_draw_begin(void) { + int previous = sw_faithful_authority(); + if (s_hd_native_authority) sw_set_faithful_authority(1); + return previous; +} +static void native_draw_end(int previous) { sw_set_faithful_authority(previous); } /* The sub-pixel / perspective override describes exactly one triangle; drop it * once that triangle has been submitted so a later prim can never inherit it. */ static inline void precise_consumed(void) { s_pc_valid = 0; s_pq_valid = 0; s_projected_uv_valid = 0; } @@ -3824,80 +3977,127 @@ void gl_renderer_draw_projected_triangle(const PSXProjectedVertex vertices[3], } static void glb_draw_flat_triangle(int x0,int y0,int x1,int y1,int x2,int y2,uint16_t col) { - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_draw_flat_triangle(x0,y0,x1,y1,x2,y2,col); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_triangle(x0,y0,col, x1,y1,col, x2,y2,col, s_semi_en?s_semi_mode:-1); precise_consumed(); } static void glb_draw_gouraud_triangle(int x0,int y0,uint16_t c0,int x1,int y1,uint16_t c1,int x2,int y2,uint16_t c2) { - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_draw_gouraud_triangle(x0,y0,c0,x1,y1,c1,x2,y2,c2); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_triangle(x0,y0,c0, x1,y1,c1, x2,y2,c2, s_semi_en?s_semi_mode:-1); precise_consumed(); } static void glb_fill_rect(int x,int y,int w,int h,uint16_t c){ if (pass_refuse_write("fill", x, y, w, h)) return; - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_fill_rect(x,y,w,h,c); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_fill(x,y,w,h,c); } static void glb_copy_rect(int sx,int sy,int dx,int dy,int w,int h){ if (pass_refuse_write("copy", dx, dy, w, h)) return; - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_copy_rect(sx,sy,dx,dy,w,h); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_copy_rect(sx,sy,dx,dy,w,h); } static void glb_draw_textured_triangle(int x0,int y0,int u0,int v0,int x1,int y1,int u1,int v1,int x2,int y2,int u2,int v2,uint16_t cx,uint16_t cy,uint16_t tp){ - if (cpu_raster_required()) + const int native_required = cpu_raster_required(); + if (native_required && !s_hd_native_authority) sw_draw_textured_triangle(x0,y0,u0,v0,x1,y1,u1,v1,x2,y2,u2,v2,cx,cy,tp); - if (!s_raster_ok) return; - int xs[3]={x0,x1,x2}, ys[3]={y0,y1,y2}, us[3]={u0,u1,u2}, vs[3]={v0,v1,v2}; - float mr=s_mod_r/255.0f, mg=s_mod_g/255.0f, mb=s_mod_b/255.0f; - float col[9]={mr,mg,mb, mr,mg,mb, mr,mg,mb}; - gpu_textured_triangle(xs,ys,us,vs,col,tp,cx,cy,s_mod_raw, s_semi_en?s_semi_mode:-1, NULL); + if (s_raster_ok) { + int xs[3]={x0,x1,x2}, ys[3]={y0,y1,y2}, us[3]={u0,u1,u2}, vs[3]={v0,v1,v2}; + float mr=s_mod_r/255.0f, mg=s_mod_g/255.0f, mb=s_mod_b/255.0f; + float col[9]={mr,mg,mb, mr,mg,mb, mr,mg,mb}; + gpu_textured_triangle(xs,ys,us,vs,col,tp,cx,cy,s_mod_raw, s_semi_en?s_semi_mode:-1, NULL); + } + if (native_required && s_hd_native_authority) { + int faithful = native_draw_begin(); + sw_draw_textured_triangle(x0,y0,u0,v0,x1,y1,u1,v1,x2,y2,u2,v2,cx,cy,tp); + native_draw_end(faithful); + } precise_consumed(); } static void glb_draw_shaded_textured_triangle(int x0,int y0,int u0,int v0,uint32_t c0,int x1,int y1,int u1,int v1,uint32_t c1,int x2,int y2,int u2,int v2,uint32_t c2,uint16_t cx,uint16_t cy,uint16_t tp,int raw){ - if (cpu_raster_required()) + const int native_required = cpu_raster_required(); + if (native_required && !s_hd_native_authority) sw_draw_shaded_textured_triangle(x0,y0,u0,v0,c0,x1,y1,u1,v1,c1,x2,y2,u2,v2,c2,cx,cy,tp,raw); - if (!s_raster_ok) return; - int xs[3]={x0,x1,x2}, ys[3]={y0,y1,y2}, us[3]={u0,u1,u2}, vs[3]={v0,v1,v2}; - uint32_t cc[3]={c0,c1,c2}; float col[9]; - for (int i=0;i<3;i++){ col[i*3+0]=(cc[i]&0xFF)/255.0f; col[i*3+1]=((cc[i]>>8)&0xFF)/255.0f; col[i*3+2]=((cc[i]>>16)&0xFF)/255.0f; } - gpu_textured_triangle(xs,ys,us,vs,col,tp,cx,cy,raw, s_semi_en?s_semi_mode:-1, NULL); + if (s_raster_ok) { + int xs[3]={x0,x1,x2}, ys[3]={y0,y1,y2}, us[3]={u0,u1,u2}, vs[3]={v0,v1,v2}; + uint32_t cc[3]={c0,c1,c2}; float col[9]; + for (int i=0;i<3;i++){ col[i*3+0]=(cc[i]&0xFF)/255.0f; col[i*3+1]=((cc[i]>>8)&0xFF)/255.0f; col[i*3+2]=((cc[i]>>16)&0xFF)/255.0f; } + gpu_textured_triangle(xs,ys,us,vs,col,tp,cx,cy,raw, s_semi_en?s_semi_mode:-1, NULL); + } + if (native_required && s_hd_native_authority) { + int faithful = native_draw_begin(); + sw_draw_shaded_textured_triangle(x0,y0,u0,v0,c0,x1,y1,u1,v1,c1,x2,y2,u2,v2,c2,cx,cy,tp,raw); + native_draw_end(faithful); + } precise_consumed(); } static void glb_draw_flat_rect(int x,int y,int w,int h,uint16_t c){ - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_draw_flat_rect(x,y,w,h,c); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_flat_rect(x,y,w,h,c, s_semi_en?s_semi_mode:-1); } static void glb_draw_textured_rect(int x,int y,int w,int h,int u,int v,uint16_t cx,uint16_t cy,uint16_t tp){ - if (cpu_raster_required()) + const int native_required = cpu_raster_required(); + if (native_required && !s_hd_native_authority) sw_draw_textured_rect(x,y,w,h,u,v,cx,cy,tp); - if (!s_raster_ok) return; - gpu_textured_rect(x,y,w,h, u,v, u+w,v+h, cx,cy,tp, s_semi_en?s_semi_mode:-1); + if (s_raster_ok) + gpu_textured_rect(x,y,w,h, u,v, u+w,v+h, cx,cy,tp, s_semi_en?s_semi_mode:-1); + if (native_required && s_hd_native_authority) { + int faithful = native_draw_begin(); + sw_draw_textured_rect(x,y,w,h,u,v,cx,cy,tp); + native_draw_end(faithful); + } } static void glb_draw_textured_rect_scaled(int x,int y,int w,int h,int u0,int v0,int u1,int v1,uint16_t cx,uint16_t cy,uint16_t tp){ - if (cpu_raster_required()) + const int native_required = cpu_raster_required(); + if (native_required && !s_hd_native_authority) sw_draw_textured_rect_scaled(x,y,w,h,u0,v0,u1,v1,cx,cy,tp); - if (!s_raster_ok) return; - gpu_textured_rect(x,y,w,h, u0,v0, u1,v1, cx,cy,tp, s_semi_en?s_semi_mode:-1); + if (s_raster_ok) + gpu_textured_rect(x,y,w,h, u0,v0, u1,v1, cx,cy,tp, s_semi_en?s_semi_mode:-1); + if (native_required && s_hd_native_authority) { + int faithful = native_draw_begin(); + sw_draw_textured_rect_scaled(x,y,w,h,u0,v0,u1,v1,cx,cy,tp); + native_draw_end(faithful); + } } static void glb_draw_line(int x0,int y0,int x1,int y1,uint16_t c){ - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_draw_line(x0,y0,x1,y1,c); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_line(x0,y0,c, x1,y1,c, s_semi_en?s_semi_mode:-1); } static void glb_draw_shaded_line(int x0,int y0,uint16_t c0,int x1,int y1,uint16_t c1){ - if (cpu_raster_required()) + if (cpu_raster_required()) { + int faithful = native_draw_begin(); sw_draw_shaded_line(x0,y0,c0,x1,y1,c1); + native_draw_end(faithful); + } if (!s_raster_ok) return; gpu_line(x0,y0,c0, x1,y1,c1, s_semi_en?s_semi_mode:-1); } @@ -4010,7 +4210,7 @@ static void depth24_clear_skipped_fb(void) { glDisable(GL_SCISSOR_TEST); p_glBindFramebuffer(PSXGL_FRAMEBUFFER, 0); rect_add(&s_pack_dirty, x0, y0, x1, y1); - if (!s_cpu_auth_dual) { + if (!cpu_vram_authoritative()) { rect_add(&s_cpu_dirty, x0, y0, x1, y1); s_gpu_dirty = 1; /* The clear must reach an immediate CPU read too. */ } @@ -4446,6 +4646,7 @@ static int init_gpu_raster(void) { s_uMaskset = p_glGetUniformLocation(s_tex_prog, "u_maskset"); s_uFilter = p_glGetUniformLocation(s_tex_prog, "u_filter"); s_uLimits = p_glGetUniformLocation(s_tex_prog, "u_limits"); + s_uHdTexture = p_glGetUniformLocation(s_tex_prog, "u_hd_texture"); s_uBlitSrc = p_glGetUniformLocation(s_blit_prog, "u_src"); s_uBlitPass = p_glGetUniformLocation(s_blit_prog, "u_stp_pass"); s_uBlitMaskset = p_glGetUniformLocation(s_blit_prog, "u_maskset"); @@ -4536,6 +4737,8 @@ static int init_gpu_raster(void) { p_glVertexAttribPointer(8, 1, GL_FLOAT, GL_FALSE, st, (void*)(18*sizeof(float))); p_glEnableVertexAttribArray(8); /* semi */ p_glVertexAttribPointer(9, 1, GL_FLOAT, GL_FALSE, st, (void*)(19*sizeof(float))); p_glEnableVertexAttribArray(9); /* q */ p_glVertexAttribPointer(10, 1, GL_FLOAT, GL_FALSE, st, (void*)(20*sizeof(float))); p_glEnableVertexAttribArray(10); /* twin */ + p_glVertexAttribPointer(11, 4, GL_FLOAT, GL_FALSE, st, (void*)(21*sizeof(float))); p_glEnableVertexAttribArray(11); /* HD source */ + p_glVertexAttribPointer(12, 1, GL_FLOAT, GL_FALSE, st, (void*)(25*sizeof(float))); p_glEnableVertexAttribArray(12); /* HD alpha mode */ } p_glGenVertexArrays(1, &s_blit_vao); @@ -4589,7 +4792,7 @@ static int init_gpu_raster(void) { return 1; } -int gl_renderer_texture_banks_supported(void) { return s_raster_ok && !s_cpu_auth_dual; } +int gl_renderer_texture_banks_supported(void) { return s_raster_ok && !s_cpu_auth_dual && !s_hd_native_authority; } int gl_renderer_fit_wide_aspect(int disp_w, int *num, int *den) { if (!s_raster_ok || s_gl_max_dim <= 0) return 0; @@ -4851,6 +5054,7 @@ static void pass_resources_release(void); void gl_renderer_shutdown(void) { s_native_surface_enabled=s_native_surface_pending=0; + gl_renderer_clear_hd_texture_cache(); pass_resources_release(); if (s_ctx) { for (unsigned i = 1; i < 65536u; ++i) @@ -5018,6 +5222,7 @@ void gl_renderer_invalidate_present(void) { } void gl_renderer_restage_vram_after_savestate(void) { + gpu_hd_textures_reset_tracking(); if (!s_raster_ok || !s_vram) return; /* Belt-and-suspenders after boot_state VRAM apply: force CPU mirror → FBO * even if a depth24 skip swallowed the restore, then re-arm scanout-band @@ -5032,7 +5237,7 @@ void gl_renderer_restage_vram_after_savestate(void) { depth24_mark_scanout_band(); } /* Dual-raster: CPU is authority after snap; FBO is cosmetics only. */ - if (s_cpu_auth_dual) { + if (s_cpu_auth_dual || s_hd_native_authority) { s_gpu_dirty = 0; rect_clear(&s_cpu_dirty); } @@ -5046,6 +5251,23 @@ void gl_renderer_set_cpu_auth_dual(int on) { } } +void gl_renderer_set_hd_texture_mode(int on) { + on = on ? 1 : 0; + if (on == s_hd_native_authority) return; + if (s_raster_ok) { + flush_flat_batch(); flush_tex_batch(); flush_cpu_upload(); hiw_flush_queue(); + if (on) ensure_cpu(); + else { + /* Return GPU authority only after replacing the presentation + * image with native VRAM; an HD frame must never be read back. */ + gl_renderer_restage_vram_after_savestate(); + } + } + s_hd_native_authority = on; + s_gpu_dirty = 0; rect_clear(&s_cpu_dirty); + if (on) s_selected_bank_tex = 0; +} + int gl_renderer_cpu_auth_dual(void) { return s_cpu_auth_dual; } @@ -6172,7 +6394,7 @@ uint32_t gl_renderer_pass_unavailable(void) { s_interp_source != 1 || !(s_interp_source_hz > 0.0)) return PSX_MOD_RENDER_PASS_NO_PRESENTER; /* Dual raster (netplay CPU-authoritative VRAM) or a debug refusal. */ - if ((s_cpu_auth_dual && !render_pass_netplay_enabled()) || s_pass_force_refuse) + if (s_hd_native_authority || (s_cpu_auth_dual && !render_pass_netplay_enabled()) || s_pass_force_refuse) return PSX_MOD_RENDER_PASS_BACKEND; /* Windowed high-resolution mode: the presented surfaces are the window * tiles at s_out_scale, which a pass (backing up s_hr_fbo at @@ -6711,7 +6933,7 @@ uint32_t gl_renderer_stereo_unavailable(void) { pass_refusal_init(); if (!s_ctx || !s_raster_ok || !s_hr_fbo || gpu_display_is_depth24()) return PSX_MOD_RENDER_PASS_NO_PRESENTER; - if (s_cpu_auth_dual || s_hiw || g_wide_w > 0 || s_pass_force_refuse > 0) + if (s_cpu_auth_dual || s_hd_native_authority || s_hiw || g_wide_w > 0 || s_pass_force_refuse > 0) return PSX_MOD_RENDER_PASS_BACKEND; return PSX_MOD_RENDER_PASS_READY; } diff --git a/runtime/src/gpu_hd_textures.cpp b/runtime/src/gpu_hd_textures.cpp new file mode 100644 index 00000000..190b70ac --- /dev/null +++ b/runtime/src/gpu_hd_textures.cpp @@ -0,0 +1,272 @@ +#include "gpu_hd_textures.h" +#include "gpu_gl_renderer.h" +#include "hd_texture_pack.h" +#include "duckstation_texture_pack.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr size_t kVramWords = 1024u * 512u; +struct Session { + std::string root; + HdTexturePack* beetle = nullptr; + DuckTexturePack* duck = nullptr; + bool replacements = false; + bool dump = false; + bool dump_failed = false; + GpuHdTextureDiag diag{}; + ~Session() { hd_texture_pack_destroy(beetle); duck_texture_pack_destroy(duck); } +}; +struct Lease { + HdTexturePixels beetle{}; + DuckTexturePixels duck{}; + ~Lease() { hd_texture_pixels_release(&beetle); duck_texture_pixels_release(&duck); } +}; +std::unique_ptr session; +const uint16_t* native_vram = nullptr; +uint64_t generation = 1; + +void fail(char* error, size_t capacity, const char* message) { + if (error && capacity) std::snprintf(error, capacity, "%s", message); +} +bool is_beetle_root(const std::filesystem::path& root) { + std::error_code error; + if (std::filesystem::exists(root / "Hashes.ini", error)) return true; + if (std::filesystem::exists(root.parent_path() / "Hashes.ini", error)) return true; + const auto name = root.filename().u8string(); + const std::string suffix = "-texture-replacements"; + return name.size() >= suffix.size() && + name.compare(name.size() - suffix.size(), suffix.size(), suffix) == 0; +} +int window_coord(int value, int mask, int offset) { + return (value & ~(mask * 8)) | ((offset & mask) * 8); +} +bool make_query(uint16_t texpage, uint16_t cx, uint16_t cy, const int limits[4], + uint32_t window, HdTextureDrawQuery& query) { + if (!native_vram || !limits || ((texpage >> 7) & 3u) == 3u) return false; + if (limits[0] < 0 || limits[1] < 0 || limits[2] > 255 || limits[3] > 255 || + limits[0] > limits[2] || limits[1] > limits[3]) return false; + int first[2] = {255,255}, last[2] = {0,0}; + for (int axis = 0; axis < 2; ++axis) { + int mask = (window >> (axis * 5)) & 31; + int offset = (window >> (10 + axis * 5)) & 31; + for (int p = limits[axis]; p <= limits[axis + 2]; ++p) { + const int mapped = window_coord(p, mask, offset); + first[axis] = std::min(first[axis], mapped); + last[axis] = std::max(last[axis], mapped); + } + } + query = {}; + query.page_x = (texpage & 15u) * 64u; + query.page_y = ((texpage >> 4) & 1u) * 256u; + query.depth = (texpage >> 7) & 3u; + query.u_first = static_cast(first[0]); + query.u_last = static_cast(last[0]); + query.v_first = static_cast(first[1]); + query.v_last = static_cast(last[1]); + query.clut_x = cx; query.clut_y = cy; + query.vram = native_vram; query.vram_word_count = kVramWords; + return true; +} +void dump_query(const HdTextureDrawQuery& query, int semi) { + if (!session->dump || session->dump_failed) return; + char error[256]{}; + const int status = duck_texture_pack_dump_draw(session->duck, &query, semi, error, sizeof(error)); + if (status == HD_TEXTURE_LOOKUP_FOUND) ++session->diag.dumped_textures; + if (status < 0) { + std::fprintf(stderr, "psxrecomp: texture dump stopped: %s\n", error); + /* Keep the configured raster authority until a safe session boundary. + * Tearing it down inside a pre-draw query would lose command order. */ + session->dump_failed = true; + } +} +} + +extern "C" int gpu_hd_textures_configure(const char* root, int replacements, + int dump, char* error, size_t capacity) { + if (error && capacity) error[0] = 0; + if ((!root || !root[0]) && !replacements && !dump) { + gpu_hd_textures_shutdown(); return 1; + } + if (!root || !root[0]) { fail(error, capacity, "Select an external texture-pack directory."); return 0; } + try { + auto next = std::make_unique(); + next->root = root; next->replacements = replacements != 0; next->dump = dump != 0; + const bool beetle = is_beetle_root(std::filesystem::u8path(root)); + if (beetle) { + if (!hd_texture_pack_create(root, &next->beetle, error, capacity)) return 0; + } else { + char ignored[256]{}; + hd_texture_pack_create(root, &next->beetle, ignored, sizeof(ignored)); + } + if (!next->beetle) { + auto directory = std::filesystem::u8path(root); + if (directory.filename().u8string() == "replacements") directory = directory.parent_path(); + std::filesystem::create_directories(directory / "replacements"); + std::filesystem::create_directories(directory / "dumps"); + } + if (!duck_texture_pack_create(root, &next->duck, error, capacity)) return 0; + if (session && session->root == next->root) { + if (!duck_texture_pack_copy_tracking(next->duck, session->duck)) { + fail(error, capacity, "Could not preserve texture-upload tracking during reload."); return 0; + } + if (session->beetle && next->beetle) { + uint8_t* saved = nullptr; size_t saved_size = 0; + if (!hd_texture_pack_tracking_state_save(session->beetle, &saved, &saved_size)) { + fail(error, capacity, "Could not save Beetle texture-upload tracking during reload."); return 0; + } + const int restored = hd_texture_pack_tracking_state_load(next->beetle, saved, saved_size); + std::free(saved); + if (!restored) { fail(error, capacity, "Could not restore Beetle texture-upload tracking during reload."); return 0; } + } + } + /* This also flushes all queued users before old replacement textures + * and pixel leases may be destroyed. */ + gl_renderer_set_hd_texture_mode(next->replacements || next->dump); + gl_renderer_clear_hd_texture_cache(); + session = std::move(next); ++generation; + DuckTexturePackInfo info{}; + duck_texture_pack_get_info(session->duck, &info); + size_t replacements_found = info.replacement_count; + size_t ambiguous = info.ambiguous_count; + if (session->beetle) { + HdTexturePackInfo beetle_info{}; + hd_texture_pack_get_info(session->beetle, &beetle_info); + replacements_found = beetle_info.unique_key_count; + ambiguous = beetle_info.ambiguous_key_count; + } + std::fprintf(stdout, "psxrecomp: texture pack '%s': %zu replacements, %zu ambiguous, %zu ignored\n", + session->root.c_str(), replacements_found, ambiguous, info.ignored_count); + if (info.diagnostic && info.diagnostic[0]) + std::fprintf(stderr, "psxrecomp: texture pack: %s\n", info.diagnostic); + return 1; + } catch (const std::exception& exception) { + fail(error, capacity, exception.what()); return 0; + } +} +extern "C" void gpu_hd_textures_shutdown(void) { + gl_renderer_set_hd_texture_mode(0); + gl_renderer_clear_hd_texture_cache(); + session.reset(); ++generation; +} +extern "C" int gpu_hd_textures_reload(char* error, size_t capacity) { + if (!session) { fail(error, capacity, "No texture-pack session is active."); return 0; } + const std::string root = session->root; + return gpu_hd_textures_configure(root.c_str(), session->replacements, session->dump, error, capacity); +} +extern "C" void gpu_hd_textures_set_dump_enabled(int enabled) { + if (!session || session->dump == (enabled != 0)) return; + gl_renderer_set_hd_texture_mode(session->replacements || enabled); + session->dump = enabled != 0; + session->dump_failed = false; +} +extern "C" int gpu_hd_textures_replacements_enabled(void) { return session && session->replacements; } +extern "C" int gpu_hd_textures_dump_enabled(void) { return session && session->dump; } +extern "C" int gpu_hd_textures_active(void) { return session && (session->replacements || session->dump); } +extern "C" void gpu_hd_textures_get_diag(GpuHdTextureDiag* out) { + if (!out) return; + *out = {}; + if (!session) { out->root = ""; return; } + *out = session->diag; + out->root = session->root.c_str(); out->active = gpu_hd_textures_active(); + out->replacements = session->replacements; out->dump = session->dump && !session->dump_failed; + out->format = session->beetle ? 1 : 2; + if (session->beetle) { HdTexturePackInfo info{}; hd_texture_pack_get_info(session->beetle, &info); out->replacement_count = info.unique_key_count; } + else { DuckTexturePackInfo info{}; duck_texture_pack_get_info(session->duck, &info); out->replacement_count = info.replacement_count; } +} +extern "C" void gpu_hd_textures_note_applied(void) { if (session) ++session->diag.applied_draws; } +extern "C" void gpu_hd_textures_set_vram(const uint16_t* vram) { native_vram = vram; gpu_hd_textures_reset_tracking(); } +extern "C" void gpu_hd_textures_reset_tracking(void) { + if (!session) return; + hd_texture_pack_reset_tracking(session->beetle); + duck_texture_pack_reset_tracking(session->duck); +} +extern "C" void gpu_hd_textures_invalidate(int x, int y, int width, int height) { + if (!session || width < 1 || height < 1) return; + if (width > 1024 || height > 512) { gpu_hd_textures_reset_tracking(); return; } + hd_texture_pack_invalidate(session->beetle, x & 1023, y & 511, width, height); + duck_texture_pack_invalidate(session->duck, x & 1023, y & 511, width, height); +} +extern "C" void gpu_hd_textures_track_upload(int x, int y, int width, int height, + const uint16_t* words) { + if (!session || !native_vram || !words || width < 1 || width > 1024 || height < 1 || height > 512) return; + if (width == 1024 && height == 512) { gpu_hd_textures_reset_tracking(); return; } + /* gpu.c's A0 staging already contains post-mask words. Read the canonical + * mirror here also for host-initiated transfers which use raw input. */ + const size_t count = static_cast(width) * height; + std::unique_ptr applied(new (std::nothrow) uint16_t[count]); + if (!applied) { gpu_hd_textures_reset_tracking(); return; } + for (int row = 0; row < height; ++row) + for (int col = 0; col < width; ++col) + applied[static_cast(row) * width + col] = + native_vram[((y + row) & 511) * 1024 + ((x + col) & 1023)]; + hd_texture_pack_track_upload(session->beetle, x & 1023, y & 511, width, height, + applied.get(), count, nullptr); + duck_texture_pack_track_upload(session->duck, x & 1023, y & 511, width, height, + native_vram, kVramWords); +} +extern "C" int gpu_hd_textures_acquire_draw(uint16_t tp, uint16_t cx, uint16_t cy, + const int limits[4], uint32_t window, + int semi, GpuHdTextureImage* image) { + if (!image) return 0; + *image = {}; + if (!session) return 0; + ++session->diag.draw_queries; + HdTextureDrawQuery query{}; + if (!make_query(tp, cx, cy, limits, window, query)) return 0; + dump_query(query, semi); + if (!session->replacements) return 0; + auto lease = std::unique_ptr(new (std::nothrow) Lease); + if (!lease) return 0; + if (session->beetle) { + HdTextureMatch match{}; + if (hd_texture_pack_match(session->beetle, &query, &match) != HD_TEXTURE_LOOKUP_FOUND) return 0; + ++session->diag.matched_draws; + hd_texture_pack_request_decode(session->beetle, match.entry.texture_hash, match.entry.palette_hash); + if (hd_texture_pack_acquire_decoded(session->beetle, match.entry.texture_hash, + match.entry.palette_hash, &lease->beetle) != HD_TEXTURE_LOOKUP_FOUND) return 0; + const unsigned pixels_per_word = 4u >> query.depth; + image->rgba = lease->beetle.rgba; image->width = lease->beetle.width; + image->height = lease->beetle.height; image->stride = lease->beetle.stride; + image->source_width = match.upload_width_words * pixels_per_word; + image->source_height = match.upload_height; + image->origin_u = query.u_first - (match.source_word_x * pixels_per_word + (query.u_first % pixels_per_word)); + image->origin_v = query.v_first - match.source_y; + image->cache_key = (static_cast(match.entry.texture_hash) << 32) | match.entry.palette_hash; + image->alpha_mode = 1; + } else { + DuckTextureMatch match{}; + if (duck_texture_pack_match_draw(session->duck, &query, semi, &match) != HD_TEXTURE_LOOKUP_FOUND) return 0; + ++session->diag.matched_draws; + duck_texture_pack_request_decode(session->duck, match.entry_id); + if (duck_texture_pack_acquire_decoded(session->duck, match.entry_id, &lease->duck) != HD_TEXTURE_LOOKUP_FOUND) return 0; + image->rgba = lease->duck.rgba; image->width = lease->duck.width; + image->height = lease->duck.height; image->stride = lease->duck.stride; + image->source_width = match.source_width; image->source_height = match.source_height; + image->origin_u = match.origin_u; image->origin_v = match.origin_v; + image->cache_key = match.entry_id; + image->alpha_mode = match.key.semitransparent ? 3 : 2; + } + image->generation = generation; image->lease = lease.release(); + ++session->diag.ready_draws; + return 1; +} +extern "C" void gpu_hd_textures_release_image(GpuHdTextureImage* image) { + if (!image) return; + delete static_cast(image->lease); *image = {}; +} +extern "C" void gpu_hd_textures_observe_draw(uint16_t tp, uint16_t cx, uint16_t cy, + const int limits[4], uint32_t window, int semi) { + if (!session || !session->dump || session->dump_failed) return; + HdTextureDrawQuery query{}; + if (make_query(tp, cx, cy, limits, window, query)) { + ++session->diag.draw_queries; dump_query(query, semi); + } +} diff --git a/runtime/src/gpu_render.c b/runtime/src/gpu_render.c index a7cc5606..7a04d105 100644 --- a/runtime/src/gpu_render.c +++ b/runtime/src/gpu_render.c @@ -12,6 +12,8 @@ #include "gpu_render.h" #include "gpu_sw_renderer.h" +#include "gpu_hd_textures.h" +#include "gpu_uv.h" #include static const GpuRenderBackend SW_BACKEND = { @@ -65,6 +67,47 @@ extern const GpuRenderBackend *vk_backend_get(void); static const GpuRenderBackend *g_b = &SW_BACKEND; static GrBackend g_effective = GR_BACKEND_SOFTWARE; +static int g_hd_offset_x, g_hd_offset_y; +static uint32_t g_hd_texture_window; +static int g_hd_semi, g_hd_perspective; +static void hd_observe_triangle(int x0,int y0,int u0,int v0,int x1,int y1,int u1,int v1, + int x2,int y2,int u2,int v2,uint16_t cx,uint16_t cy,uint16_t tp) { + if (g_effective == GR_BACKEND_OPENGL || !gpu_hd_textures_dump_enabled()) return; + if (g_effective == GR_BACKEND_VULKAN) g_b->vram_read(0,0); + int limits[4], xs[3]={x0,x1,x2},ys[3]={y0,y1,y2},us[3]={u0,u1,u2},vs[3]={v0,v1,v2}; + if (g_hd_perspective) psx_uv_tri_world_limits(us,vs,limits); + else psx_uv_tri_limits(xs,ys,us,vs,limits); + gpu_hd_textures_observe_draw(tp,cx,cy,limits,g_hd_texture_window,g_hd_semi); +} +static void hd_observe_rect(int u0,int v0,int u1,int v1,uint16_t cx,uint16_t cy,uint16_t tp) { + if (g_effective == GR_BACKEND_OPENGL || !gpu_hd_textures_dump_enabled()) return; + if (g_effective == GR_BACKEND_VULKAN) g_b->vram_read(0,0); + int limits[4]; psx_uv_rect_limits(u0,v0,u1,v1,limits); + gpu_hd_textures_observe_draw(tp,cx,cy,limits,g_hd_texture_window,g_hd_semi); +} + +/* Residency describes guest native words, so any draw/copy which can touch + * an upload ends its identity. Run after submission: an overlapping texture + * draw must resolve its source before the destination invalidates it. */ +static void hd_invalidate_draw(int x0, int y0, int x1, int y1) { + int ax0, ay0, ax1, ay1; + if (!gpu_hd_textures_active()) return; + g_b->get_draw_area(&ax0, &ay0, &ax1, &ay1); + x0 += g_hd_offset_x; x1 += g_hd_offset_x; + y0 += g_hd_offset_y; y1 += g_hd_offset_y; + if (x0 < ax0) x0 = ax0; if (x1 > ax1) x1 = ax1; + if (y0 < ay0) y0 = ay0; if (y1 > ay1) y1 = ay1; + if (x0 <= x1 && y0 <= y1) + gpu_hd_textures_invalidate(x0, y0, x1 - x0 + 1, y1 - y0 + 1); +} +static void hd_invalidate_triangle(int x0, int y0, int x1, int y1, int x2, int y2) { + int xmin = x0, xmax = x0, ymin = y0, ymax = y0; + if (x1 < xmin) xmin = x1; if (x2 < xmin) xmin = x2; + if (x1 > xmax) xmax = x1; if (x2 > xmax) xmax = x2; + if (y1 < ymin) ymin = y1; if (y2 < ymin) ymin = y2; + if (y1 > ymax) ymax = y1; if (y2 > ymax) ymax = y2; + hd_invalidate_draw(xmin, ymin, xmax, ymax); +} void gr_set_backend(GrBackend backend) { if (backend == GR_BACKEND_OPENGL) { @@ -95,14 +138,14 @@ void gr_set_backend(GrBackend backend) { GrBackend gr_backend(void) { return g_effective; } /* ---- Dispatch wrappers (one line each; forward to the active backend) ---- */ -void gr_init(uint16_t *vram) { g_b->init(vram); } +void gr_init(uint16_t *vram) { g_b->init(vram); gpu_hd_textures_set_vram(vram); g_hd_offset_x = g_hd_offset_y = 0; g_hd_texture_window = 0; g_hd_semi = g_hd_perspective = 0; } void gr_set_scale(int scale) { g_b->set_scale(scale); } int gr_scale(void) { return g_b->scale(); } void gr_set_texture_filter(int bilinear) { g_b->set_texture_filter(bilinear); } int gr_texture_filter(void) { return g_b->texture_filter(); } -void gr_set_semi_transparency(int e, int m) { g_b->set_semi_transparency(e, m); } +void gr_set_semi_transparency(int e, int m) { g_b->set_semi_transparency(e, m); g_hd_semi=e; } void gr_set_mask_bits(int s, int c) { g_b->set_mask_bits(s, c); } -void gr_set_texture_window(uint32_t raw) { g_b->set_texture_window(raw); } +void gr_set_texture_window(uint32_t raw) { g_b->set_texture_window(raw); g_hd_texture_window=raw; } void gr_set_color_modulation(int r, int g, int b, int raw) { g_b->set_color_modulation(r, g, b, raw); } void gr_set_precise_triangle(int enabled, int32_t x0, int32_t y0, int32_t x1, int32_t y1, int32_t x2, int32_t y2) { @@ -110,44 +153,60 @@ void gr_set_precise_triangle(int enabled, int32_t x0, int32_t y0, int32_t x1, in g_b->set_precise_triangle(enabled, x0, y0, x1, y1, x2, y2); } void gr_set_perspective_triangle(int enabled, float q0, float q1, float q2) { + g_hd_perspective = enabled && q0 > 0 && q1 > 0 && q2 > 0; if (g_b->set_perspective_triangle) g_b->set_perspective_triangle(enabled, q0, q1, q2); } -void gr_fill_rect(int x, int y, int w, int h, uint16_t c) { g_b->fill_rect(x, y, w, h, c); } -void gr_copy_rect(int sx, int sy, int dx, int dy, int w, int h) { g_b->copy_rect(sx, sy, dx, dy, w, h); } +void gr_fill_rect(int x, int y, int w, int h, uint16_t c) { g_b->fill_rect(x, y, w, h, c); gpu_hd_textures_invalidate(x, y, w, h); } +void gr_copy_rect(int sx, int sy, int dx, int dy, int w, int h) { g_b->copy_rect(sx, sy, dx, dy, w, h); gpu_hd_textures_invalidate(dx, dy, w, h); } void gr_draw_flat_triangle(int x0, int y0, int x1, int y1, int x2, int y2, uint16_t c) { g_b->draw_flat_triangle(x0, y0, x1, y1, x2, y2, c); + hd_invalidate_triangle(x0, y0, x1, y1, x2, y2); + g_hd_perspective = 0; } void gr_draw_gouraud_triangle(int x0, int y0, uint16_t c0, int x1, int y1, uint16_t c1, int x2, int y2, uint16_t c2) { g_b->draw_gouraud_triangle(x0, y0, c0, x1, y1, c1, x2, y2, c2); + hd_invalidate_triangle(x0, y0, x1, y1, x2, y2); + g_hd_perspective = 0; } void gr_draw_textured_triangle(int x0, int y0, int u0, int v0, int x1, int y1, int u1, int v1, int x2, int y2, int u2, int v2, uint16_t clut_x, uint16_t clut_y, uint16_t texpage) { + hd_observe_triangle(x0,y0,u0,v0,x1,y1,u1,v1,x2,y2,u2,v2,clut_x,clut_y,texpage); g_b->draw_textured_triangle(x0, y0, u0, v0, x1, y1, u1, v1, x2, y2, u2, v2, clut_x, clut_y, texpage); + hd_invalidate_triangle(x0, y0, x1, y1, x2, y2); + g_hd_perspective = 0; } void gr_draw_shaded_textured_triangle(int x0, int y0, int u0, int v0, uint32_t c0, int x1, int y1, int u1, int v1, uint32_t c1, int x2, int y2, int u2, int v2, uint32_t c2, uint16_t clut_x, uint16_t clut_y, uint16_t texpage, int raw) { + hd_observe_triangle(x0,y0,u0,v0,x1,y1,u1,v1,x2,y2,u2,v2,clut_x,clut_y,texpage); g_b->draw_shaded_textured_triangle(x0, y0, u0, v0, c0, x1, y1, u1, v1, c1, x2, y2, u2, v2, c2, clut_x, clut_y, texpage, raw); + hd_invalidate_triangle(x0, y0, x1, y1, x2, y2); + g_hd_perspective = 0; } -void gr_draw_flat_rect(int x, int y, int w, int h, uint16_t c) { g_b->draw_flat_rect(x, y, w, h, c); } +void gr_draw_flat_rect(int x, int y, int w, int h, uint16_t c) { g_b->draw_flat_rect(x, y, w, h, c); if (w > 0 && h > 0) hd_invalidate_draw(x,y,x+w-1,y+h-1); } void gr_draw_textured_rect(int x, int y, int w, int h, int u, int v, uint16_t clut_x, uint16_t clut_y, uint16_t texpage) { + hd_observe_rect(u,v,u+w,v+h,clut_x,clut_y,texpage); g_b->draw_textured_rect(x, y, w, h, u, v, clut_x, clut_y, texpage); + if (w > 0 && h > 0) hd_invalidate_draw(x,y,x+w-1,y+h-1); } void gr_draw_textured_rect_scaled(int x, int y, int w, int h, int u0, int v0, int u1, int v1, uint16_t clut_x, uint16_t clut_y, uint16_t texpage) { + hd_observe_rect(u0,v0,u1,v1,clut_x,clut_y,texpage); g_b->draw_textured_rect_scaled(x, y, w, h, u0, v0, u1, v1, clut_x, clut_y, texpage); + if (w > 0 && h > 0) hd_invalidate_draw(x,y,x+w-1,y+h-1); } -void gr_draw_line(int x0, int y0, int x1, int y1, uint16_t c) { g_b->draw_line(x0, y0, x1, y1, c); } +void gr_draw_line(int x0, int y0, int x1, int y1, uint16_t c) { g_b->draw_line(x0, y0, x1, y1, c); hd_invalidate_triangle(x0,y0,x1,y1,x1,y1); } void gr_draw_shaded_line(int x0, int y0, uint16_t c0, int x1, int y1, uint16_t c1) { g_b->draw_shaded_line(x0, y0, c0, x1, y1, c1); + hd_invalidate_triangle(x0,y0,x1,y1,x1,y1); } int gr_render_display(uint32_t *o, int p, int dx, int dy, int dw, int dh) { return g_b->render_display(o, p, dx, dy, dw, dh); @@ -155,13 +214,14 @@ int gr_render_display(uint32_t *o, int p, int dx, int dy, int dw, int dh) { int gr_render_display_hires(uint32_t *o, int p, int dx, int dy, int dw, int dh) { return g_b->render_display_hires(o, p, dx, dy, dw, dh); } -void gr_vram_write(int x, int y, uint16_t pixel) { g_b->vram_write(x, y, pixel); } +void gr_vram_upload_begin(int x, int y, int w, int h) { gpu_hd_textures_invalidate(x,y,w,h); } +void gr_vram_write(int x, int y, uint16_t pixel) { g_b->vram_write(x, y, pixel); gpu_hd_textures_invalidate(x,y,1,1); } uint16_t gr_vram_read(int x, int y) { return g_b->vram_read(x, y); } -void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *d) { g_b->vram_transfer_in(x, y, w, h, d); } +void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *d) { g_b->vram_transfer_in(x, y, w, h, d); gpu_hd_textures_track_upload(x,y,w,h,d); } void gr_vram_transfer_out(int x, int y, int w, int h, uint16_t *d) { g_b->vram_transfer_out(x, y, w, h, d); } void gr_set_draw_area(int x1, int y1, int x2, int y2){ g_b->set_draw_area(x1, y1, x2, y2); } void gr_get_draw_area(int *x1, int *y1, int *x2, int *y2) { g_b->get_draw_area(x1, y1, x2, y2); } -void gr_set_draw_offset(int x, int y) { g_b->set_draw_offset(x, y); } +void gr_set_draw_offset(int x, int y) { g_b->set_draw_offset(x, y); g_hd_offset_x=x; g_hd_offset_y=y; } /* Native-wide compositor — present only on backends that supply it. */ int gr_wide_supported(void) { return g_b->render_wide_display != 0; } diff --git a/runtime/src/mod_builtin_hd_textures.c b/runtime/src/mod_builtin_hd_textures.c new file mode 100644 index 00000000..952d0408 --- /dev/null +++ b/runtime/src/mod_builtin_hd_textures.c @@ -0,0 +1,22 @@ +/* Framework implementation shared by title-owned HD texture pack manifests. + * The owning feature declares directory resource "pack" and boolean options + * "replacements" (default true) and "dump" (default false). */ +#include "mod_plugins.h" + +#include + +static int hd_texture_option(const char* id, int fallback) { + char value[16] = ""; + if (!psx_mod_current_option_value(id, value, sizeof(value))) return fallback; + return strcmp(value, "true") == 0; +} + +static void builtin_hd_textures_activate(void) { + (void)psx_mod_set_hd_texture_pack("pack", hd_texture_option("replacements", 1), + hd_texture_option("dump", 0)); +} + +PSX_MOD_CONSTRUCTOR(psx_register_builtin_hd_textures_plugin) { + (void)psx_mod_register_activation_plugin("psx.hd-textures", + builtin_hd_textures_activate); +} diff --git a/runtime/src/mod_packages.cpp b/runtime/src/mod_packages.cpp index bfc5af09..bbd68238 100644 --- a/runtime/src/mod_packages.cpp +++ b/runtime/src/mod_packages.cpp @@ -774,7 +774,7 @@ std::string canonical_resolution(const std::vector& ordered, out << resource.format << ':' << resource.bytes->size() << ':' << resource.sha256; else - out << resource.path.string(); + out << resource.path.u8string(); out << '\n'; } return out.str(); @@ -870,7 +870,7 @@ fs::path effective_resource_path(const ModPackage& package, if (!selected) return {}; const auto resource = selected->resources.find(id); return resource == selected->resources.end() ? fs::path{} : - fs::path(resource->second); + fs::u8path(resource->second); } bool prospective_feature_enabled( @@ -3206,7 +3206,7 @@ bool ModPackageManager::set_feature_resource_path( return false; } if (!resource->shared_source.empty()) sources_[resource->shared_source] = path; - else selections_[package_id].features[feature_id].resources[resource_id] = path.string(); + else selections_[package_id].features[feature_id].resources[resource_id] = path.u8string(); return true; } @@ -3458,7 +3458,7 @@ bool ModPackageManager::prepare_resources(const std::string& game_id, const auto* resource = find_resource(*package, feature.id, name); if (!resource || resource->input_only || resource->sha256.empty()) throw std::runtime_error("preparer returned an undeclared or unverified output"); - pending[id].features[feature.id].resources[name] = path.string(); + pending[id].features[feature.id].resources[name] = path.u8string(); } for (const auto& r : package->resources) if (r.feature_id == feature.id && !r.input_only && r.required && !outputs.count(r.id)) diff --git a/runtime/src/mod_runtime.cpp b/runtime/src/mod_runtime.cpp index 3e4fa78d..8efa62b1 100644 --- a/runtime/src/mod_runtime.cpp +++ b/runtime/src/mod_runtime.cpp @@ -6,6 +6,7 @@ #include "mod_packages.h" #include "mod_plugins.h" #include "gpu.h" +#include "gpu_hd_textures.h" #include "psx_memory.h" #include "render_pass_projection.h" #include "psx_sha256.h" @@ -1021,7 +1022,7 @@ int provider_feature_resource_get(void*, const char* package_id, copy_text(out->label, sizeof(out->label), resource.label); copy_text(out->description, sizeof(out->description), resource.description); - copy_text(out->path, sizeof(out->path), path.string()); + copy_text(out->path, sizeof(out->path), path.u8string()); copy_text(out->status, sizeof(out->status), path.empty() ? "Not selected" : (verified ? "Selected" : "Selected path is missing")); @@ -1045,7 +1046,7 @@ int provider_feature_resource_set_path(void*, const char* package_id, return mutate([&](std::string& error) { return state().manager.set_feature_resource_path( package_id, feature_id, resource_id, - std::filesystem::path(path), &error); + std::filesystem::u8path(path), &error); }); } @@ -1287,6 +1288,43 @@ bool mod_runtime_initialize(const std::filesystem::path& root, "psxrecomp: mod selection kept but inactive: %s is not " "in this build's mod catalog\n", dormant.c_str()); + /* psx.hd-textures has a shared directory-resource contract, independent of + * the title package id. Keep owner files outside the build-owned bundled + * tree, and give the launcher a useful Open folder action on first use. */ + std::string texture_game_id = game_id; + for (char& c : texture_game_id) + if (!std::isalnum(static_cast(c)) && c != '-' && c != '_') + c = '_'; + if (texture_game_id.empty()) texture_game_id = "game"; + for (const auto& entry : s.manager.packages()) { + const ModPackage* package = s.manager.selected_package(entry.first); + if (!package || !std::any_of(package->targets.begin(), package->targets.end(), + [&](const ModTarget& target) { + return target.game_id == game_id || target.game_id == "*"; + })) continue; + for (const ModPlugin& plugin : package->plugins) { + if (plugin.id != "psx.hd-textures" || + !s.manager.feature_resource_path(package->id, plugin.feature_id, "pack").empty()) + continue; + const auto resource = std::find_if(package->resources.begin(), package->resources.end(), + [&](const ModResource& r) { + return r.feature_id == plugin.feature_id && r.id == "pack" && + (r.format == "directory" || r.format == "folder"); + }); + if (resource == package->resources.end()) continue; + std::error_code ec; + const auto pack = std::filesystem::absolute( + root / "texture-packs" / texture_game_id, ec); + if (!ec) std::filesystem::create_directories(pack / "replacements", ec); + if (!ec) std::filesystem::create_directories(pack / "dumps", ec); + std::string directory_error; + if (ec || !s.manager.set_feature_resource_path(package->id, + plugin.feature_id, "pack", pack, &directory_error)) { + std::fprintf(stderr, "psxrecomp: texture pack folder: %s\n", + ec ? ec.message().c_str() : directory_error.c_str()); + } + } + } if (!sha256_file(exe_path, s.exe_sha256, &s.error)) { /* Release installs commonly do not carry a loose PS-X EXE; game-id and * expected-byte guards remain available in that case. */ @@ -2129,6 +2167,7 @@ extern "C" void mod_runtime_activate_plugins(void) { psx_projection_reset_session(); gpu_ws_set_native_scene_predicate(nullptr); psx_ram_reset_size_request(); + gpu_hd_textures_shutdown(); if (!s.initialized || !s.plan.ok) return; s.disc_extents.clear(); s.audio_tracks.clear(); @@ -2194,7 +2233,7 @@ extern "C" int psx_mod_current_resource_path(const char* resource_id, resource.feature_id != s.current_plugin->feature_id || resource.id != resource_id) continue; - const std::string text = resource.path.string(); + const std::string text = resource.path.u8string(); if (text.empty() || text.size() + 1 > (size_t)out_size) return 0; std::memcpy(out, text.c_str(), text.size() + 1); return 1; @@ -2202,6 +2241,49 @@ extern "C" int psx_mod_current_resource_path(const char* resource_id, return 0; } +extern "C" int psx_mod_current_option_value(const char* option_id, + char* out, uint32_t out_size) { + using namespace PSXRecompV4; + if (out && out_size) out[0] = '\0'; + const auto* plugin = state().current_plugin; + if (!plugin) return 0; + return psx_mod_option_value(plugin->package_id.c_str(), plugin->feature_id.c_str(), + option_id, out, out_size); +} + +extern "C" int psx_mod_set_hd_texture_pack(const char* resource_id, + int replacements_enabled, int dump_enabled) { + char root[4096] = ""; + char error[512] = ""; + if (!psx_mod_current_resource_path(resource_id, root, sizeof(root))) { + std::fprintf(stderr, "psxrecomp: HD textures require a selected pack folder\n"); + return 0; + } + if (!gpu_hd_textures_configure(root, replacements_enabled, dump_enabled, + error, sizeof(error))) { + std::fprintf(stderr, "psxrecomp: HD textures: %s\n", error); + return 0; + } + std::fprintf(stdout, "psxrecomp: HD textures: %s (replacements %s, dumping %s)\n", + root, replacements_enabled ? "on" : "off", dump_enabled ? "on" : "off"); + return 1; +} + +extern "C" int psx_mod_set_hd_texture_dump(int enabled) { + GpuHdTextureDiag info{}; + gpu_hd_textures_get_diag(&info); + if (!info.root || !info.root[0]) return 0; + gpu_hd_textures_set_dump_enabled(enabled); + return 1; +} + +extern "C" int psx_mod_reload_hd_texture_pack(void) { + char error[512] = ""; + const int ok = gpu_hd_textures_reload(error, sizeof(error)); + if (!ok) std::fprintf(stderr, "psxrecomp: HD textures: %s\n", error); + return ok; +} + extern "C" uint8_t psx_mod_read_byte(uint32_t address) { return psx_read_byte(address); } diff --git a/runtime/src/png_write.h b/runtime/src/png_write.h index 8347b46a..2c38a410 100644 --- a/runtime/src/png_write.h +++ b/runtime/src/png_write.h @@ -55,8 +55,10 @@ static void png_chunk(FILE *f, const char *type, const uint8_t *data, size_t len if (len) fwrite(data, 1, len, f); png_put_be32(f, crc ^ 0xFFFFFFFFu); } -/* Write an RGB (3 bytes/pixel, top-down) buffer as a PNG. Returns 1 on success. */ -static int png_write_rgb(FILE *f, const uint8_t *rgb, uint32_t w, uint32_t h) { +/* Write top-down RGB/RGBA data. This internal helper has static linkage so + * the public wrappers stay usable by the existing C debug servers. */ +static int png_write_channels(FILE *f, const uint8_t *pixels, uint32_t w, + uint32_t h, unsigned channels) { static const uint8_t sig[8] = { 137,80,78,71,13,10,26,10 }; fwrite(sig, 1, 8, f); @@ -64,20 +66,20 @@ static int png_write_rgb(FILE *f, const uint8_t *rgb, uint32_t w, uint32_t h) { ihdr[0]=(uint8_t)(w>>24); ihdr[1]=(uint8_t)(w>>16); ihdr[2]=(uint8_t)(w>>8); ihdr[3]=(uint8_t)w; ihdr[4]=(uint8_t)(h>>24); ihdr[5]=(uint8_t)(h>>16); ihdr[6]=(uint8_t)(h>>8); ihdr[7]=(uint8_t)h; ihdr[8]=8; /* bit depth */ - ihdr[9]=2; /* color type 2 = truecolor RGB */ + ihdr[9]=(uint8_t)(channels == 4 ? 6 : 2); /* RGBA or RGB */ ihdr[10]=0; /* compression */ ihdr[11]=0; /* filter */ ihdr[12]=0; /* interlace */ png_chunk(f, "IHDR", ihdr, sizeof ihdr); /* Filtered raw scanlines: each row prefixed with filter byte 0 (None). */ - size_t raw_len = (size_t)h * (1 + (size_t)w * 3); + size_t raw_len = (size_t)h * (1 + (size_t)w * channels); uint8_t *raw = (uint8_t *)malloc(raw_len); if (!raw) return 0; for (uint32_t y = 0; y < h; y++) { - uint8_t *row = raw + (size_t)y * (1 + (size_t)w * 3); + uint8_t *row = raw + (size_t)y * (1 + (size_t)w * channels); row[0] = 0; - memcpy(row + 1, rgb + (size_t)y * w * 3, (size_t)w * 3); + memcpy(row + 1, pixels + (size_t)y * w * channels, (size_t)w * channels); } /* zlib stream: 2-byte header + stored DEFLATE blocks + 4-byte Adler32. */ @@ -106,7 +108,16 @@ static int png_write_rgb(FILE *f, const uint8_t *rgb, uint32_t w, uint32_t h) { png_chunk(f, "IDAT", z, zi); free(z); png_chunk(f, "IEND", NULL, 0); - return 1; + return !ferror(f); +} + +/* Write an RGB (3 bytes/pixel, top-down) buffer as a PNG. Returns 1 on success. */ +static int png_write_rgb(FILE *f, const uint8_t *rgb, uint32_t w, uint32_t h) { + return png_write_channels(f, rgb, w, h, 3); +} +/* Preserve raw 8-bit RGBA, including DuckStation's ST alpha convention. */ +static int png_write_rgba(FILE *f, const uint8_t *rgba, uint32_t w, uint32_t h) { + return png_write_channels(f, rgba, w, h, 4); } #endif /* PSX_PNG_WRITE_H */ diff --git a/runtime/tests/gpu_exec_stubs.inc b/runtime/tests/gpu_exec_stubs.inc index 43e7a138..37d106a8 100644 --- a/runtime/tests/gpu_exec_stubs.inc +++ b/runtime/tests/gpu_exec_stubs.inc @@ -357,6 +357,7 @@ int gr_render_display_hires(uint32_t *out, int pitch, int dx, int dy, int dw, in (void)out; (void)pitch; (void)dx; (void)dy; (void)dw; (void)dh; return 0; } +void gr_vram_upload_begin(int x,int y,int w,int h) { (void)x; (void)y; (void)w; (void)h; } void gr_vram_write(int x, int y, uint16_t pixel) { (void)x; (void)y; (void)pixel; } uint16_t gr_vram_read(int x, int y) { (void)x; (void)y; return 0; } void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *data) { diff --git a/runtime/tests/gpu_hd_texture_stubs.inc b/runtime/tests/gpu_hd_texture_stubs.inc new file mode 100644 index 00000000..ef44dac4 --- /dev/null +++ b/runtime/tests/gpu_hd_texture_stubs.inc @@ -0,0 +1,14 @@ +/* Existing GL fixtures deliberately run without any external pack. The HD + * fixture links the real session adapter instead. */ +#ifndef PSX_TEST_HD_TEXTURE_PACK +int gpu_hd_textures_acquire_draw(uint16_t tp, uint16_t cx, uint16_t cy, + const int limits[4], uint32_t window, int semi, GpuHdTextureImage* image) { + (void)tp; (void)cx; (void)cy; (void)limits; (void)window; (void)semi; + memset(image, 0, sizeof(*image)); return 0; +} +void gpu_hd_textures_release_image(GpuHdTextureImage* image) { + memset(image, 0, sizeof(*image)); +} +void gpu_hd_textures_note_applied(void) {} +void gpu_hd_textures_reset_tracking(void) {} +#endif diff --git a/runtime/tests/psx_ram_runtime_map_stubs.c b/runtime/tests/psx_ram_runtime_map_stubs.c index edef5b7d..80a6fd64 100644 --- a/runtime/tests/psx_ram_runtime_map_stubs.c +++ b/runtime/tests/psx_ram_runtime_map_stubs.c @@ -170,3 +170,4 @@ void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *d) { (void)x; (void)y; memcpy(s_vram, d, (size_t)w * (size_t)h * 2u); g_stub_device_restores++; } +void gr_vram_upload_begin(int x,int y,int w,int h) { (void)x; (void)y; (void)w; (void)h; } diff --git a/runtime/tests/run_gl_texture_filter.py b/runtime/tests/run_gl_texture_filter.py index c7c6605a..a58ff7ce 100644 --- a/runtime/tests/run_gl_texture_filter.py +++ b/runtime/tests/run_gl_texture_filter.py @@ -1,6 +1,7 @@ """Compile and check palette-aware minification on a hidden real GL context.""" import argparse import json +import os from pathlib import Path import platform import subprocess @@ -13,14 +14,19 @@ def main(): parser.add_argument('--sdl-library',required=True) parser.add_argument('--output',required=True) parser.add_argument('--fixture',type=Path) + parser.add_argument('--hd-pack', action='store_true') + parser.add_argument('--cxx') + parser.add_argument('--argument', action='append', default=[]) args=parser.parse_args() fw=Path(__file__).resolve().parents[2] out=Path(args.output).resolve(); out.mkdir(parents=True,exist_ok=True) receipt=[] - def run(command): + def run(command,environment=None): command=[str(x) for x in command] - r=subprocess.run(command,cwd=out,capture_output=True,text=True,encoding='utf-8',errors='replace') - receipt.append(dict(cmd=command,exit=r.returncode,stdout=r.stdout,stderr=r.stderr)) + env=os.environ.copy() + if environment: env.update(environment) + r=subprocess.run(command,cwd=out,capture_output=True,text=True,encoding='utf-8',errors='replace',env=env) + receipt.append(dict(cmd=command,env=environment,exit=r.returncode,stdout=r.stdout,stderr=r.stderr)) (out/'receipt.json').write_text(json.dumps(receipt,indent=2)+'\n',encoding='utf-8') print(r.stdout,r.stderr[-4000:],flush=True) if r.returncode: raise subprocess.CalledProcessError(r.returncode,command) @@ -28,7 +34,7 @@ def main(): '-ffunction-sections','-fdata-sections','-I',fw/'runtime/include','-I',fw/'runtime/src'] for include in args.sdl_include.split(';'): if include: flags+=['-I',include] - sources=[('probe',args.fixture or fw/'runtime/tests/test_gl_texture_filter.c'), + sources=[('probe',args.fixture.resolve() if args.fixture else fw/'runtime/tests/test_gl_texture_filter.c'), ('sw',fw/'runtime/src/gpu_sw_renderer.c'),('fi',fw/'runtime/src/frame_interpolation.c'), ('rp',fw/'runtime/src/render_pass_plan.c')] if (fw/'runtime/src/psx_openxr.c').exists(): @@ -36,6 +42,13 @@ def main(): objects=[] for name,source in sources: obj=out/(name+'.o'); run([args.cc,*flags,'-c',source,'-o',obj]); objects.append(obj) + if args.hd_pack: + if not args.cxx: parser.error('--hd-pack requires --cxx') + cppflags=[arg if arg!='-std=gnu11' else '-std=c++17' for arg in flags] + for name in ('gpu_hd_textures','hd_texture_pack','duckstation_texture_pack'): + obj=out/(name+'.o') + run([args.cxx,*cppflags,'-c',fw/'runtime/src'/(name+'.cpp'),'-o',obj]) + objects.append(obj) exe=out/('filter.exe' if platform.system()=='Windows' else 'filter') libraries=['-lm'] if platform.system()=='Windows': @@ -47,8 +60,15 @@ def main(): libraries+=['-liconv','-Wl,-dead_strip'] else: libraries+=['-lGL','-ldl','-lpthread'] if platform.system()!='Darwin': libraries+=['-Wl,--gc-sections'] - run([args.cc,*objects,args.sdl_library,'-o',exe,*libraries]) - run([exe]) + run([args.cxx if args.hd_pack else args.cc,*objects,args.sdl_library,'-o',exe,*libraries]) + arguments=list(args.argument) + if args.hd_pack: + pack=out/'pack'; (pack/'replacements').mkdir(parents=True,exist_ok=True) + (pack/'beetle'/'demo-texture-replacements').mkdir(parents=True,exist_ok=True) + arguments=[pack] + run([exe,*arguments],{'PSX_GL_HIRES_WINDOW':'0'} if args.hd_pack else None) + if args.hd_pack: run([exe,*arguments],{'PSX_GL_HIRES_WINDOW':'1'}) + if args.hd_pack: run([exe,'--native-baseline'],{'PSX_GL_HIRES_WINDOW':'1'}) return 0 if __name__=='__main__': diff --git a/runtime/tests/test_duckstation_texture_pack.cpp b/runtime/tests/test_duckstation_texture_pack.cpp new file mode 100644 index 00000000..b5126650 --- /dev/null +++ b/runtime/tests/test_duckstation_texture_pack.cpp @@ -0,0 +1,309 @@ +#define STB_IMAGE_IMPLEMENTATION +#include "../third_party/stb_image.h" +#include "duckstation_texture_pack.h" +#include "../src/png_write.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace { +int failures = 0; +void check(bool ok, const char* what) { + if (!ok) { std::fprintf(stderr,"FAIL: %s\n",what); ++failures; } +} +using Pack = std::unique_ptr; +Pack create(const fs::path& root) { + DuckTexturePack* pack = nullptr; char error[512]{}; + check(duck_texture_pack_create(root.u8string().c_str(),&pack,error,sizeof(error)) == 1,error); + return Pack(pack,&duck_texture_pack_destroy); +} +std::vector words(size_t n) { + std::vector result(n); + for (size_t i = 0; i < n; ++i) result[i] = static_cast((i*7919)^0xa51c); + return result; +} +void place(std::vector& vram, unsigned x, unsigned y, unsigned w, + const std::vector& source) { + for (size_t i = 0; i < source.size(); ++i) vram[(y+i/w)*1024+x+i%w] = source[i]; +} +HdTextureDrawQuery query(std::vector& vram, uint16_t page_x, uint16_t page_y, + uint8_t depth, uint8_t u0, uint8_t u1, uint8_t v0, uint8_t v1, + uint16_t clut_x = 512, uint16_t clut_y = 400) { + return {page_x,page_y,depth,u0,u1,v0,v1,clut_x,clut_y,vram.data(),vram.size()}; +} +void png(const fs::path& path, unsigned w = 4, unsigned h = 4) { + fs::create_directories(path.parent_path()); + std::vector pixels(size_t(w)*h*4); + constexpr uint8_t alpha[] = {0,127,128,242,243,255}; + for (size_t i = 0; i < size_t(w)*h; ++i) { + pixels[i*4] = 23; pixels[i*4+1] = 57; pixels[i*4+2] = 91; pixels[i*4+3] = alpha[i%6]; + } +#ifdef _WIN32 + FILE* file = _wfopen(path.c_str(),L"wb"); +#else + FILE* file = std::fopen(path.c_str(),"wb"); +#endif + check(file != nullptr,"open original test PNG"); + if (file) { check(png_write_rgba(file,pixels.data(),w,h) == 1,"encode original test PNG"); std::fclose(file); } +} +DuckTexturePixels decoded(DuckTexturePack* pack, uint64_t id) { + DuckTexturePixels pixels{}; + check(duck_texture_pack_request_decode(pack,id) >= 0,"queue PNG decode"); + const auto end = std::chrono::steady_clock::now()+std::chrono::seconds(5); + int status = HD_TEXTURE_LOOKUP_NONE; + do { + status = duck_texture_pack_acquire_decoded(pack,id,&pixels); + if (status != HD_TEXTURE_LOOKUP_NONE) break; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } while (std::chrono::steady_clock::now() < end); + check(status == HD_TEXTURE_LOOKUP_FOUND,"background PNG decode becomes ready"); + return pixels; +} +bool decode_fails(DuckTexturePack* pack, uint64_t id) { + int status = duck_texture_pack_request_decode(pack,id); + const auto end = std::chrono::steady_clock::now()+std::chrono::seconds(5); + while (status == HD_TEXTURE_LOOKUP_NONE && std::chrono::steady_clock::now() < end) { + DuckTexturePixels pixels{}; + status = duck_texture_pack_acquire_decoded(pack,id,&pixels); + duck_texture_pixels_release(&pixels); + if (status == HD_TEXTURE_LOOKUP_NONE) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return status == HD_TEXTURE_LOOKUP_ERROR; +} + +void fixed_hash_vectors() { + /* Independently produced by Python xxhash 3.6.0's native wheel. These + * literal expectations cover XXH3's short/medium/long-input boundaries. */ + const std::pair vectors[] = { + {0,0x2d06800538d394c2ULL},{1,0xc5779b2f56fbe2e1ULL},{2,0x0edb68960f55aef6ULL}, + {3,0xfce9d54aec3f9d32ULL},{4,0x3631d2a04ec85311ULL},{8,0x000ca76cb9b329dcULL}, + {9,0xab6a9091cf532d8aULL},{16,0x9cbcdea9305d7528ULL},{17,0x2f133c3e193cbe48ULL}, + {32,0x3e4a087da5cd5990ULL},{64,0xc3e3ef9ad7f08dacULL},{65,0x740c4e7d1690d606ULL}, + {120,0x5942d561231419b2ULL},{121,0x152cb5857c4535f3ULL},{128,0x351cd56a8fe50e66ULL}, + {129,0x38a45de7ae84906eULL},{256,0x0005d0b83a005f4eULL},{8192,0xc68a00dcca652920ULL} + }; + for (const auto& v : vectors) { + const auto input = words(v.first); + check(duck_texture_hash_words_le(input.data(),input.size()) == v.second,"fixed independent LE16 XXH3 vector"); + } + std::vector vram(1024*512,0xf00d); + place(vram,101,39,2,words(8)); + check(duck_texture_hash_rect(vram.data(),vram.size(),101,39,2,4) == 0x000ca76cb9b329dcULL, + "strided native rectangle excludes unrelated VRAM words"); + check(duck_texture_hash_rect(vram.data(),vram.size(),1023,0,2,1) == 0,"wrapped hash rectangle rejected"); +} + +void names() { + DuckTextureKey key{}; + check(duck_texture_parse_name("texupload-P4-0123456789abcdef-FEDCBA9876543210-64x256-0-192-64x64-P0-14.png",&key) == 1, + "literal upstream documented upload filename"); + check(key.source_hash == 0x0123456789abcdefULL && key.palette_hash == 0xfedcba9876543210ULL && + key.source_width_words == 64 && key.offset_y == 192 && key.width == 64 && key.palette_max == 14, + "filename dimensions distinguish source words from expanded texels"); + check(duck_texture_parse_name("texpage-STC16-0123456789ABCDEF-256x256-8-16-32x64.PNG",&key) == 1 && + key.kind == DUCK_TEXTURE_PAGE && key.semitransparent && key.depth == 2 && !key.palette_hash, + "direct ST filename omits palette fields"); + const char* invalid[] = { + "texpage-P4-12345678-0123456789ABCDEF-64x256-0-0-4x1-P0-15.png", + "texpage-P4-0123456789ABCDEF-0123456789ABCDEF-64x256-0-0-3x1-P0-15.png", + "texpage-P4-0123456789ABCDEF-0123456789ABCDEF-64x256-0-0-4x1-P0-16.png", + "texpage-P8-0123456789ABCDEF-0123456789ABCDEF-128x256-0-0-4x1-P15-0.png", + "texupload-C16-0123456789ABCDEF-0x256-0-0-4x1.png", + "texupload-C16-0123456789ABCDEF-65536x1-0-0-4x1.png", + "texpage-C16-0123456789ABCDEF-256x256-255-0-2x1.png", + "texpage-C16-0123456789ABCDEF-256x256-0-0-4x1.png.old", + "../texpage-C16-0123456789ABCDEF-256x256-0-0-4x1.png" + }; + for (const char* name : invalid) check(!duck_texture_parse_name(name,&key),"targeted malformed/unsupported name rejected"); +} + +void page_and_decode(const fs::path& root) { + const char* name = "texpage-P4-3631D2A04EC85311-9CBCDEA9305D7528-64x256-4-2-8x2-P0-15.png"; + png(root/"replacements"/"nested"/name,16,4); + std::vector vram(1024*512); + place(vram,65,258,2,words(4)); place(vram,512,400,16,words(16)); + auto pack = create(root); auto q = query(vram,64,256,0,5,10,2,3); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"fixed subpage fixture matches"); + check(match.origin_u == 4 && match.origin_v == 2 && match.source_width == 8 && match.source_height == 2, + "crop extent and origin are page-space texels"); + auto pixels = decoded(pack.get(),match.entry_id); + check(pixels.width == 16 && pixels.height == 4,"arbitrary PNG scaling uses image dimensions"); + constexpr uint8_t alpha[] = {0,127,128,242,243,255}; + if (pixels.rgba) for (unsigned i = 0; i < 6; ++i) + check(pixels.rgba[i*4+3] == alpha[i],"decode retains raw alpha boundary bytes"); + const uint16_t old = vram[258*1024+65]; + vram[258*1024+65] ^= 1; duck_texture_pack_invalidate(pack.get(),65,258,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"write invalidates positive page cache"); + vram[258*1024+65] = old; duck_texture_pack_invalidate(pack.get(),65,258,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"write invalidates negative page cache"); + vram[400*1024+512] ^= 1; duck_texture_pack_invalidate(pack.get(),512,400,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"CLUT write invalidates cached match"); + pack.reset(); + check(pixels.rgba && pixels.rgba[0] == 23,"pixel lease outlives pack and decode worker"); + duck_texture_pixels_release(&pixels); +} + +void upload(const fs::path& root) { + png(root/"replacements"/"texupload-C16-000CA76CB9B329DC-4x2-2-0-2x2.png",4,8); + std::vector vram(1024*512); place(vram,66,257,4,words(8)); + auto pack = create(root); auto q = query(vram,64,256,2,4,5,1,2); + DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"upload names require original source identity"); + check(duck_texture_pack_track_upload(pack.get(),66,257,4,2,vram.data(),vram.size()),"track post-write native upload"); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND && match.origin_u == 4 && match.origin_v == 1, + "upload crop has correct source origin"); + duck_texture_pack_invalidate(pack.get(),66,257,1,1); /* outside replacement crop */ + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"partial source overwrite drops entire upload identity"); + duck_texture_pack_track_upload(pack.get(),66,257,4,2,vram.data(),vram.size()); + duck_texture_pack_reset_tracking(pack.get()); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"savestate/reset clears upload identity"); + check(!duck_texture_pack_track_upload(pack.get(),1023,0,2,1,vram.data(),vram.size()),"wrapped upload explicitly falls back"); +} + +void reload_tracking(const fs::path& root) { + std::vector vram(1024*512); place(vram,66,257,4,words(8)); + auto old = create(root); + check(duck_texture_pack_track_upload(old.get(),66,257,4,2,vram.data(),vram.size()),"capture upload before replacement file exists"); + png(root/"replacements"/"texupload-C16-000CA76CB9B329DC-4x2-2-0-2x2.png"); + auto refreshed = create(root); auto q = query(vram,64,256,2,4,5,1,2); DuckTextureMatch match{}; + check(duck_texture_pack_match(refreshed.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"newly indexed replacement initially has no upload identity"); + check(duck_texture_pack_copy_tracking(refreshed.get(),old.get()),"same-session reload copies upload tracking"); + check(duck_texture_pack_match(refreshed.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"copied tracking makes newly indexed replacement immediately match"); + duck_texture_pack_invalidate(refreshed.get(),66,257,1,1); + check(duck_texture_pack_match(refreshed.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"destination tracking invalidates independently"); + auto second = create(root); duck_texture_pack_copy_tracking(second.get(),old.get()); + check(duck_texture_pack_match(second.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"destination invalidation leaves original tracker intact"); +} + +void palettes(const fs::path& root) { + std::vector vram(1024*512); place(vram,128,0,1,words(2)); + place(vram,1008,400,16,words(16)); + const char* edge = "texpage-P8-0EDB68960F55AEF6-9CBCDEA9305D7528-128x256-0-0-2x2-P0-255.png"; + png(root/"edge"/"replacements"/edge); + auto pack = create(root/"edge"); auto q = query(vram,128,0,1,0,1,0,1,1008,400); DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"full P8 edge CLUT hashes remaining 16 words"); + vram[400*1024] = 42; duck_texture_pack_invalidate(pack.get(),0,400,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"full P8 CLUT hash does not wrap into row start"); + const char* partial = "texpage-P8-0EDB68960F55AEF6-3631D2A04EC85311-128x256-0-0-2x2-P8-11.png"; + png(root/"partial"/"replacements"/partial); + place(vram,512,400,16,words(16)); q.clut_x = 512; + pack = create(root/"partial"); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"reduced P8 min8-max11 hashes prefix4 per upstream pin"); + vram[400*1024+520] ^= 1; duck_texture_pack_invalidate(pack.get(),520,400,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"pinned prefix quirk ignores palette8 despite declared minimum8"); + vram[400*1024+512] ^= 1; duck_texture_pack_invalidate(pack.get(),512,400,1,1); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"reduced P8 prefix change stops replacement"); + q.clut_x = 1023; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"misaligned/edge reduced CLUT rejected"); +} + +void st_alias_duplicates(const fs::path& root) { + const char* ordinary = "texpage-C16-000CA76CB9B329DC-256x256-0-0-4x2.png"; + const char* st = "texpage-STC16-000CA76CB9B329DC-256x256-0-0-4x2.png"; + png(root/"paired"/"replacements"/ordinary); png(root/"paired"/"replacements"/st); + std::vector vram(1024*512); place(vram,64,0,4,words(8)); auto q = query(vram,64,0,2,0,3,0,1); + auto pack = create(root/"paired"); DuckTextureMatch match{}; + check(duck_texture_pack_match_draw(pack.get(),&q,0,&match) == HD_TEXTURE_LOOKUP_FOUND && !match.key.semitransparent, + "ordinary primitive prefers ordinary filename"); + check(duck_texture_pack_match_draw(pack.get(),&q,1,&match) == HD_TEXTURE_LOOKUP_FOUND && match.key.semitransparent, + "semitransparent primitive prefers ST filename"); + png(root/"alias"/"replacements"/"art"/"sample.png"); + fs::create_directories(root/"alias"); + std::ofstream config(root/"alias"/"config.yaml"); + config << "MaxVRAMWriteCoalesceWidth: 1\nAliases:\n " << std::string(ordinary).substr(0,std::strlen(ordinary)-4) + << ": |\n art/sample.png\n"; config.close(); + pack = create(root/"alias"); + check(duck_texture_pack_match_draw(pack.get(),&q,1,&match) == HD_TEXTURE_LOOKUP_FOUND && !match.key.semitransparent, + "bounded literal alias and opposite-convention fallback"); + DuckTexturePackInfo info{}; duck_texture_pack_get_info(pack.get(),&info); + check(info.ignored_count && std::strstr(info.diagnostic,"MaxVRAMWriteCoalesceWidth"),"unsupported config option explicitly diagnosed"); + png(root/"duplicate"/"replacements"/"a"/ordinary); png(root/"duplicate"/"replacements"/"b"/ordinary); + pack = create(root/"duplicate"); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_AMBIGUOUS,"recursive duplicate keys fall back deterministically"); + q.u_first = 254; q.u_last = 1; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_NONE,"wrapped UV interval falls back"); +} + +void failed_payloads(const fs::path& root) { + const char* name = "texpage-C16-000CA76CB9B329DC-256x256-0-0-4x2.png"; + png(root/"budget"/"replacements"/name); + std::vector vram(1024*512); place(vram,64,0,4,words(8)); auto q = query(vram,64,0,2,0,3,0,1); + auto pack = create(root/"budget"); DuckTextureMatch match{}; + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"over-budget payload identity remains valid metadata"); + duck_texture_pack_set_decode_budget(pack.get(),8); + check(decode_fails(pack.get(),match.entry_id),"over-budget decoded image fails before pixel allocation"); + fs::create_directories(root/"malformed"/"replacements"); + std::ofstream invalid(root/"malformed"/"replacements"/name,std::ios::binary); invalid << "not PNG"; invalid.close(); + pack = create(root/"malformed"); + check(duck_texture_pack_match(pack.get(),&q,&match) == HD_TEXTURE_LOOKUP_FOUND,"malformed PNG identity can be indexed without render-thread decode"); + check(decode_fails(pack.get(),match.entry_id),"malformed PNG fails asynchronously for native fallback"); + const fs::path oversized = root/"dimension"/"replacements"/name; + png(oversized); + std::ifstream file(oversized,std::ios::binary); std::vector bytes((std::istreambuf_iterator(file)),{}); file.close(); + /* Forge a valid IHDR width above the decode dimension bound. Keeping the + * tiny original IDAT makes the fixture inexpensive and verifies rejection + * using image metadata before an enormous allocation could happen. */ + bytes[16] = 0; bytes[17] = 0; bytes[18] = 0x23; bytes[19] = 0x28; /* 9000 */ + const uint32_t crc = png_crc_update(0xffffffffu,bytes.data()+12,17)^0xffffffffu; + for (unsigned i = 0; i < 4; ++i) bytes[29+i] = static_cast(crc >> (24-8*i)); + std::ofstream changed(oversized,std::ios::binary|std::ios::trunc); + changed.write(reinterpret_cast(bytes.data()),bytes.size()); changed.close(); + pack = create(root/"dimension"); duck_texture_pack_match(pack.get(),&q,&match); + check(decode_fails(pack.get(),match.entry_id),"oversized PNG IHDR fails within native fallback bounds"); +} + +void dumping(const fs::path& root) { + auto pack = create(root); std::vector vram(1024*512); + vram[0] = 0x3210; vram[400*1024+1] = 0x001f; vram[400*1024+2] = 0x8000; vram[400*1024+3] = 0xffff; + auto q = query(vram,0,0,0,0,3,0,0,0,400); char error[512]{}; + check(duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"queue native ST page dump"); + check(duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)) == HD_TEXTURE_LOOKUP_NONE,"dump session deduplicates without filesystem access"); + pack.reset(); /* required to finish queued background writes */ + const fs::path file = root/"dumps"/"texpage-STP4-A92BF769517E0361-59786C612FB45363-64x256-0-0-4x1-P0-15.png"; + check(fs::is_regular_file(file),"fixed independent hash filename published at worker shutdown"); + int width = 0, height = 0, channels = 0; + std::ifstream encoded_file(file,std::ios::binary); + std::vector encoded((std::istreambuf_iterator(encoded_file)),{}); + auto* image = encoded.empty() ? nullptr : stbi_load_from_memory(encoded.data(),static_cast(encoded.size()),&width,&height,&channels,4); + constexpr uint8_t expected[] = {0,0,0,0,255,0,0,255,0,0,0,128,255,255,255,128}; + check(image && width == 4 && height == 1 && !std::memcmp(image,expected,sizeof(expected)),"dump decodes native indices/CLUT and ST alpha"); + if (image) stbi_image_free(image); + std::ofstream owner(file,std::ios::binary|std::ios::trunc); owner << "owner edit"; owner.close(); + pack = create(root); duck_texture_pack_dump_draw(pack.get(),&q,1,error,sizeof(error)); pack.reset(); + std::ifstream edited(file,std::ios::binary); std::string payload((std::istreambuf_iterator(edited)),{}); + check(payload == "owner edit","existing owner dump is never overwritten"); + pack = create(root/"upload"); + duck_texture_pack_track_upload(pack.get(),0,0,1,1,vram.data(),vram.size()); + check(duck_texture_pack_dump_draw(pack.get(),&q,0,error,sizeof(error)) == HD_TEXTURE_LOOKUP_FOUND,"tracked source emits texupload dump"); + pack.reset(); + check(fs::is_regular_file(root/"upload"/"dumps"/"texupload-P4-A92BF769517E0361-59786C612FB45363-1x1-0-0-4x1-P0-15.png"), + "upload dump retains original source dimensions"); + /* PNG fixtures remain independent of the VRAM pointer used by replacement + * matching, so dumping never reads already-replaced pixels. */ +} +} // namespace + +int main() { + const fs::path root = fs::temp_directory_path()/fs::u8path("psx-duck-texture-\xCE\xA9-"+ + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + try { + fixed_hash_vectors(); names(); page_and_decode(root/"page"); upload(root/"upload"); reload_tracking(root/"reload"); + palettes(root/"palettes"); st_alias_duplicates(root/"keys"); failed_payloads(root/"invalid"); dumping(root/"dump"); + /* Every artifact is created beneath this fresh, explicitly named test + * root. No user-provided paths participate in recursive cleanup. */ + fs::remove_all(root); + } catch (const std::exception& e) { check(false,e.what()); } + if (failures) { std::fprintf(stderr,"test_duckstation_texture_pack: %d failure(s)\n",failures); return 1; } + std::puts("PASS: DuckStation format, independent hashes, native matching, decode and dumps"); + return 0; +} diff --git a/runtime/tests/test_gl_hd_texture_pack.c b/runtime/tests/test_gl_hd_texture_pack.c new file mode 100644 index 00000000..d5b65cf0 --- /dev/null +++ b/runtime/tests/test_gl_hd_texture_pack.c @@ -0,0 +1,251 @@ +/* Real GL + real format/decoder + renderer facade. Source-owned fixtures. */ +#define PSX_TEST_HD_TEXTURE_PACK 1 +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#include "../third_party/stb_image.h" +#define main scale_fixture_main +#include "test_gl_scale_invariance.c" +#undef main +#include "gpu_render.c" +#include "duckstation_texture_pack.h" +#include "hd_texture_pack.h" +#include "png_write.h" + +const GpuRenderBackend* vk_backend_get(void) { return NULL; } +void present_shot_done(int ok) { (void)ok; } +int host_osd_needs_present(void) { return 0; } +int psx_present_vsync_owns_cadence(void) { return 0; } +void latency_ring_mark(LatencyStage stage) { (void)stage; } +void psx_host_sleep_ms(unsigned ms) { SDL_Delay(ms); } +int present_shot_take(char* out,int n) { (void)out; (void)n; return 0; } +int host_osd_image(const uint32_t** p,int* w,int* h) { (void)p; (void)w; (void)h; return 0; } +int host_osd_volume_image(const uint32_t** p,int* w,int* h) { return host_osd_image(p,w,h); } +void host_osd_present_done(void) {} +int psx_rewind_overlay_image(const uint32_t** p,int* w,int* h) { return host_osd_image(p,w,h); } +float psx_rewind_slide(void) { return 0; } +int psx_savestate_menu_overlay_image(const uint32_t** p,int* w,int* h) { return host_osd_image(p,w,h); } + +static uint16_t source_words[4*4]; +static uint16_t reference[1024*512]; +static uint8_t pixels[64*64*4]; +static const uint16_t texture_page=8u|(2u<<7); + +static void pack_png(const char* root,int st) { + DuckTextureKey key={0}; + key.source_hash=duck_texture_hash_words_le(source_words,16); + key.source_width_words=4; key.source_height=4; + key.width=4; key.height=4; key.kind=DUCK_TEXTURE_UPLOAD; + key.depth=HD_TEXTURE_DEPTH_16BPP; key.semitransparent=st; + char stem[256],path[2048]; + check(duck_texture_format_name(&key,stem,sizeof(stem)),"fixture filename"); + snprintf(path,sizeof(path),"%s/replacements/%s.png",root,stem); + uint8_t rgba[16*16*4]; + for(int y=0;y<16;++y) for(int x=0;x<16;++x) { + uint8_t* at=rgba+(y*16+x)*4; + at[0]=(x&1)?0:255; at[1]=0; at[2]=(x&1)?255:0; at[3]=255; + if(st) { + const uint8_t alpha[6]={0,127,128,242,243,255}; + at[3]=alpha[x%6]; + if(y>=8) at[0]=at[1]=at[2]=0; + } else if(y>=8) { + at[0]=at[1]=at[2]=0; + at[3]=(x&1)?128:127; + } + } + FILE* file=fopen(path,"wb"); + check(file!=NULL,"open fixture PNG"); + if(file) { check(png_write_rgba(file,rgba,16,16),"write fixture PNG"); fclose(file); } +} +static void state(void) { + gr_set_draw_area(0,0,1023,511); gr_set_draw_offset(0,0); + gr_set_mask_bits(0,0); gr_set_semi_transparency(0,0); + gr_set_texture_window(0); gr_set_color_modulation(128,128,128,1); +} +static void native_scene(void) { + state(); + gr_vram_transfer_in(512,0,4,4,source_words); + gr_fill_rect(0,0,64,64,0x4210); + gr_draw_textured_rect(16,16,4,4,0,0,0,0,texture_page); + for(int mode=0;mode<4;++mode) { + gr_set_semi_transparency(1,mode); + gr_draw_textured_rect(20+mode*5,20,4,4,0,0,0,0,texture_page); + } + gr_set_semi_transparency(0,0); + gr_set_mask_bits(1,0); gr_draw_flat_rect(40,40,8,8,0x001f); + gr_set_mask_bits(0,1); gr_draw_textured_rect(38,38,12,12,0,0,0,0,texture_page); + gr_set_mask_bits(0,0); + gr_set_texture_filter(1); + gr_set_perspective_triangle(1,1.0f,0.5f,0.25f); + gr_set_precise_triangle(1,(4<<16)+32768,30<<16,14<<16,30<<16,4<<16,38<<16); + gr_draw_textured_triangle(4,30,0,0,14,30,3,0,4,38,0,3,0,0,texture_page); + gr_set_texture_filter(0); + gr_draw_line(3,50,53,50,0x7fff); + gr_copy_rect(16,16,2,2,4,4); + uint16_t masked[4]={0x7c00,0x7c00,0x7c00,0x7c00}; + gr_set_mask_bits(0,1); gr_vram_transfer_in(40,40,4,1,masked); + gr_set_mask_bits(0,0); + gr_fill_rect(1022,510,4,4,0x1234); +} +static void wait_ready(int st) { + const int bounds[4]={0,0,3,3}; + int ready=0; + for(int i=0;i<2000 && !ready;++i) { + GpuHdTextureImage image={0}; + ready=gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,st,&image); + gpu_hd_textures_release_image(&image); + if(!ready) SDL_Delay(1); + } + check(ready,"async replacement decode completed"); +} +static void capture(void) { + uint32_t argb[64*64]; + check(gl_renderer_capture_display_hires(argb,64*4,0,0,16,16)==64*64,"high-resolution capture"); + for(int i=0;i<64*64;++i) { + pixels[i*4]=(uint8_t)(argb[i]>>16); pixels[i*4+1]=(uint8_t)(argb[i]>>8); + pixels[i*4+2]=(uint8_t)argb[i]; pixels[i*4+3]=(uint8_t)(argb[i]>>24); + } +} +static const uint8_t* sample(int x,int y) { return pixels+(y*64+x)*4; } + +int main(int argc,char** argv) { + if(argc==2 && !strcmp(argv[1],"--native-baseline")) { + char* native_args[]={"hd-baseline","4","twin","1"}; + return scale_fixture_main(4,native_args); + } + if(argc!=2) return 2; + for(int i=0;i<16;++i) source_words[i]=i%4==3?0:i%4==2?0x8000:i%4==1?0x83e0:0x03e0; + pack_png(argv[1],0); pack_png(argv[1],1); + gr_set_backend(GR_BACKEND_SOFTWARE); gr_init(reference); + sw_set_faithful_authority(1); native_scene(); sw_set_faithful_authority(0); + if(SDL_Init(SDL_INIT_VIDEO)!=0) return 2; + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION,3); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION,3); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK,SDL_GL_CONTEXT_PROFILE_CORE); + SDL_Window* win=SDL_CreateWindow("HD pack regression",0,0,128,128,SDL_WINDOW_OPENGL|SDL_WINDOW_HIDDEN); + if(!win) return 2; + gr_set_backend(GR_BACKEND_OPENGL); gr_init(vram); gr_set_scale(4); + char error[512]={0}; + check(gpu_hd_textures_configure(argv[1],1,1,error,sizeof(error)),error); + gl_renderer_set_swap_interval(0); + if(!gl_renderer_init_context(win)) return 2; + if(s_hiw) check(hiw_ensure(0,64)!=NULL,"high-resolution window allocated before draws"); + state(); gr_vram_transfer_in(512,0,4,4,source_words); + wait_ready(0); wait_ready(1); + check(gpu_hd_textures_reload(error,sizeof(error)),"live pack reload succeeds"); + /* No new upload: a reload must retain identities of resident textures. */ + wait_ready(0); wait_ready(1); + native_scene(); gl_renderer_sync_cpu(); + check(memcmp(vram,reference,sizeof(vram))==0,"HD native VRAM matches faithful software across all operations"); + uint16_t readback[64*64]; + gr_vram_transfer_out(0,0,64,64,readback); + int unchanged=1; + for(int y=0;y<64;++y) for(int x=0;x<64;++x) unchanged&=readback[y*64+x]==reference[y*1024+x]; + check(unchanged,"guest GPUREAD stays native with replacements"); + + /* Source rectangle is four native texels, replacement is sixteen. The + * first native texel contains alternating red and blue HD columns. */ + state(); gr_fill_rect(0,0,16,16,0x03e0); + gr_draw_textured_rect(4,4,4,4,0,0,0,0,texture_page); capture(); + const uint8_t* a=sample(16,16); const uint8_t* b=sample(17,16); + check(a[0]>200 && a[2]<20 && b[2]>200 && b[0]<20, + "replacement detail survives inside one native texel at 4x"); + check(vram[4*1024+4]==0x03e0,"presentation red/blue never enters guest VRAM"); + check(sample(16,24)[1]>200,"non-ST alpha127 cuts out black"); + check(sample(17,24)[0]==0 && sample(17,24)[1]==0 && sample(17,24)[2]==0, + "non-ST alpha128 retains opaque black"); + + /* ST alpha bytes encode PSX classes, including alpha0 with colored RGB. + * Mode 1 is additive, so the green destination survives STP texels. */ + state(); gr_fill_rect(0,0,16,16,0x03e0); gr_set_semi_transparency(1,1); + gr_draw_textured_rect(4,4,4,4,0,0,0,0,texture_page); capture(); + check(sample(16,16)[0]>200 && sample(16,16)[1]>200,"ST colored alpha0 stays semitransparent"); + check(sample(19,16)[1]>200,"ST alpha242 stays semitransparent"); + check(sample(17,16)[1]>200 && sample(18,16)[1]>200,"ST alpha127 and alpha128 stay semitransparent"); + check(sample(20,16)[0]>200 && sample(20,16)[1]<20,"ST alpha243 is opaque"); + check(sample(21,16)[2]>200 && sample(21,16)[1]<20,"ST alpha255 is opaque"); + check(sample(16,24)[1]>200,"ST black RGBA zero cuts out"); + check(sample(20,24)[1]>200,"ST opaque black follows native-zero cutout"); + check(sample(17,24)[0]==0 && sample(17,24)[1]>200 && sample(17,24)[2]==0, + "ST black alpha127 remains occupied semitransparent black"); + gr_set_semi_transparency(0,0); + + /* A queued draw must consume its source before a later source overwrite. + * The next draw falls back because upload identity has been invalidated. */ + gr_draw_textured_rect(0,0,4,4,0,0,0,0,texture_page); + uint16_t overwrite[16]; for(int i=0;i<16;++i) overwrite[i]=0x7c00; + gr_vram_transfer_in(512,0,4,4,overwrite); + gr_draw_textured_rect(8,0,4,4,0,0,0,0,texture_page); capture(); + check(sample(0,0)[0]>200,"queued replacement retained before source overwrite"); + check(sample(32,0)[2]>200 && sample(32,0)[0]<20,"next draw sees new native source"); + check(vram[8]==0x7c00,"new source word retained natively"); + state(); gr_vram_transfer_in(512,0,4,4,source_words); + gr_draw_textured_rect(513,0,3,1,0,0,0,0,texture_page); + gr_vram_transfer_out(512,0,4,1,readback); + check(readback[0]==0x03e0 && readback[1]==0x03e0 && readback[2]==0x03e0 && readback[3]==0x03e0, + "self-overlap keeps sequential native texture reads"); + const int bounds[4]={0,0,3,3}; GpuHdTextureImage lease={0}; + check(!gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), + "self-overlap invalidates upload only after its draw"); + gpu_hd_textures_release_image(&lease); + gr_vram_transfer_in(512,0,4,4,source_words); + check(gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), + "upload is resident before an interrupted A0"); + gpu_hd_textures_release_image(&lease); + gr_vram_upload_begin(512,0,4,4); + check(!gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), + "A0 header invalidates identity even if payload is later aborted"); + gpu_hd_textures_release_image(&lease); + gr_vram_transfer_in(512,0,4,4,source_words); + gl_renderer_restage_vram_after_savestate(); + check(!gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), + "savestate restage cannot fabricate original upload identity"); + gpu_hd_textures_release_image(&lease); + gr_vram_transfer_in(512,0,4,4,source_words); + check(gpu_hd_textures_acquire_draw(texture_page,0,0,bounds,0,0,&lease), + "fresh upload restores identity after state load"); + gpu_hd_textures_release_image(&lease); + check(!gl_renderer_texture_banks_supported(),"texture banks explicitly refuse HD authority"); + check(gl_renderer_pass_unavailable()!=PSX_MOD_RENDER_PASS_READY,"passes refuse HD authority"); + check(gl_renderer_stereo_unavailable()!=PSX_MOD_RENDER_PASS_READY,"stereo refuses HD authority"); + GpuHdTextureDiag diag; gpu_hd_textures_get_diag(&diag); + check(diag.applied_draws>0,"replacement diagnostics record actual GL submissions"); + /* Preserve the established Beetle numeric-key path with source origin + * offset, native cutout and native STP authority. */ + char beetle_root[2048],beetle_png[2304],hashes[2304]; + snprintf(beetle_root,sizeof(beetle_root),"%s/beetle",argv[1]); + snprintf(hashes,sizeof(hashes),"%s/Hashes.ini",beetle_root); + FILE* hf=fopen(hashes,"wb"); if(hf) fclose(hf); + snprintf(beetle_png,sizeof(beetle_png),"%s/demo-texture-replacements/%x-0.png",beetle_root, + hd_texture_crc32_words_le(source_words,16)); + uint8_t beetle_rgba[16*16*4]; + for(int y=0;y<16;++y) for(int x=0;x<16;++x) { + uint8_t* at=beetle_rgba+(y*16+x)*4; + at[0]=x<4?255:0; at[1]=x>=8?255:0; at[2]=x>=4&&x<8?255:0; at[3]=255; + } + FILE* bf=fopen(beetle_png,"wb"); check(bf!=NULL,"Beetle PNG open"); + if(bf){check(png_write_rgba(bf,beetle_rgba,16,16),"Beetle PNG fixture");fclose(bf);} + check(gpu_hd_textures_configure(beetle_root,1,0,error,sizeof(error)),"Beetle session opens"); + state(); gr_vram_transfer_in(512,0,4,4,source_words); + wait_ready(0); + gr_vram_transfer_in(516,0,4,4,source_words); + gr_fill_rect(0,0,16,16,0x03e0); + gr_draw_textured_rect(4,4,2,4,5,0,0,0,texture_page); capture(); + check(sample(16,16)[2]>200 && sample(16,16)[0]<20,"Beetle partial draw keeps upload-relative UV origin"); + check(vram[4*1024+4]==0x03e0 && vram[517]==0x83e0, + "Beetle image preserves native draw RGB and source STP"); + gr_draw_textured_rect(8,4,4,4,4,0,0,0,texture_page); capture(); + check(sample(44,16)[1]>200,"Beetle native transparent zero still cuts out replacement"); + memcpy(reference,vram,sizeof(vram)); + gl_renderer_set_cpu_auth_dual(1); + gpu_hd_textures_shutdown(); + check(!s_hd_native_authority,"session teardown clears independent authority"); + check(gl_renderer_cpu_auth_dual(),"HD teardown preserves netplay authority"); + check(memcmp(reference,vram,sizeof(vram))==0,"teardown cannot read presentation pixels into native VRAM"); + gl_renderer_set_cpu_auth_dual(0); + check(gl_renderer_texture_banks_supported(),"session teardown restores backend capabilities"); + check(glGetError()==GL_NO_ERROR,"HD GL errors"); + printf("scale=%d checks=%d failures=%d applied=%llu dumps=%llu\n",gr_scale(),checks,failures, + (unsigned long long)diag.applied_draws,(unsigned long long)diag.dumped_textures); + gl_renderer_shutdown(); SDL_DestroyWindow(win); SDL_Quit(); + return failures?1:0; +} diff --git a/runtime/tests/test_gl_readback_region.c b/runtime/tests/test_gl_readback_region.c index 29153f73..b51e33a6 100644 --- a/runtime/tests/test_gl_readback_region.c +++ b/runtime/tests/test_gl_readback_region.c @@ -1,5 +1,6 @@ /* Original source-owned GL readback-coherence regression. No retail payload. */ #include "gpu_gl_renderer.c" +#include "gpu_hd_texture_stubs.inc" #include "mod_texture_banks.c" #include "gpu_timeline.c" uint64_t psx_cycle_count=0; diff --git a/runtime/tests/test_gl_scale_invariance.c b/runtime/tests/test_gl_scale_invariance.c index e0a13a97..c0b2f4f0 100644 --- a/runtime/tests/test_gl_scale_invariance.c +++ b/runtime/tests/test_gl_scale_invariance.c @@ -63,6 +63,7 @@ * only the authoritative surface, never the window's tiles. * Original source-owned scene; no retail payload. */ #include "gpu_gl_renderer.c" +#include "gpu_hd_texture_stubs.inc" #include "mod_texture_banks.c" uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t n,uint32_t a){(void)n;(void)a;return 0;} uint32_t psx_mod_read_word(uint32_t a){(void)a;return 0;} @@ -82,6 +83,8 @@ void psx_ws_dbg_gate_frame_snapshot(void){} void gpu_depth24_upload_span_reset(void){} /* Netplay unsplit view and forward-pass opt-in: off in these fixtures. */ int gpu_ws_netplay_local_viewport_width(void){return 0;} +int gpu_ws_background_requires_full_composite(void){return 0;} +void gpu_timeline_note(uint8_t kind,uint32_t a,uint32_t b){(void)kind;(void)a;(void)b;} int render_pass_netplay_enabled(void){return 0;} /* Guest clock the renderer's stereo-pair freshness reads; test stand-ins, as * in test_gl_readback_region.c. */ diff --git a/runtime/tests/test_gpu_gp0_history_guard.py b/runtime/tests/test_gpu_gp0_history_guard.py index dd77a53c..4a44c74d 100644 --- a/runtime/tests/test_gpu_gp0_history_guard.py +++ b/runtime/tests/test_gpu_gp0_history_guard.py @@ -28,6 +28,7 @@ DRIVER_C = r""" #include extern uint64_t s_frame_count; +extern int test_upload_begins, test_upload_commits, test_upload_rect[4]; uint32_t gpu_get_opcode_count(uint8_t op); uint64_t gpu_get_gp0_count(void); void gpu_get_gp0_stats(uint64_t* nop, uint64_t* fill, uint64_t* draw, @@ -184,6 +185,28 @@ int main(int argc, char **argv) { copy_dumped > 0 ? copy_entries[0].csp : 0u, poly_dumped, poly_dumped > 0 ? poly_entries[0].opcode : 0u, poly_dumped > 0 ? poly_entries[0].cmd[0] : 0u); + /* Native upload identity is invalidated at the header, even when GP1 + * aborts a partially written payload. Mask rules stay in gpu.c and bulk + * commits carry only the final native words. */ + gpu_write_gp1(0x00000000u); + int begins = test_upload_begins, commits = test_upload_commits; + gpu_write_gp0(0xA0000000u); gpu_write_gp0((511u<<16)|1023u); + gpu_write_gp0((2u<<16)|4u); + if (test_upload_begins != begins+1 || test_upload_rect[0]!=1023 || + test_upload_rect[1]!=511 || test_upload_rect[2]!=4 || test_upload_rect[3]!=2) return 3; + gpu_write_gp0(0x33332222u); + gpu_write_gp1(0x01000000u); + if (test_upload_commits!=commits || gpu_vram_peek(1023,511)!=0x2222u || + gpu_vram_peek(0,511)!=0x3333u) return 4; + gpu_write_gp0(0xE6000001u); + gpu_write_gp0(0xA0000000u); gpu_write_gp0((100u<<16)|32u); + gpu_write_gp0((1u<<16)|2u); gpu_write_gp0(0x00020001u); + if (gpu_vram_peek(32,100)!=0x8001u || gpu_vram_peek(33,100)!=0x8002u) return 5; + gpu_write_gp0(0xE6000002u); + gpu_write_gp0(0xA0000000u); gpu_write_gp0((100u<<16)|32u); + gpu_write_gp0((1u<<16)|2u); gpu_write_gp0(0x12341234u); + if (gpu_vram_peek(32,100)!=0x8001u || gpu_vram_peek(33,100)!=0x8002u || + test_upload_commits!=commits+2 || test_upload_begins!=begins+3) return 6; return 0; } """ @@ -191,6 +214,7 @@ int main(int argc, char **argv) { STUBS_C = r""" #include "gpu.h" #include "gpu_render.h" +#include "gpu_gl_renderer.h" #include "cpu_state.h" #include "pgxp.h" #include "color_lut.h" @@ -250,6 +274,19 @@ void ws_ui_group_assign(WsUiGroupItem *items, size_t count, int32_t ws_ui_anchor_for_bounds(int32_t x, int32_t width, int32_t display_width) { (void)x; (void)width; return display_width / 2; } int gte_geometry_correction_enabled(void) { return 0; } +int gte_nclip_native_wide_sign(int32_t value,int* sign) { (void)value; (void)sign; return 0; } +int gte_nclip_native_wide_previous_sign(int32_t value,int* sign) { (void)value; (void)sign; return 0; } +void pgxp_set_projection_tracking(int enabled) { (void)enabled; } +int pgxp_projection_tracking(void) { return 0; } +int pgxp_load_projection(uint32_t address,uint32_t word,PGXPProjection* projection) { + (void)address; (void)word; (void)projection; return 0; +} +void gl_renderer_note_wide_triangle_recovery(int enabled) { (void)enabled; } +int gl_renderer_projective_supported(void) { return 0; } +void gl_renderer_draw_projected_triangle(const PSXProjectedVertex vertices[3],uint16_t page, + uint16_t cx,uint16_t cy,int raw,int semi,int perspective) { + (void)vertices; (void)page; (void)cx; (void)cy; (void)raw; (void)semi; (void)perspective; +} void pgxp_set_enabled(int enabled) { (void)enabled; } int pgxp_get_gte_sxy_checked(uint32_t slot, uint32_t packed, int require_valid, int32_t *x16, int32_t *y16) @@ -308,9 +345,15 @@ int ws_active(void) { return 0; } int ws_engaged(void) { return 0; } void gr_init(uint16_t *vram) { g_vram = vram; } +int test_upload_begins, test_upload_commits, test_upload_rect[4]; +void gr_vram_upload_begin(int x,int y,int w,int h) { + ++test_upload_begins; test_upload_rect[0]=x; test_upload_rect[1]=y; + test_upload_rect[2]=w; test_upload_rect[3]=h; +} uint16_t gr_vram_read(int x, int y) { return g_vram[(y & 511) * 1024 + (x & 1023)]; } void gr_vram_transfer_in(int x, int y, int w, int h, const uint16_t *pixels) { + ++test_upload_commits; for (int yy = 0; yy < h; yy++) for (int xx = 0; xx < w; xx++) g_vram[((y + yy) & 511) * 1024 + ((x + xx) & 1023)] = pixels[yy * w + xx]; diff --git a/runtime/tests/test_mod_resident.cpp b/runtime/tests/test_mod_resident.cpp index e4a6fc62..57ab7198 100644 --- a/runtime/tests/test_mod_resident.cpp +++ b/runtime/tests/test_mod_resident.cpp @@ -7,6 +7,7 @@ #include "psx_sha256.h" #include "psx_lobby_client.h" #include "gpu.h" +#include "gpu_hd_textures.h" #include #include @@ -44,6 +45,12 @@ extern "C" void psx_host_write_word(uint32_t a, uint32_t v) { psx_write_word(a, extern "C" uint32_t psx_mod_memory_alloc(uint32_t, uint32_t) { return 0; } extern "C" uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t, uint32_t) { return 0; } extern "C" void psx_ram_reset_size_request(void) {} +extern "C" void gpu_hd_textures_shutdown(void) {} +extern "C" int gpu_hd_textures_configure(const char*, int, int, char*, size_t) { return 1; } +extern "C" void gpu_hd_textures_set_dump_enabled(int) {} +extern "C" int gpu_hd_textures_active(void) { return 1; } +extern "C" void gpu_hd_textures_get_diag(GpuHdTextureDiag* out) { *out = {}; } +extern "C" int gpu_hd_textures_reload(char*, size_t) { return 1; } extern "C" void psx_projection_reset_session(void) {} extern "C" void gpu_ws_set_native_scene_predicate(int (*)(void)) {} extern "C" int psx_ws_x_margin(void) { return 0; } diff --git a/runtime/tests/test_mod_runtime.cpp b/runtime/tests/test_mod_runtime.cpp index a3f941a6..02b069b6 100644 --- a/runtime/tests/test_mod_runtime.cpp +++ b/runtime/tests/test_mod_runtime.cpp @@ -5,6 +5,7 @@ #include "psx_lobby_client.h" #include "gpu.h" +#include "gpu_hd_textures.h" #include "cpu_state.h" #include @@ -82,6 +83,36 @@ extern "C" uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t, uint32_t) { return 0; } extern "C" void psx_ram_reset_size_request(void) {} +static int hd_shutdown_calls, hd_configure_calls, hd_dump_value; +static int hd_replacements_value, hd_reload_calls; +static std::string hd_root; +extern "C" void gpu_hd_textures_shutdown(void) { ++hd_shutdown_calls; hd_root.clear(); } +extern "C" int gpu_hd_textures_configure(const char* root, int replacements, int dump, + char*, size_t) { + ++hd_configure_calls; + hd_root = root; + hd_replacements_value = replacements; + hd_dump_value = dump; + return 1; +} +extern "C" void gpu_hd_textures_set_dump_enabled(int enabled) { + hd_dump_value = enabled; +} +extern "C" int gpu_hd_textures_active(void) { + return !hd_root.empty() && (hd_replacements_value || hd_dump_value); +} +extern "C" void gpu_hd_textures_get_diag(GpuHdTextureDiag* out) { + *out = {}; + out->root = hd_root.c_str(); +} +extern "C" int gpu_hd_textures_reload(char*, size_t) { ++hd_reload_calls; return 1; } +static void test_hd_activation(void) { + char replacements[16], dump[16]; + if (!psx_mod_current_option_value("replacements", replacements, sizeof(replacements)) || + !psx_mod_current_option_value("dump", dump, sizeof(dump))) { ++failures; return; } + if (!psx_mod_set_hd_texture_pack("pack", std::string(replacements) == "true", + std::string(dump) == "true")) ++failures; +} extern "C" void psx_projection_reset_session(void) {} extern "C" void gpu_ws_set_native_scene_predicate(int (*)(void)) {} extern "C" int psx_ws_x_margin(void) { return 0; } @@ -1325,6 +1356,71 @@ int main() { check(psx_mod_read_disc_file("S0/LEVEL.NSF", nullptr, 0, &bytes) && bytes == 69400576u, "files grown past 64 MiB keep their effective size"); } + { + const fs::path hd_mods = root / "hd-mods"; + write_text(hd_mods / "bundled" / "runtime.hd" / "1.0.0" / "manifest.toml", + "format_version = 5\nid = \"runtime.hd\"\nversion = \"1.0.0\"\n" + "name = \"HD\"\nresolver = \"declarative\"\nsave_compatibility = \"shared\"\n" + "[[target]]\ngame_id = \"SCUS-94236\"\n" + "[[feature]]\nid = \"textures\"\nname = \"HD\"\ndefault_enabled = false\n" + "[[resource]]\nfeature = \"textures\"\nid = \"pack\"\nlabel = \"Pack\"\n" + "format = \"directory\"\nrequired = true\n" + "[[option]]\nfeature = \"textures\"\nid = \"replacements\"\n" + "label = \"Load\"\ntype = \"boolean\"\ndefault = \"true\"\n" + "[[option]]\nfeature = \"textures\"\nid = \"dump\"\n" + "label = \"Dump\"\ntype = \"boolean\"\ndefault = \"false\"\n" + "[[plugin]]\nfeature = \"textures\"\nid = \"psx.hd-textures\"\n"); + write_text(hd_mods / "state.toml", + "format_version = 2\n[[feature]]\npackage_id = \"runtime.hd\"\n" + "id = \"textures\"\nenabled = true\n[feature.values]\ndump = \"true\"\n"); + check(psx_mod_register_activation_plugin("psx.hd-textures", test_hd_activation), + "HD activation plugin registers"); + check(PSXRecompV4::mod_runtime_initialize(hd_mods, "SCUS-94236", 0, {}, &error), + "HD mods initialize"); + const fs::path expected = fs::absolute(hd_mods / "texture-packs" / "SCUS-94236"); + check(fs::is_directory(expected / "dumps") && fs::is_directory(expected / "replacements"), + "default HD directories exist before first launch, even without dumping"); + check(PSXRecompV4::mod_runtime_commit(stock_path, &error), + "default pack resource commits without manual folder selection"); + const int shutdown_before = hd_shutdown_calls; + mod_runtime_activate_plugins(); + check(hd_shutdown_calls == shutdown_before + 1 && hd_configure_calls == 1 && + hd_root == expected.string() && hd_replacements_value == 1 && hd_dump_value == 1, + "HD activation clears previous session and reads owning committed options"); + char value[16] = "old"; + check(!psx_mod_current_option_value("dump", value, sizeof(value)) && value[0] == '\0', + "owning option context does not escape the trusted callback"); + check(!psx_mod_set_hd_texture_pack("pack", 1, 1), + "pack configuration requires the trusted resource callback context"); + hd_replacements_value = 0; + check(psx_mod_set_hd_texture_dump(0) && hd_dump_value == 0, + "live dump control forwards the requested switch"); + check(!gpu_hd_textures_active() && psx_mod_set_hd_texture_dump(1) && hd_dump_value == 1, + "dump-only session can turn capture back on after turning it off"); + check(psx_mod_reload_hd_texture_pack() && hd_reload_calls == 1, + "pack reload forwards to renderer"); + check(PSXRecompV4::mod_runtime_clear_for_netplay(&error), "HD plan clears"); + mod_runtime_activate_plugins(); + check(hd_shutdown_calls == shutdown_before + 2 && hd_configure_calls == 1, + "disabled HD feature leaves no previous session pack active"); + check(!psx_mod_set_hd_texture_dump(1), "live dump control refuses an unconfigured session"); + const fs::path custom = hd_mods / fs::u8path(u8"chosen pack # % \u65e5\u672c"); + fs::create_directories(custom); + write_text(hd_mods / "state.toml", + "format_version = 2\n[[feature]]\npackage_id = \"runtime.hd\"\n" + "id = \"textures\"\nenabled = true\n[feature.resources]\npack = \"" + + custom.generic_u8string() + "\"\n"); + check(PSXRecompV4::mod_runtime_initialize(hd_mods, "SCUS-94236", 0, {}, &error) && + PSXRecompV4::mod_runtime_commit(stock_path, &error), + "chosen Unicode pack path resolves and persists through commit"); + mod_runtime_activate_plugins(); + check(hd_root == custom.generic_u8string(), "renderer receives the chosen folder as UTF-8"); + check(PSXRecompV4::mod_runtime_initialize(hd_mods, "SCUS-94236", 0, {}, &error) && + PSXRecompV4::mod_runtime_commit(stock_path, &error), + "committed Unicode pack path reloads without corruption"); + mod_runtime_activate_plugins(); + check(hd_root == custom.generic_u8string(), "Unicode pack root survives the complete session cycle"); + } fs::remove_all(root, ec); if (failures) return 1; std::cout << "mod runtime tests passed\n"; diff --git a/runtime/third_party/xxhash.LICENSE b/runtime/third_party/xxhash.LICENSE new file mode 100644 index 00000000..e4c5da72 --- /dev/null +++ b/runtime/third_party/xxhash.LICENSE @@ -0,0 +1,26 @@ +xxHash Library +Copyright (c) 2012-2021 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/runtime/third_party/xxhash.h b/runtime/third_party/xxhash.h new file mode 100644 index 00000000..78fc2e8d --- /dev/null +++ b/runtime/third_party/xxhash.h @@ -0,0 +1,7238 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2023 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((__unused__)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((__const__)) +# define XXH_PUREF __attribute__((__pure__)) +# define XXH_MALLOCF __attribute__((__malloc__)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 3 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + * @see @ref streaming_example "Streaming Example" + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((__noescape__)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + * @see @ref streaming_example "Streaming Example" + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Unless set explicitly, determined automatically. + */ +# define XXH_SCALAR 0 /*!< Portable scalar version */ +# define XXH_SSE2 1 /*!< SSE2 for Pentium 4, Opteron, all x86_64. */ +# define XXH_AVX2 2 /*!< AVX2 for Haswell and Bulldozer */ +# define XXH_AVX512 3 /*!< AVX512 for Skylake and Icelake */ +# define XXH_NEON 4 /*!< NEON for most ARMv7-A, all AArch64, and WASM SIMD128 */ +# define XXH_VSX 5 /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +# define XXH_SVE 6 /*!< SVE for some ARMv8-A and ARMv9-A */ +# define XXH_LSX 7 /*!< LSX (128-bit SIMD) for LoongArch64 */ + + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + * @see @ref streaming_example "Streaming Example" + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t acc[4]; /*!< Accumulator lanes */ + unsigned char buffer[16]; /*!< Internal buffer for partial reads. */ + XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t acc[4]; /*!< Accumulator lanes */ + unsigned char buffer[32]; /*!< Internal buffer for partial reads.. */ + XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# define XXH_ALIGN(n) _Alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either: + * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); + +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The memory segment to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret used to alter hash result predictably. + * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN) + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(): contract is the same. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(). Contract is identical. + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(). Contract is identical. + * + * Note: there was a bug in an earlier version of this function (<= v0.8.2) + * that would make it generate an incorrect hash value + * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX + * and @p secret is different from XXH3_generateSecret_fromSeed(). + * As stated in the contract, the correct hash result must be + * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX. + * Results generated by this older version are wrong, hence not comparable. + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((__unused__)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__)) +# define XXH_NO_INLINE static __attribute__((__noinline__)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if defined(XXH_INLINE_ALL) +# define XXH_STATIC XXH_FORCE_INLINE +#else +# define XXH_STATIC static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((__aligned__(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +#elif XXH_HAS_BUILTIN(__builtin_stdc_rotate_left) +# define XXH_rotl32 __builtin_stdc_rotate_left +# define XXH_rotl64 __builtin_stdc_rotate_left +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Sets up the initial accumulator state for XXH32(). + */ +XXH_FORCE_INLINE void +XXH32_initAccs(xxh_u32 *acc, xxh_u32 seed) +{ + XXH_ASSERT(acc != NULL); + acc[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + acc[1] = seed + XXH_PRIME32_2; + acc[2] = seed + 0; + acc[3] = seed - XXH_PRIME32_1; +} + +/*! + * @internal + * @brief Consumes a block of data for XXH32(). + * + * @return the end input pointer. + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH32_consumeLong( + xxh_u32 *XXH_RESTRICT acc, + xxh_u8 const *XXH_RESTRICT input, + size_t len, + XXH_alignment align +) +{ + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + XXH_ASSERT(acc != NULL); + XXH_ASSERT(input != NULL); + XXH_ASSERT(len >= 16); + do { + acc[0] = XXH32_round(acc[0], XXH_get32bits(input)); input += 4; + acc[1] = XXH32_round(acc[1], XXH_get32bits(input)); input += 4; + acc[2] = XXH32_round(acc[2], XXH_get32bits(input)); input += 4; + acc[3] = XXH32_round(acc[3], XXH_get32bits(input)); input += 4; + } while (input < limit); + + return input; +} + +/*! + * @internal + * @brief Merges the accumulator lanes together for XXH32() + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_mergeAccs(const xxh_u32 *acc) +{ + XXH_ASSERT(acc != NULL); + return XXH_rotl32(acc[0], 1) + XXH_rotl32(acc[1], 7) + + XXH_rotl32(acc[2], 12) + XXH_rotl32(acc[3], 18); +} + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + xxh_u32 acc[4]; + XXH32_initAccs(acc, seed); + + input = XXH32_consumeLong(acc, input, len, align); + + h32 = XXH32_mergeAccs(acc); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + XXH32_initAccs(statePtr->acc, seed); + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + XXH_ASSERT(state->bufferedSize < sizeof(state->buffer)); + if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + { const xxh_u8* xinput = (const xxh_u8*)input; + const xxh_u8* const bEnd = xinput + len; + + if (state->bufferedSize) { /* non-empty buffer: complete first */ + XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); + xinput += sizeof(state->buffer) - state->bufferedSize; + /* then process one round */ + (void)XXH32_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); + state->bufferedSize = 0; + } + + XXH_ASSERT(xinput <= bEnd); + if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { + /* Process the remaining data */ + xinput = XXH32_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); + } + + if (xinput < bEnd) { + /* Copy the leftover to the tmp buffer */ + XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); + state->bufferedSize = (unsigned)(bEnd-xinput); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH32_mergeAccs(state->acc); + } else { + h32 = state->acc[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, state->buffer, state->bufferedSize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((__aligned__(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Sets up the initial accumulator state for XXH64(). + */ +XXH_FORCE_INLINE void +XXH64_initAccs(xxh_u64 *acc, xxh_u64 seed) +{ + XXH_ASSERT(acc != NULL); + acc[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + acc[1] = seed + XXH_PRIME64_2; + acc[2] = seed + 0; + acc[3] = seed - XXH_PRIME64_1; +} + +/*! + * @internal + * @brief Consumes a block of data for XXH64(). + * + * @return the end input pointer. + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH64_consumeLong( + xxh_u64 *XXH_RESTRICT acc, + xxh_u8 const *XXH_RESTRICT input, + size_t len, + XXH_alignment align +) +{ + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + XXH_ASSERT(acc != NULL); + XXH_ASSERT(input != NULL); + XXH_ASSERT(len >= 32); + do { + /* reroll on 32-bit */ + if (sizeof(void *) < sizeof(xxh_u64)) { + size_t i; + for (i = 0; i < 4; i++) { + acc[i] = XXH64_round(acc[i], XXH_get64bits(input)); + input += 8; + } + } else { + acc[0] = XXH64_round(acc[0], XXH_get64bits(input)); input += 8; + acc[1] = XXH64_round(acc[1], XXH_get64bits(input)); input += 8; + acc[2] = XXH64_round(acc[2], XXH_get64bits(input)); input += 8; + acc[3] = XXH64_round(acc[3], XXH_get64bits(input)); input += 8; + } + } while (input < limit); + + return input; +} + +/*! + * @internal + * @brief Merges the accumulator lanes together for XXH64() + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_mergeAccs(const xxh_u64 *acc) +{ + XXH_ASSERT(acc != NULL); + { + xxh_u64 h64 = XXH_rotl64(acc[0], 1) + XXH_rotl64(acc[1], 7) + + XXH_rotl64(acc[2], 12) + XXH_rotl64(acc[3], 18); + /* reroll on 32-bit */ + if (sizeof(void *) < sizeof(xxh_u64)) { + size_t i; + for (i = 0; i < 4; i++) { + h64 = XXH64_mergeRound(h64, acc[i]); + } + } else { + h64 = XXH64_mergeRound(h64, acc[0]); + h64 = XXH64_mergeRound(h64, acc[1]); + h64 = XXH64_mergeRound(h64, acc[2]); + h64 = XXH64_mergeRound(h64, acc[3]); + } + return h64; + } +} + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +XXH_STATIC XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { /* Process a large block of data */ + xxh_u64 acc[4]; + XXH64_initAccs(acc, seed); + + input = XXH64_consumeLong(acc, input, len, align); + + h64 = XXH64_mergeAccs(acc); + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + XXH64_initAccs(statePtr->acc, seed); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + state->total_len += len; + + XXH_ASSERT(state->bufferedSize <= sizeof(state->buffer)); + if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + { const xxh_u8* xinput = (const xxh_u8*)input; + const xxh_u8* const bEnd = xinput + len; + + if (state->bufferedSize) { /* non-empty buffer => complete first */ + XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); + xinput += sizeof(state->buffer) - state->bufferedSize; + /* and process one round */ + (void)XXH64_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); + state->bufferedSize = 0; + } + + XXH_ASSERT(xinput <= bEnd); + if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { + /* Process the remaining data */ + xinput = XXH64_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); + } + + if (xinput < bEnd) { + /* Copy the leftover to the tmp buffer */ + XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); + state->bufferedSize = (unsigned)(bEnd-xinput); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH64_mergeAccs(state->acc); + } else { + h64 = state->acc[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, state->buffer, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__loongarch_sx) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# elif defined(__loongarch_sx) +# define XXH_VECTOR XXH_LSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_LSX /* lsx */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((__may_alias__)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +#if (XXH_VECTOR == XXH_LSX) +#define _LSX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + __m128i* const xacc = (__m128i *) acc; + const __m128i* const xinput = (const __m128i *) input; + const __m128i* const xsecret = (const __m128i *) secret; + + for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = __lsx_vld(xinput + i, 0); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = __lsx_vld(xsecret + i, 0); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); + // __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = __lsx_vmulwev_d_wu(data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = __lsx_vshuf4i_w(data_vec, _LSX_SHUFFLE(1, 0, 3, 2)); + __m128i const sum = __lsx_vadd_d(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = __lsx_vadd_d(product, sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + __m128i* const xacc = (__m128i*) acc; + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = __lsx_vreplgr2vr_w((int)XXH_PRIME32_1); + + for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = __lsx_vsrli_d(acc_vec, 47); + __m128i const data_vec = __lsx_vxor_v(acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = __lsx_vld(xsecret + i, 0); + __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = __lsx_vsrli_d(data_key, 32); + __m128i const prod_lo = __lsx_vmulwev_d_wu(data_key, prime32); + __m128i const prod_hi = __lsx_vmulwev_d_wu(data_key_hi, prime32); + xacc[i] = __lsx_vadd_d(prod_lo, __lsx_vslli_d(prod_hi, 32)); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_LSX) +#define XXH3_accumulate_512 XXH3_accumulate_512_lsx +#define XXH3_accumulate XXH3_accumulate_lsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_lsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH_PUREF XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +/* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + +static XXH_PUREF XXH64_hash_t +XXH3_finalizeLong_64b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 len) +{ + return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, len * XXH_PRIME64_1); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_64b(acc, (const xxh_u8*)secret, (xxh_u64)len); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to @align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_finalizeLong_64b(acc, secret, (xxh_u64)state->totalLen); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +static XXH_PUREF XXH128_hash_t +XXH3_finalizeLong_128b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, xxh_u64 len) +{ + XXH128_hash_t h128; + h128.low64 = XXH3_finalizeLong_64b(acc, secret, len); + h128.high64 = XXH3_mergeAccs(acc, secret + secretSize + - XXH_STRIPE_LEN - XXH_SECRET_MERGEACCS_START, + ~(len * XXH_PRIME64_2)); + return h128; +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_128b(acc, secret, secretSize, (xxh_u64)len); +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_128b(acc, secret, state->secretLimit + XXH_STRIPE_LEN, (xxh_u64)state->totalLen); + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->useSeed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n