diff --git a/CMakePresets.json b/CMakePresets.json index cdbea71e..f0173934 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -12,6 +12,7 @@ "CMAKE_CXX_COMPILER": "clang++", "CMAKE_C_FLAGS": "-march=x86-64-v3 -w", "CMAKE_CXX_FLAGS": "-march=x86-64-v3 -w", + "PYTHON_EXECUTABLE": "python", "REXSDK_DIR": "${sourceDir}/thirdparty/rexglue-sdk" }, "condition": { @@ -68,6 +69,26 @@ "inherits": "windows-base", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, + { + "name": "win-amd64-vulkan-relwithdebinfo", + "displayName": "Windows AMD64 Vulkan RelWithDebInfo", + "inherits": "windows-base", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "REXGLUE_USE_D3D12": "ON", + "REXGLUE_USE_VULKAN": "ON" + } + }, + { + "name": "win-amd64-vulkan-only-relwithdebinfo", + "displayName": "Windows AMD64 Vulkan-Only RelWithDebInfo", + "inherits": "windows-base", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "REXGLUE_USE_D3D12": "OFF", + "REXGLUE_USE_VULKAN": "ON" + } + }, { "name": "linux-amd64-relwithdebinfo", "displayName": "Linux AMD64 RelWithDebInfo", @@ -79,6 +100,8 @@ { "name": "win-amd64-debug", "configurePreset": "win-amd64-debug" }, { "name": "win-amd64-release", "configurePreset": "win-amd64-release" }, { "name": "win-amd64-relwithdebinfo", "configurePreset": "win-amd64-relwithdebinfo" }, + { "name": "win-amd64-vulkan-relwithdebinfo", "configurePreset": "win-amd64-vulkan-relwithdebinfo" }, + { "name": "win-amd64-vulkan-only-relwithdebinfo", "configurePreset": "win-amd64-vulkan-only-relwithdebinfo" }, { "name": "linux-amd64-debug", "configurePreset": "linux-amd64-debug" }, { "name": "linux-amd64-release", "configurePreset": "linux-amd64-release" }, { "name": "linux-amd64-relwithdebinfo", "configurePreset": "linux-amd64-relwithdebinfo" } diff --git a/ac6recomp_config.toml b/ac6recomp_config.toml index 5aeba91a..f1757327 100644 --- a/ac6recomp_config.toml +++ b/ac6recomp_config.toml @@ -10610,6 +10610,36 @@ name = "ac6PresentIntervalHook" registers = ["r10"] jump_address_on_true = 0x821EFF74 +# Cutscene clamp signal. +# +# sub_82184460 is CAce6DemoManager::Exec (the per-frame update of the in-engine +# cinematics system, NU::FW::IngameCinematics). It runs once per game frame ONLY +# while a cutscene is playing (the demo task is scheduled when a cutscene starts +# and removed when it ends), and drives the Sequencer that advances the cut +# timeline (sub_821848A0 -> off_823F9B28 vtable[101] step / [102] is-done). +# +# Firing this hook stamps a "cinematic ticked this frame" timestamp; the FPS +# unlock hooks (present-interval / delta-divisor / flip) suspend themselves while +# that stamp is fresh, so cutscenes render at the game's native ~30fps cadence +# and don't run at double speed. Gated at runtime by the ac6_cutscene_clamp CVar. +[[midasm_hook]] +address = 0x82184460 +name = "ac6CinematicTickHook" +registers = ["r3"] + +# Cutscene clamp signal (EM variant). +# +# sub_821856F8 is CX360DemoManagerEM::Exec - the per-frame update of the "Event +# Movie" in-engine cinematics manager (the DD manager above drives a different +# demo path; the in-mission story cutscenes go through EM). Same shape as the DD +# Exec: state 0 -> load (sub_821857B0), state 1 -> cinematic update sub_82186080 +# (accumulate delta -> step Sequencer off_823F9B28). Runs once per frame only +# while an EM cutscene is scheduled. Same clamp signal as the DD hook. +[[midasm_hook]] +address = 0x821856F8 +name = "ac6CinematicTickHook" +registers = ["r3"] + # AC6 PAC stream-worker dispatch probe. # # rex_sub_82343E18 is the PAC streamer's queue pump. At 0x82343E78 it issues a diff --git a/default.xex.i64 b/default.xex.i64 index 8547992e..73d79269 100644 Binary files a/default.xex.i64 and b/default.xex.i64 differ diff --git a/docs/ac6_asset_extraction_walkthrough.txt b/docs/ac6_asset_extraction_walkthrough.txt index 6ac405a2..f0ce68de 100644 --- a/docs/ac6_asset_extraction_walkthrough.txt +++ b/docs/ac6_asset_extraction_walkthrough.txt @@ -5,11 +5,12 @@ AC6 Asset Extraction Walkthrough Goal: go from a fresh clone of this repository to decoded AC6 asset files (textures, FHM containers, SWG metadata) on disk. -The recompiled binary patches the guest decompressor at runtime via a midasm -hook (see docs/ac6_extraction_roadmap.md). When the env var -AC6_DUMP_PAC_DECODED=1 is set, every PAC entry the game touches is written -to disk in already-decoded form. The asset pipeline then turns those raw -buffers into typed FHM children, NTXR textures, etc. +The PAC archives can be decoded fully offline. The extractor reads DATA.TBL, +pulls each DATA00/01.PAC entry, applies the AC6 mode-1 descramble + raw DEFLATE +path, and then turns the decoded buffers into typed FHM children, NTXR textures, +audio banks, SWG metadata, etc. Runtime dumps are still useful for assets the +game synthesizes or touches only through live workflows, but they are no longer +required for normal PAC extraction. -------------------------------------------------------------------------------- @@ -62,7 +63,7 @@ has nothing to capture. -------------------------------------------------------------------------------- -3. Run the game with PAC dumping enabled +3. Optional: run the game with PAC dumping enabled -------------------------------------------------------------------------------- Use the helper launcher from PowerShell at the repo root: @@ -83,7 +84,7 @@ Optional switches (only set these when you need them): Adds PPC back-chain stack=[...] traces on each PAC NtReadFile call. Useful for debugging the stream worker; not needed for routine runs. -Play long enough for the streamer to load the assets you care about. As a +This is optional. Play long enough for the streamer to load the assets you care about. As a rough guide: - Title screen + intro: enough for the boot/menu PACs. - One mission start: enough for that mission's PAC entries. @@ -94,7 +95,7 @@ When you are done, close the game window normally. -------------------------------------------------------------------------------- -4. Verify the decoded dumps +4. Optional: verify runtime decoded dumps -------------------------------------------------------------------------------- The dumper writes to (relative to the repo root): @@ -120,42 +121,52 @@ The first 4 bytes of any mode-1 dump should be 70,72,77,32 (ASCII "FHM "). 5. Run the asset extraction pipeline -------------------------------------------------------------------------------- -From the repo root, with the dumps in place: +From the repo root: python tools\run_ac6_asset_pipeline.py The driver runs four stages in order: - 1. extract_ac6_pac.py - Pulls the raw 126 entries directly out of DATA00/01.PAC offline. - Outputs to out/ac6_pac_extracted_raw/. + 1. extract_ac6_pac.py --decompress + Pulls all entries directly out of DATA00/01.PAC offline and decodes + mode-1 compressed entries. Outputs to out/ac6_pac_extracted_raw/. 2. extract_ac6_runtime_fhm.py - Walks every entry_*_mode*.bin in out/ac6_pac_runtime_dump/ and + Walks every decoded PAC blob in out/ac6_pac_extracted_raw/files/ and descends into FHM containers, writing typed children to out/ac6_runtime_fhm_typed/. - 3. parse_ac6_swg.py + 3. extract_ac6_mdlp_parts.py + Splits embedded MDLP NDXR mesh chunks into named part folders and writes + manifests with face counts, bounds, UV/color/normal counts, material + texture hashes, primitive format histograms, primary-assembly grouping, + and LOD duplicate classification. + + 4. parse_ac6_swg.py Parses the UI sprite/widget metadata (.swg children) into out/ac6_runtime_swg_parsed/. - 4. export_ac6_ntxr.py + 5. export_ac6_ntxr.py Converts NTXR texture entries into DDS/TGA in out/ac6_runtime_ntxr_exported/. -Override any output path with --raw-out, --typed-out, --swg-out, --ntxr-out. -Add --skip-pac-extract if you only want to re-process the runtime dumps. +Override any output path with --raw-out, --typed-out, --mdlp-out, --swg-out, +--ntxr-out. Add --skip-pac-extract if you only want to re-process existing +decoded PAC files. Add --include-runtime-dumps if you also want to merge entry_* +dumps from a live capture session. -------------------------------------------------------------------------------- 6. Where the output lives -------------------------------------------------------------------------------- - out/ac6_pac_runtime_dump/ Raw decoded buffers, one file per entry. - out/ac6_pac_extracted_raw/ 126 raw (mode-0) entries pulled offline. + out/ac6_pac_runtime_dump/ Optional runtime decoded buffers. + out/ac6_pac_extracted_raw/ Offline raw/decompressed PAC entries. out/ac6_runtime_fhm_typed/ FHM children classified by magic (NTXR textures, BFX/BSN audio banks, MDLP/NSXR models, SWG UI, etc.). + out/ac6_mdlp_parts/ MDLP packages copied with unique names, + split NDXR mesh parts, and mesh manifests. out/ac6_runtime_swg_parsed/ JSON metadata for UI sprites. out/ac6_runtime_ntxr_exported/ DDS/TGA files (one per texture entry). @@ -165,6 +176,9 @@ Add --skip-pac-extract if you only want to re-process the runtime dumps. -------------------------------------------------------------------------------- * "no entry_*_mode1_*.bin files appeared" + Runtime dumps are optional for the offline pipeline. This only matters if + you explicitly ran with --include-runtime-dumps or are debugging live + streamer behavior. - The game did not stream any compressed entries during the session. Boot further or load a mission and try again. - AC6_DUMP_PAC_DECODED was not set. Always launch via the helper script, @@ -190,8 +204,10 @@ Add --skip-pac-extract if you only want to re-process the runtime dumps. regardless of log level. * "extract_ac6_runtime_fhm.py reports 0 containers" - - The dump dir is empty or the files are still .compressed.bin. - Re-run with the hook fix above. + - Confirm extract_ac6_pac.py was run with --decompress and produced + out/ac6_pac_extracted_raw/files/DATA0x/compressed/*.decompressed.bin. + - If you are using --include-runtime-dumps, the dump dir may be empty or + still contain .compressed.bin files. Re-run with the hook fix above. * "log files rotate and the early dumper lines are gone" - At trace-level logging the rotating buffer fills in seconds. Do not @@ -209,7 +225,7 @@ Add --skip-pac-extract if you only want to re-process the runtime dumps. 8. Quick reference: env vars -------------------------------------------------------------------------------- - AC6_DUMP_PAC_DECODED=1 Required. Enables the dumper sink. + AC6_DUMP_PAC_DECODED=1 Optional. Enables the runtime dumper sink. AC6_TRACE_PAC_WORK_ITEMS=1 Optional. Lifts [fs] log category to info, enables L1/L2 streamer-worker probes. AC6_TRACE_PAC_STACKS=1 Optional. PPC back-chain on PAC NtReadFile. diff --git a/docs/ac6_asset_pipeline.md b/docs/ac6_asset_pipeline.md index a3133e7a..284dd5e9 100644 --- a/docs/ac6_asset_pipeline.md +++ b/docs/ac6_asset_pipeline.md @@ -5,7 +5,7 @@ This pipeline extracts and converts the AC6 assets we currently understand. It automates these stages: 1. `DATA.TBL` + `DATA00.PAC` / `DATA01.PAC` index extraction -2. Runtime PAC decode dump parsing +2. Offline mode-1 PAC decompression 3. Recursive `FHM` extraction 4. `SWG` UI metadata parsing 5. `NTXR` texture export @@ -45,7 +45,7 @@ extraction; it remains useful only for assets synthesized at runtime. - The game assets exist at: - `C:\ext\New folder\AC6_recomp\out\build\win-amd64-relwithdebinfo\assets` -- If you want runtime-decoded content included, you must already have: +- Optional: if you want runtime-synthesized content included, you must already have: - `C:\ext\New folder\AC6_recomp\out\ac6_pac_runtime_dump` To collect runtime dumps in future runs: @@ -74,8 +74,7 @@ python .\tools\run_ac6_asset_pipeline.py This uses the default paths: - Asset root: `out\build\win-amd64-relwithdebinfo\assets` -- Raw PAC output: `out\ac6_pac_extracted_raw` -- Runtime dump input: `out\ac6_pac_runtime_dump` +- Raw/decompressed PAC output: `out\ac6_pac_extracted_raw` - Typed FHM output: `out\ac6_runtime_fhm_typed` - SWG output: `out\ac6_runtime_swg_parsed` - Texture output: `out\ac6_runtime_ntxr_exported` @@ -83,7 +82,7 @@ This uses the default paths: The wrapper prints a final JSON summary with the current corpus totals, including: - PAC entries extracted -- runtime `FHM` container count +- offline `FHM` container count - parsed `SWG` files - exported/skipped `NTXR` textures @@ -95,17 +94,16 @@ Use a custom asset root: python .\tools\run_ac6_asset_pipeline.py --asset-root 'C:\path\to\assets' ``` -Skip PAC re-extraction and only process existing dumps: +Skip PAC re-extraction and process the existing offline decoded files: ```powershell python .\tools\run_ac6_asset_pipeline.py --skip-pac-extract ``` -Recommended workflow after a new play session: +Include optional runtime dumps in addition to the offline decoded PAC corpus: ```powershell -powershell -ExecutionPolicy Bypass -File .\tools\launch_ac6_with_pac_dump.ps1 -python .\tools\run_ac6_asset_pipeline.py --skip-pac-extract +python .\tools\run_ac6_asset_pipeline.py --include-runtime-dumps ``` Extract only PAC entries marked raw: @@ -118,7 +116,7 @@ python .\tools\run_ac6_asset_pipeline.py --raw-only - Raw PAC extraction: - `C:\ext\New folder\AC6_recomp\out\ac6_pac_extracted_raw` -- Parsed runtime FHM corpus: +- Parsed FHM corpus: - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_fhm_typed` - Parsed SWG metadata: - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_swg_parsed` diff --git a/out/build/win-amd64-relwithdebinfo/ac6recomp.toml b/out/build/win-amd64-relwithdebinfo/ac6recomp.toml index 6db32e01..72f12399 100644 --- a/out/build/win-amd64-relwithdebinfo/ac6recomp.toml +++ b/out/build/win-amd64-relwithdebinfo/ac6recomp.toml @@ -1,17 +1,15 @@ # Auto-generated cvar configuration -ac6_render_capture = false -ac6_unlock_fps = false -ac6_native_graphics_require_capture = false -vsync = true -guest_vblank_sync_to_refresh = true -host_present_from_non_ui_thread = false -d3d12_allow_variable_refresh_rate_and_tearing = false log_level = "debug" log_file = "ac6recomp.log" +direct_host_resolve = false +vfetch_index_rounding_bias = true +guest_vblank_sync_to_refresh = true +window_width = 1920 +window_height = 1080 video_mode_width = 1920 video_mode_height = 1080 resolution = "1080p" -direct_host_resolve = false -window_width = 1920 -window_height = 1080 -vfetch_index_rounding_bias = true \ No newline at end of file +d3d12_allow_variable_refresh_rate_and_tearing = false +host_present_from_non_ui_thread = false +ac6_performance_mode = false +ac6_unlock_fps = true diff --git a/src/ac6_native_graphics.cpp b/src/ac6_native_graphics.cpp index a995f7cb..cd624b98 100644 --- a/src/ac6_native_graphics.cpp +++ b/src/ac6_native_graphics.cpp @@ -25,6 +25,7 @@ REXCVAR_DEFINE_BOOL(ac6_force_safe_draw_resolution_scale, true, "AC6/NativeGraph "Force AC6 hybrid backend fixes mode to use 1x draw resolution scaling until the scaled path is fixed"); REXCVAR_DEFINE_BOOL(ac6_force_safe_direct_host_resolve, true, "AC6/NativeGraphics", "Force AC6 hybrid backend fixes mode to keep direct_host_resolve disabled until the AC6 crash is fixed"); +REXCVAR_DECLARE(std::string, ac6_graphics_backend); namespace ac6::graphics { namespace { @@ -69,6 +70,9 @@ void SyncRuntimeFlags() { g_runtime_status.authoritative_renderer_active = g_runtime_status.enabled && g_runtime_status.mode != GraphicsRuntimeMode::kDisabled; + if (g_runtime_status.authoritative_renderer_name.empty()) { + g_runtime_status.authoritative_renderer_name = REXCVAR_GET(ac6_graphics_backend); + } } } // namespace @@ -140,6 +144,7 @@ void OnFrameBoundary(rex::memory::Memory* memory) { auto* kernel_state = ts->context()->kernel_state; if (auto* concrete_graphics = dynamic_cast(kernel_state->graphics_system())) { + g_runtime_status.authoritative_renderer_name = concrete_graphics->name(); concrete_graphics->GetLastSwapSubmission(&swap_submission, &swap_sequence); guest_vblank_interval_ticks = concrete_graphics->guest_vblank_interval_ticks(); last_guest_vblank_tick = concrete_graphics->last_vblank_interrupt_guest_tick(); diff --git a/src/ac6_native_graphics.h b/src/ac6_native_graphics.h index 51d717ae..71b4004b 100644 --- a/src/ac6_native_graphics.h +++ b/src/ac6_native_graphics.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -23,6 +24,7 @@ struct NativeGraphicsRuntimeStatus { GraphicsRuntimeMode mode = GraphicsRuntimeMode::kHybridBackendFixes; bool capture_enabled = false; bool authoritative_renderer_active = false; + std::string authoritative_renderer_name; uint32_t draw_resolution_scale_x = 1; uint32_t draw_resolution_scale_y = 1; bool direct_host_resolve = true; diff --git a/src/ac6_native_graphics_overlay.cpp b/src/ac6_native_graphics_overlay.cpp index 63660829..8c4e74ee 100644 --- a/src/ac6_native_graphics_overlay.cpp +++ b/src/ac6_native_graphics_overlay.cpp @@ -38,8 +38,11 @@ void NativeGraphicsStatusDialog::OnDraw(ImGuiIO& io) { ImGui::Text("mode: %.*s", static_cast(ToString(status.mode).size()), ToString(status.mode).data()); ImGui::Text("authoritative renderer: %s", - status.authoritative_renderer_active ? "RexGlue/Xenia D3D12 backend" - : "disabled"); + status.authoritative_renderer_active + ? (status.authoritative_renderer_name.empty() + ? "RexGlue/Xenia backend" + : status.authoritative_renderer_name.c_str()) + : "disabled"); ImGui::Text("capture active: %s", status.capture_enabled ? "yes" : "no"); ImGui::Text("draw resolution scale: %ux%u", status.draw_resolution_scale_x, status.draw_resolution_scale_y); diff --git a/src/ac6_texture_overrides.h b/src/ac6_texture_overrides.h index 9e5ba2e1..7223ef4b 100644 --- a/src/ac6_texture_overrides.h +++ b/src/ac6_texture_overrides.h @@ -7,7 +7,11 @@ #include #include -#include +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include namespace ac6::textures { diff --git a/src/ac6recomp_app.h b/src/ac6recomp_app.h index 55e873d6..58c1f05c 100644 --- a/src/ac6recomp_app.h +++ b/src/ac6recomp_app.h @@ -1,13 +1,24 @@ #pragma once #include +#include +#include +#include #include +#if REX_HAS_D3D12 +#include +#endif +#if REX_HAS_VULKAN +#include +#endif #include "ac6_native_graphics.h" #include "ac6_native_graphics_overlay.h" #include "generated/ac6recomp_config.h" +REXCVAR_DECLARE(std::string, ac6_graphics_backend); + class Ac6recompApp : public rex::ReXApp { public: using rex::ReXApp::ReXApp; @@ -27,6 +38,26 @@ class Ac6recompApp : public rex::ReXApp { void OnPreSetup(rex::RuntimeConfig& config) override { REXLOG_INFO("Ac6recompApp::OnPreSetup"); rex::ReXApp::OnPreSetup(config); + + const std::string requested_backend = REXCVAR_GET(ac6_graphics_backend); +#if REX_HAS_VULKAN + if (requested_backend == "vulkan" || requested_backend == "auto") { + config.graphics = + REX_GRAPHICS_BACKEND(rex::graphics::vulkan::VulkanGraphicsSystem); + REXLOG_INFO("Ac6recompApp: selected Vulkan graphics backend"); + return; + } +#endif +#if REX_HAS_D3D12 + if (requested_backend == "d3d12" || requested_backend == "auto") { + config.graphics = + REX_GRAPHICS_BACKEND(rex::graphics::d3d12::D3D12GraphicsSystem); + REXLOG_INFO("Ac6recompApp: selected D3D12 graphics backend"); + return; + } +#endif + REXLOG_WARN("Ac6recompApp: requested graphics backend '{}' is not available in this build", + requested_backend); } void OnPostSetup() override { diff --git a/src/main.cpp b/src/main.cpp index 3b8b35d2..f3a1d939 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -27,7 +27,9 @@ REXCVAR_DECLARE(bool, ac6_texture_swaps_dump_enabled); REXCVAR_DECLARE(bool, vsync); REXCVAR_DECLARE(bool, guest_vblank_sync_to_refresh); REXCVAR_DECLARE(bool, host_present_from_non_ui_thread); +#if REX_HAS_D3D12 REXCVAR_DECLARE(bool, d3d12_allow_variable_refresh_rate_and_tearing); +#endif REXCVAR_DECLARE(bool, vfetch_index_rounding_bias); REXCVAR_DECLARE(int32_t, video_mode_width); REXCVAR_DECLARE(int32_t, video_mode_height); @@ -35,6 +37,20 @@ REXCVAR_DECLARE(std::string, resolution); REXCVAR_DECLARE(int32_t, window_width); REXCVAR_DECLARE(int32_t, window_height); +#if REX_HAS_VULKAN +#define AC6_DEFAULT_GRAPHICS_BACKEND "vulkan" +#elif REX_HAS_D3D12 +#define AC6_DEFAULT_GRAPHICS_BACKEND "d3d12" +#else +#define AC6_DEFAULT_GRAPHICS_BACKEND "auto" +#endif + +REXCVAR_DEFINE_STRING(ac6_graphics_backend, AC6_DEFAULT_GRAPHICS_BACKEND, + "AC6/NativeGraphics", + "Host graphics backend: vulkan, d3d12, or auto") + .allowed({"vulkan", "d3d12", "auto"}) + .lifecycle(rex::cvar::Lifecycle::kInitOnly); + REXCVAR_DEFINE_BOOL(ac6_performance_mode, true, "AC6/Performance", "Disable all diagnostics, logging, and development overlays for maximum runtime performance"); @@ -82,9 +98,11 @@ void ApplyAc6DefaultSettings() { if (!rex::cvar::HasNonDefaultValue("host_present_from_non_ui_thread")) { REXCVAR_SET(host_present_from_non_ui_thread, false); } +#if REX_HAS_D3D12 if (!rex::cvar::HasNonDefaultValue("d3d12_allow_variable_refresh_rate_and_tearing")) { REXCVAR_SET(d3d12_allow_variable_refresh_rate_and_tearing, false); } +#endif if (!rex::cvar::HasNonDefaultValue("vfetch_index_rounding_bias")) { REXCVAR_SET(vfetch_index_rounding_bias, true); } diff --git a/src/render_hooks.cpp b/src/render_hooks.cpp index cf16cb5a..a5eca64d 100644 --- a/src/render_hooks.cpp +++ b/src/render_hooks.cpp @@ -11,6 +11,9 @@ REXCVAR_DEFINE_BOOL(ac6_unlock_fps, false, "AC6", "Unlock frame rate to 60fps"); REXCVAR_DEFINE_BOOL(ac6_timing_hooks_enabled, true, "AC6", "Enable AC6 timing hooks that alter the game's presentation cadence"); +REXCVAR_DEFINE_BOOL(ac6_cutscene_clamp, true, "AC6", + "Suspend the 60fps unlock during in-engine cutscenes so they " + "play at native ~30fps instead of double speed"); using Clock = std::chrono::steady_clock; @@ -21,12 +24,44 @@ std::atomic g_fps{0.0}; std::atomic g_frame_count{0}; Clock::time_point g_frame_start{}; +// Wall-clock (steady) ms of the last in-engine cutscene tick. The cutscene +// hook (ac6CinematicTickHook) runs on the game thread; the timing hooks read +// this on the present/GPU path, hence the atomic. INT64_MIN = never ticked. +std::atomic g_last_cinematic_tick_ms{INT64_MIN}; + +// The demo-manager Exec runs every game frame while a cutscene plays (~16-33ms +// apart). A few-frame decay keeps the clamp asserted across the cutscene and +// auto-releases shortly after it ends. +constexpr int64_t kCinematicDecayMs = 100; + +int64_t NowMs() { + return std::chrono::duration_cast( + Clock::now().time_since_epoch()) + .count(); +} + bool AreTimingHooksActive() { - return REXCVAR_GET(ac6_timing_hooks_enabled) && REXCVAR_GET(ac6_unlock_fps); + return REXCVAR_GET(ac6_timing_hooks_enabled) && REXCVAR_GET(ac6_unlock_fps) && + !ac6::IsCinematicActive(); } } // namespace +namespace ac6 { + +bool IsCinematicActive() { + if (!REXCVAR_GET(ac6_cutscene_clamp)) { + return false; + } + const int64_t last = g_last_cinematic_tick_ms.load(std::memory_order_relaxed); + if (last == INT64_MIN) { + return false; + } + return (NowMs() - last) <= kCinematicDecayMs; +} + +} // namespace ac6 + bool ac6FlipIntervalHook() { return AreTimingHooksActive(); } @@ -61,6 +96,15 @@ void ac6PresentTimingHook(PPCRegister& /*r31*/) { g_frame_start = now; } +void ac6CinematicTickHook(PPCRegister& /*r3*/) { + // A demo-manager Exec (DD: sub_82184460 / EM: sub_821856F8) ran this frame -> + // an in-engine cutscene is playing. Stamp the time so AreTimingHooksActive() + // suspends the 60fps unlock until the cutscene ends (the stamp goes stale a + // few frames after the last tick), letting the frame-locked cinematic + // Sequencer play at native ~30fps instead of double speed. + g_last_cinematic_tick_ms.store(NowMs(), std::memory_order_relaxed); +} + namespace ac6 { FrameStats GetFrameStats() { diff --git a/src/render_hooks.h b/src/render_hooks.h index 51f8ac94..2fd7c77b 100644 --- a/src/render_hooks.h +++ b/src/render_hooks.h @@ -7,6 +7,7 @@ REXCVAR_DECLARE(bool, ac6_unlock_fps); REXCVAR_DECLARE(bool, ac6_timing_hooks_enabled); +REXCVAR_DECLARE(bool, ac6_cutscene_clamp); namespace ac6 { @@ -18,9 +19,18 @@ struct FrameStats { FrameStats GetFrameStats(); +// True while an in-engine cutscene (NU::FW::IngameCinematics, driven by +// CAce6DemoManager::Exec) has ticked within the last decay window. Used by the +// timing hooks to suspend the 60fps unlock so cutscenes play at native cadence. +bool IsCinematicActive(); + } // namespace ac6 bool ac6FlipIntervalHook(); bool ac6PresentIntervalHook(PPCRegister& r10); void ac6DeltaDivisorHook(PPCRegister& r29); void ac6PresentTimingHook(PPCRegister& r31); + +// Fires once per frame from the demo-manager Exec while a cutscene is playing. +// r3 = the demo-manager `this`; unused (presence-of-call is the signal). +void ac6CinematicTickHook(PPCRegister& r3); diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/spirv_translator.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/spirv_translator.h index c51ea546..d26f9502 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/spirv_translator.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/spirv_translator.h @@ -33,7 +33,7 @@ class SpirvShaderTranslator : public ShaderTranslator { // TODO(Triang3l): Change to 0xYYYYMMDD once it's out of the rapid // prototyping stage (easier to do small granular updates with an // incremental counter). - static constexpr uint32_t kVersion = 12; + static constexpr uint32_t kVersion = 13; enum class DepthStencilMode : uint32_t { kNoModifiers, @@ -577,6 +577,7 @@ class SpirvShaderTranslator : public ShaderTranslator { void StartFragmentShaderBeforeMain(); void StartFragmentShaderInMain(); void CompleteFragmentShaderInMain(); + void CompleteFragmentShader_DSV_DepthTo24Bit(); // Updates the current flow control condition (to be called in the beginning // of exec and in jumps), closing the previous conditionals if needed. @@ -946,6 +947,10 @@ class SpirvShaderTranslator : public ShaderTranslator { // With fragment shader interlock, variables in the main function. // Otherwise, framebuffer color attachment outputs. std::array output_or_var_fragment_data_; + // Function-scoped staging variable for guest oDepth writes. FSI consumes this + // through var_main_fragment_depth_; FBO copies it to gl_FragDepth after guest + // control flow is complete. + spv::Id output_or_var_fragment_depth_; // For host render targets and only when needed - float. spv::Id output_fragment_depth_; // For host render targets and only when needed - int[1]. diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/command_processor.h b/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/command_processor.h index 9ad94903..9965801b 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/command_processor.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/command_processor.h @@ -758,11 +758,9 @@ class VulkanCommandProcessor : public CommandProcessor { uint32_t host_index = UINT32_MAX; bool valid = false; } active_occlusion_query_; - struct VertexBufferState { - uint32_t address = UINT32_MAX; - uint32_t size = UINT32_MAX; - }; - std::array vertex_buffer_states_{}; + static constexpr uint32_t kVertexFetchConstantCount = 96; + // Bit is set when the vertex buffer at that index has been requested in the + // current frame. Cleared between frames and on fetch constant writes. uint64_t vertex_buffers_in_sync_[2] = {}; std::unordered_map readback_buffers_; std::unordered_map memexport_readback_buffers_; diff --git a/thirdparty/rexglue-sdk/src/graphics/flags.cpp b/thirdparty/rexglue-sdk/src/graphics/flags.cpp index 19b986d6..da11d6e4 100644 --- a/thirdparty/rexglue-sdk/src/graphics/flags.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/flags.cpp @@ -31,6 +31,8 @@ REXCVAR_DEFINE_BOOL(use_fuzzy_alpha_epsilon, true, "GPU", REXCVAR_DEFINE_BOOL(vfetch_index_rounding_bias, false, "GPU/Shader", "Apply small epsilon bias to vertex fetch indices before " "flooring to fix black triangles caused by RCP precision"); +REXCVAR_DEFINE_BOOL(draw_resolution_scaled_texture_offsets, true, "GPU/Shader", + "Scale texture offsets with draw resolution"); REXCVAR_DEFINE_BOOL(gpu_debug_markers, false, "GPU", "Insert debug markers into GPU command streams for tools " "like PIX and RenderDoc. Automatically enabled when " diff --git a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator_fetch.cpp b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator_fetch.cpp index 31842c0c..1098e0ef 100644 --- a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator_fetch.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator_fetch.cpp @@ -24,9 +24,6 @@ #include #include -REXCVAR_DEFINE_BOOL(draw_resolution_scaled_texture_offsets, true, "GPU/Shader", - "Scale texture offsets with draw resolution"); - namespace rex::graphics { using namespace ucode; diff --git a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator.cpp b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator.cpp index c7097056..74a3f15c 100644 --- a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator.cpp @@ -129,6 +129,7 @@ void SpirvShaderTranslator::Reset() { output_per_vertex_member_cull_distance_ = UINT32_MAX; type_output_per_vertex_ = spv::NoResult; output_per_vertex_ = spv::NoResult; + output_or_var_fragment_depth_ = spv::NoResult; output_fragment_depth_ = spv::NoResult; output_fragment_sample_mask_ = spv::NoResult; @@ -2923,6 +2924,7 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() { type_float_, "gl_FragDepth"); builder_->addDecoration(output_fragment_depth_, spv::DecorationBuiltIn, spv::BuiltInFragDepth); + builder_->addDecoration(output_fragment_depth_, spv::DecorationInvariant); main_interface_.push_back(output_fragment_depth_); } if (alpha_to_coverage_possible && features_.sample_rate_shading) { @@ -2957,17 +2959,18 @@ void SpirvShaderTranslator::StartFragmentShaderInMain() { // to the execution mask GPUs naturally have. } + if (current_shader().writes_depth()) { + output_or_var_fragment_depth_ = + builder_->createVariable(spv::NoPrecision, spv::StorageClassFunction, type_float_, + "xe_var_fragment_depth", const_float_0_); + } + if (edram_fragment_shader_interlock_) { // Initialize color output variables with fragment shader interlock. std::fill(output_or_var_fragment_data_.begin(), output_or_var_fragment_data_.end(), spv::NoResult); - var_main_fragment_depth_ = spv::NoResult; + var_main_fragment_depth_ = output_or_var_fragment_depth_; var_main_fsi_color_written_ = spv::NoResult; - if (current_shader().writes_depth()) { - var_main_fragment_depth_ = - builder_->createVariable(spv::NoPrecision, spv::StorageClassFunction, type_float_, - "xe_var_fragment_depth", const_float_0_); - } uint32_t color_targets_written = current_shader().writes_color_targets(); if (color_targets_written) { static const char* const kFragmentDataVariableNames[] = { @@ -3616,8 +3619,7 @@ void SpirvShaderTranslator::StoreResult(const InstructionResult& result, spv::Id // Writes X to scalar gl_FragDepth. assert_true(used_write_mask == 0b0001); assert_true(current_shader().writes_depth()); - target_pointer = - edram_fragment_shader_interlock_ ? var_main_fragment_depth_ : output_fragment_depth_; + target_pointer = output_or_var_fragment_depth_; // Guest depth output is expected to be [0, 1]. is_clamped = true; } break; diff --git a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator_rb.cpp b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator_rb.cpp index 765c1417..c0db7a01 100644 --- a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator_rb.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/spirv_translator_rb.cpp @@ -1365,45 +1365,7 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() { } } - if (!edram_fragment_shader_interlock_ && output_fragment_depth_ != spv::NoResult) { - Modification::DepthStencilMode depth_stencil_mode = - GetSpirvShaderModification().pixel.depth_stencil_mode; - if (depth_stencil_mode == Modification::DepthStencilMode::kFloat24Truncating || - depth_stencil_mode == Modification::DepthStencilMode::kFloat24Rounding) { - // For oDepth, depth is already in guest [0, 1]. - // Without oDepth, reconstruct guest [0, 1] from host [0, 0.5] by - // doubling gl_FragCoord.z and saturating. - spv::Id depth; - if (current_shader().writes_depth()) { - depth = builder_->createLoad(output_fragment_depth_, spv::NoPrecision); - } else { - assert_true(input_fragment_coordinates_ != spv::NoResult); - id_vector_temp_.clear(); - id_vector_temp_.push_back(builder_->makeIntConstant(2)); - depth = builder_->createLoad( - builder_->createAccessChain(spv::StorageClassInput, input_fragment_coordinates_, - id_vector_temp_), - spv::NoPrecision); - if (IsSampleRate()) { - // Statically use gl_SampleID to keep this path at sample frequency. - assert_true(input_sample_id_ != spv::NoResult); - builder_->createLoad(input_sample_id_, spv::NoPrecision); - } - depth = builder_->createTriBuiltinCall( - type_float_, ext_inst_glsl_std_450_, GLSLstd450NClamp, - builder_->createNoContractionBinOp(spv::OpFMul, type_float_, depth, - builder_->makeFloatConstant(2.0f)), - const_float_0_, const_float_1_); - } - // Convert guest [0, 1] float32 to float24 and back to host [0, 0.5]. - spv::Id depth_float24 = SpirvShaderTranslator::PreClampedDepthTo20e4( - *builder_, depth, depth_stencil_mode == Modification::DepthStencilMode::kFloat24Rounding, - false, ext_inst_glsl_std_450_); - depth = SpirvShaderTranslator::Depth20e4To32(*builder_, depth_float24, 0, true, false, - ext_inst_glsl_std_450_); - builder_->createStore(depth, output_fragment_depth_); - } - } + CompleteFragmentShader_DSV_DepthTo24Bit(); if (edram_fragment_shader_interlock_) { if (block_fsi_if_after_depth_stencil_merge) { @@ -1425,6 +1387,72 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() { } } +void SpirvShaderTranslator::CompleteFragmentShader_DSV_DepthTo24Bit() { + if (edram_fragment_shader_interlock_ || output_fragment_depth_ == spv::NoResult) { + return; + } + + Modification::DepthStencilMode depth_stencil_mode = + GetSpirvShaderModification().pixel.depth_stencil_mode; + bool convert_float24_depth = + depth_stencil_mode == Modification::DepthStencilMode::kFloat24Truncating || + depth_stencil_mode == Modification::DepthStencilMode::kFloat24Rounding; + bool shader_writes_depth = current_shader().writes_depth(); + if (!shader_writes_depth && !convert_float24_depth) { + return; + } + + // For oDepth, depth is already in guest [0, 1]. Without oDepth, reconstruct + // guest [0, 1] from host [0, 0.5] by doubling gl_FragCoord.z and saturating. + spv::Id depth; + if (shader_writes_depth) { + assert_true(output_or_var_fragment_depth_ != spv::NoResult); + depth = builder_->createLoad(output_or_var_fragment_depth_, spv::NoPrecision); + } else { + assert_true(input_fragment_coordinates_ != spv::NoResult); + id_vector_temp_.clear(); + id_vector_temp_.push_back(builder_->makeIntConstant(2)); + depth = builder_->createLoad( + builder_->createAccessChain(spv::StorageClassInput, input_fragment_coordinates_, + id_vector_temp_), + spv::NoPrecision); + if (IsSampleRate()) { + // Statically use gl_SampleID to keep this path at sample frequency. + assert_true(input_sample_id_ != spv::NoResult); + builder_->createLoad(input_sample_id_, spv::NoPrecision); + } + depth = builder_->createTriBuiltinCall( + type_float_, ext_inst_glsl_std_450_, GLSLstd450NClamp, + builder_->createNoContractionBinOp(spv::OpFMul, type_float_, depth, + builder_->makeFloatConstant(2.0f)), + const_float_0_, const_float_1_); + } + + if (convert_float24_depth) { + // Convert guest [0, 1] float32 to float24 and back to host [0, 0.5]. + spv::Id depth_float24 = SpirvShaderTranslator::PreClampedDepthTo20e4( + *builder_, depth, depth_stencil_mode == Modification::DepthStencilMode::kFloat24Rounding, + false, ext_inst_glsl_std_450_); + depth = SpirvShaderTranslator::Depth20e4To32(*builder_, depth_float24, 0, true, false, + ext_inst_glsl_std_450_); + } else if (shader_writes_depth) { + // oDepth bypasses viewport depth scaling, so dynamically remap guest + // 0...1 to host 0...0.5 whenever the bound depth buffer is D24FS8. + spv::Id depth_float24_flag = builder_->createBinOp( + spv::OpINotEqual, type_bool_, + builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, main_system_constant_flags_, + builder_->makeUintConstant(kSysFlag_DepthFloat24)), + const_uint_0_); + depth = builder_->createTriOp( + spv::OpSelect, type_float_, depth_float24_flag, + builder_->createNoContractionBinOp(spv::OpFMul, type_float_, depth, + builder_->makeFloatConstant(0.5f)), + depth); + } + + builder_->createStore(depth, output_fragment_depth_); +} + spv::Id SpirvShaderTranslator::LoadMsaaSamplesFromFlags() { return builder_->createTriOp(spv::OpBitFieldUExtract, type_uint_, main_system_constant_flags_, builder_->makeUintConstant(kSysFlag_MsaaSamples_Shift), diff --git a/thirdparty/rexglue-sdk/src/graphics/vulkan/command_processor.cpp b/thirdparty/rexglue-sdk/src/graphics/vulkan/command_processor.cpp index cf39bf0a..1004e847 100644 --- a/thirdparty/rexglue-sdk/src/graphics/vulkan/command_processor.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/vulkan/command_processor.cpp @@ -46,6 +46,8 @@ #include #include +#include "../../../../../src/ac6_backend_fixes/ac6_backend_hooks.h" + // Legacy backend compatibility aliases for shared readback controls. REXCVAR_DEFINE_BOOL(vulkan_readback_resolve, false, "GPU/Vulkan", "Read render-to-texture results on the CPU") @@ -613,14 +615,10 @@ void VulkanCommandProcessor::InvalidateGpuMemory() { void VulkanCommandProcessor::InvalidateAllVertexBufferResidency() { vertex_buffers_in_sync_[0] = 0; vertex_buffers_in_sync_[1] = 0; - for (VertexBufferState& state : vertex_buffer_states_) { - state.address = UINT32_MAX; - state.size = UINT32_MAX; - } } void VulkanCommandProcessor::InvalidateVertexBufferResidency(uint32_t vfetch_index) { - if (vfetch_index >= vertex_buffer_states_.size()) { + if (vfetch_index >= kVertexFetchConstantCount) { return; } vertex_buffers_in_sync_[vfetch_index >> 6] &= ~(uint64_t(1) << (vfetch_index & 63)); @@ -631,10 +629,10 @@ void VulkanCommandProcessor::InvalidateVertexBufferResidencyRange(uint32_t first if (first_vfetch > last_vfetch) { std::swap(first_vfetch, last_vfetch); } - if (first_vfetch >= vertex_buffer_states_.size()) { + if (first_vfetch >= kVertexFetchConstantCount) { return; } - last_vfetch = std::min(last_vfetch, uint32_t(vertex_buffer_states_.size() - 1)); + last_vfetch = std::min(last_vfetch, kVertexFetchConstantCount - 1); for (uint32_t vfetch_index = first_vfetch; vfetch_index <= last_vfetch; ++vfetch_index) { InvalidateVertexBufferResidency(vfetch_index); } @@ -2300,6 +2298,16 @@ void VulkanCommandProcessor::IssueSwap(uint32_t frontbuffer_ptr, uint32_t frontb return; } + // Keep the AC6 frame-boundary diagnostics in sync with the D3D12 path before + // the swap source is selected and presented. + { + system::GraphicsSwapSubmission frame_boundary_submission = {}; + frame_boundary_submission.frontbuffer_virtual_address = frontbuffer_ptr; + frame_boundary_submission.frontbuffer_width = frontbuffer_width; + frame_boundary_submission.frontbuffer_height = frontbuffer_height; + graphics_system_->HandleVideoSwap(frame_boundary_submission); + } + bool skip_present_due_async_placeholder = REXCVAR_GET(async_shader_compilation) && REXCVAR_GET(vulkan_async_skip_incomplete_frames) && frame_used_async_placeholder_pipeline_; @@ -2414,6 +2422,29 @@ void VulkanCommandProcessor::IssueSwap(uint32_t frontbuffer_ptr, uint32_t frontb frontbuffer_height_unscaled, guest_output_width, guest_output_height, static_cast(frontbuffer_format)); + system::GraphicsSwapSubmission ac6_submission = {}; + uint64_t ac6_submission_sequence = 0; + graphics_system_->GetLastSwapSubmission(&ac6_submission, &ac6_submission_sequence); + if (!ac6_submission_sequence) { + ac6_submission.frontbuffer_virtual_address = frontbuffer_ptr; + ac6_submission.frontbuffer_width = frontbuffer_width; + ac6_submission.frontbuffer_height = frontbuffer_height; + } + + auto* ac6_vertex_shader = active_vertex_shader(); + auto* ac6_pixel_shader = active_pixel_shader(); + uint64_t ac6_vertex_shader_hash = + ac6_vertex_shader ? ac6_vertex_shader->ucode_data_hash() : 0; + uint64_t ac6_pixel_shader_hash = + ac6_pixel_shader ? ac6_pixel_shader->ucode_data_hash() : 0; + + ac6::backend::ReportSwapDecision( + ac6_submission, ac6_submission_sequence, + ac6::backend::SwapSourceType::kGuestSwapTexture, + swap_source_scaled, guest_output_width, guest_output_height, + frontbuffer_width_scaled, frontbuffer_height_scaled, + ac6_vertex_shader_hash, ac6_pixel_shader_hash); + system::X_VIDEO_MODE video_mode; kernel::xboxkrnl::VdQueryVideoMode(&video_mode); uint32_t display_width = std::max(uint32_t(1), uint32_t(video_mode.display_width)); @@ -4020,11 +4051,6 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type, uint32_t vfetch_index, vfetch_constant.dword_0, vfetch_constant.dword_1); return false; } - VertexBufferState& state = vertex_buffer_states_[vfetch_index]; - if (state.address == vfetch_constant.address && state.size == vfetch_constant.size) { - vertex_buffers_in_sync_[vfetch_index >> 6] |= vfetch_bit; - continue; - } if (!shared_memory_->RequestRange(vfetch_constant.address << 2, vfetch_constant.size << 2)) { REXGPU_ERROR( "Failed to request vertex buffer at 0x{:08X} (size {}) in the shared " @@ -4032,8 +4058,6 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type, uint32_t vfetch_constant.address << 2, vfetch_constant.size << 2); return false; } - state.address = vfetch_constant.address; - state.size = vfetch_constant.size; vertex_buffers_in_sync_[vfetch_index >> 6] |= vfetch_bit; } } diff --git a/thirdparty/rexglue-sdk/src/native/ui/CMakeLists.txt b/thirdparty/rexglue-sdk/src/native/ui/CMakeLists.txt index 90df6488..8192ab3e 100644 --- a/thirdparty/rexglue-sdk/src/native/ui/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/src/native/ui/CMakeLists.txt @@ -127,6 +127,7 @@ endif() if(WIN32) target_link_libraries(rexui PUBLIC dwmapi + dxgi Shcore ) else() diff --git a/tools/export_ac6_ntxr.py b/tools/export_ac6_ntxr.py index e3206f88..89638d3a 100644 --- a/tools/export_ac6_ntxr.py +++ b/tools/export_ac6_ntxr.py @@ -736,7 +736,59 @@ def extract_r8_visible(payload: bytes, storage_width: int, visible_width: int, v return b"".join(rows) -def export_ntxr(input_path: Path, output_root: Path, source_root: Path) -> dict: +def existing_export_entry(input_path: Path, output_root: Path, source_root: Path) -> dict | None: + output_base = output_root / input_path.relative_to(source_root) + json_path = output_base.with_suffix(".json") + dds_path = output_base.with_suffix(".dds") + preview_path = output_base.with_suffix(".tga") + png_path = output_base.with_suffix(".png") + if not (json_path.exists() and dds_path.exists() and preview_path.exists() and png_path.exists()): + return None + + try: + metadata = json.loads(json_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + entry = { + "source": str(input_path.relative_to(source_root)).replace("\\", "/"), + "size": input_path.stat().st_size, + "status": "exported", + "resumed": True, + "layout": metadata.get("layout"), + "format": metadata.get("format"), + "dds": str(dds_path.relative_to(output_root)).replace("\\", "/"), + "preview": str(preview_path.relative_to(output_root)).replace("\\", "/"), + "preview_png": str(png_path.relative_to(output_root)).replace("\\", "/"), + "metadata": str(json_path.relative_to(output_root)).replace("\\", "/"), + "visible_width": metadata.get("visible_width"), + "visible_height": metadata.get("visible_height"), + "storage_width": metadata.get("storage_width"), + "storage_height": metadata.get("storage_height"), + "mip_count": metadata.get("mip_count"), + "notes": metadata.get("notes"), + } + raw_png = metadata.get("raw_png") + if raw_png: + raw_path = json_path.with_name(raw_png) + if raw_path.exists(): + entry["raw_preview_png"] = str(raw_path.relative_to(output_root)).replace("\\", "/") + alpha_png = metadata.get("alpha_png") + if alpha_png: + alpha_path = json_path.with_name(alpha_png) + if alpha_path.exists(): + entry["alpha_preview_png"] = str(alpha_path.relative_to(output_root)).replace("\\", "/") + if metadata.get("face_pngs"): + entry["face_pngs"] = metadata["face_pngs"] + return entry + + +def export_ntxr(input_path: Path, output_root: Path, source_root: Path, *, skip_existing: bool = False) -> dict: + if skip_existing: + existing = existing_export_entry(input_path, output_root, source_root) + if existing is not None: + return existing + blob = input_path.read_bytes() plan, reason = classify_ntxr(blob) output_base = output_root / input_path.relative_to(source_root) @@ -964,6 +1016,17 @@ def main() -> int: default=Path("out") / "ac6_runtime_ntxr_exported", help="Output directory for exported textures", ) + parser.add_argument( + "--force", + action="store_true", + help="Re-export textures even when a complete prior export already exists", + ) + parser.add_argument( + "--limit", + type=int, + default=0, + help="Export at most this many NTXR files (0 = no limit)", + ) args = parser.parse_args() source_root = args.input.resolve() @@ -972,8 +1035,11 @@ def main() -> int: exported = [] skipped = [] - for input_path in sorted(source_root.rglob("*.ntxr")): - result = export_ntxr(input_path, output_root, source_root) + ntxr_paths = sorted(source_root.rglob("*.ntxr")) + if args.limit > 0: + ntxr_paths = ntxr_paths[:args.limit] + for input_path in ntxr_paths: + result = export_ntxr(input_path, output_root, source_root, skip_existing=not args.force) if result["status"] == "exported": exported.append(result) else: diff --git a/tools/extract_ac6_runtime_fhm.py b/tools/extract_ac6_runtime_fhm.py index f76d8645..7bd3dcca 100644 --- a/tools/extract_ac6_runtime_fhm.py +++ b/tools/extract_ac6_runtime_fhm.py @@ -4,10 +4,13 @@ from __future__ import annotations import argparse import json import re -import struct +import sys from collections import defaultdict from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ac6_fhm import ext_for_magic, parse_fhm as parse_fhm_container, safe_tag + DUMP_RE = re.compile( r"^entry_(?P\d+)_mode(?P\d+)_c(?P\d+)_u(?P\d+)" @@ -15,8 +18,14 @@ DUMP_RE = re.compile( ) +def load_manifest(path: Path) -> dict: + if not path.exists(): + return {"entries": []} + return json.loads(path.read_text(encoding="utf-8")) + + def load_manifest_entries(path: Path) -> dict[tuple[int, int], list[dict]]: - manifest = json.loads(path.read_text(encoding="utf-8")) + manifest = load_manifest(path) by_pair: dict[tuple[int, int], list[dict]] = defaultdict(list) for entry in manifest["entries"]: if entry["storage_kind"] != "compressed": @@ -25,93 +34,38 @@ def load_manifest_entries(path: Path) -> dict[tuple[int, int], list[dict]]: return by_pair -def parse_fhm(blob: bytes) -> list[dict]: - if len(blob) < 0x1C or blob[:4] != b"FHM ": - return [] - - count = struct.unpack_from(">I", blob, 0x10)[0] - if count == 0: - return [] - - table_base = 0x14 - offsets_base = table_base - sizes_base = offsets_base + (count * 4) - if sizes_base + (count * 4) > len(blob): - return [] - - offsets = [struct.unpack_from(">I", blob, offsets_base + (i * 4))[0] for i in range(count)] - sizes = [struct.unpack_from(">I", blob, sizes_base + (i * 4))[0] for i in range(count)] - - entries = [] - for index, (offset, size) in enumerate(zip(offsets, sizes)): - if offset >= len(blob): - continue - - end = offset + size - if end > len(blob): - next_offset = offsets[index + 1] if index + 1 < len(offsets) else len(blob) - end = min(next_offset, len(blob)) - if end <= offset: - continue - - child = blob[offset:end] - entries.append( - { - "index": index, - "offset": offset, - "size": len(child), - "magic": child[:4].decode("ascii", errors="replace"), - "data": child, - } - ) - return entries - - def safe_name(name: str) -> str: - return "".join(ch if ch.isalnum() or ch in ("-", "_", ".") else "_" for ch in name) - - -def magic_extension(magic: str) -> str: - normalized = magic.strip().upper() - mapping = { - "FHM": ".fhm", - "NTXR": ".ntxr", - "NSXR": ".nsxr", - "MDLP": ".mdlp", - "PLAD": ".plad", - "BFX": ".bfx", - "BSN": ".bsn", - "ACE6": ".ace6", - "NFH": ".nfh", - } - return mapping.get(normalized, ".bin") + return safe_tag(name) def extract_container(blob: bytes, container_dir: Path, output_root: Path, depth: int, max_depth: int) -> list[dict]: - children = parse_fhm(blob) + children = parse_fhm_container(blob) or [] if not children: return [] child_entries = [] for child in children: - safe_magic = safe_name(child["magic"]) - child_name = f"{child['index']:03d}_{safe_magic}{magic_extension(child['magic'])}" + safe_magic = safe_name(child.magic) + child_name = f"{child.index:03d}_{safe_magic}{ext_for_magic(child.magic)}" child_path = container_dir / child_name - child_path.write_bytes(child["data"]) + child_path.write_bytes(child.data) child_entry = { - "index": child["index"], - "offset": child["offset"], - "size": child["size"], - "magic": child["magic"], + "index": child.index, + "offset": child.offset, + "declared_size": child.declared_size, + "size": child.size, + "magic": child.magic, "path": str(child_path.relative_to(output_root)).replace("\\", "/"), } + if child.notes: + child_entry["parser_notes"] = child.notes - if depth < max_depth and child["data"][:4] == b"FHM ": - nested_dir = container_dir / f"{child['index']:03d}_{safe_magic}" + if depth < max_depth and child.data[:4] == b"FHM ": + nested_dir = container_dir / f"{child.index:03d}_{safe_magic}" nested_dir.mkdir(parents=True, exist_ok=True) - nested_children = extract_container(child["data"], nested_dir, output_root, depth + 1, + nested_children = extract_container(child.data, nested_dir, output_root, depth + 1, max_depth) if nested_children: child_entry["nested"] = nested_children @@ -121,8 +75,57 @@ def extract_container(blob: bytes, container_dir: Path, output_root: Path, depth return child_entries +def extract_blob(blob: bytes, label: str, output_root: Path, max_depth: int, + source_record: dict) -> dict: + container_dir = output_root / safe_name(label) + container_dir.mkdir(parents=True, exist_ok=True) + + children = parse_fhm_container(blob) or [] + if not children: + raw_path = container_dir / f"{safe_name(label)}.bin" + raw_path.write_bytes(blob) + return { + **source_record, + "kind": "raw", + "magic": blob[:4].decode("latin-1", errors="replace") if len(blob) >= 4 else "", + "size": len(blob), + "path": str(raw_path.relative_to(output_root)).replace("\\", "/"), + } + + child_entries = extract_container(blob, container_dir, output_root, 0, max_depth) + return { + **source_record, + "kind": "fhm", + "child_count": len(child_entries), + "children": child_entries, + } + + +def iter_offline_manifest_sources(manifest_path: Path, files_dir: Path) -> list[dict]: + manifest = load_manifest(manifest_path) + sources = [] + manifest_root = manifest_path.parent + for entry in manifest.get("entries", []): + if not entry.get("extracted"): + continue + rel_path = entry.get("path") + if not rel_path: + continue + path = manifest_root / rel_path + if files_dir and not path.is_relative_to(files_dir): + # Keep support for custom --pac-files while still trusting the + # manifest's relative paths when they point elsewhere. + alt = files_dir / Path(rel_path).name + if alt.exists(): + path = alt + if not path.exists(): + continue + sources.append({"entry": entry, "path": path}) + return sources + + def main() -> int: - parser = argparse.ArgumentParser(description="Extract child payloads from runtime-dumped AC6 FHM containers.") + parser = argparse.ArgumentParser(description="Extract child payloads from offline-decoded or runtime-dumped AC6 FHM containers.") parser.add_argument( "--dump-dir", type=Path, @@ -135,12 +138,23 @@ def main() -> int: default=Path("out") / "ac6_pac_extracted_raw" / "manifest.json", help="Manifest produced by extract_ac6_pac.py", ) + parser.add_argument( + "--pac-files", + type=Path, + default=None, + help="Decoded PAC files directory produced by extract_ac6_pac.py (default: /files)", + ) parser.add_argument( "--output", type=Path, - default=Path("out") / "ac6_runtime_fhm_extracted", + default=Path("out") / "ac6_runtime_fhm_typed", help="Output directory for parsed FHM containers and child payloads", ) + parser.add_argument( + "--include-runtime-dumps", + action="store_true", + help="Also merge entry_* runtime dumps from --dump-dir when present", + ) parser.add_argument( "--max-depth", type=int, @@ -151,14 +165,42 @@ def main() -> int: dump_dir = args.dump_dir.resolve() manifest_path = args.manifest.resolve() + pac_files = args.pac_files.resolve() if args.pac_files else manifest_path.parent / "files" output_root = args.output.resolve() output_root.mkdir(parents=True, exist_ok=True) by_pair = load_manifest_entries(manifest_path) extracted = [] + + for source in iter_offline_manifest_sources(manifest_path, pac_files): + entry = source["entry"] + path = source["path"] + blob = path.read_bytes() + label = f"idx_{entry['index']:04d}" + extracted.append( + extract_blob( + blob, + label, + output_root, + args.max_depth, + { + "source": "offline_pac", + "entry_index": entry["index"], + "pac_name": entry["pac_name"], + "storage_kind": entry["storage_kind"], + "compressed_size": entry["compressed_size"], + "decompressed_size": entry["decompressed_size"], + "source_offset": entry["offset"], + "input_path": str(path.relative_to(manifest_path.parent)).replace("\\", "/"), + }, + ) + ) + selected_dumps: dict[tuple[int, int, int, int], Path] = {} - for dump_path in sorted(dump_dir.glob("*.bin")): + runtime_dump_count = 0 + runtime_glob = sorted(dump_dir.glob("*.bin")) if args.include_runtime_dumps and dump_dir.exists() else [] + for dump_path in runtime_glob: match = DUMP_RE.match(dump_path.name) if not match: continue @@ -183,6 +225,7 @@ def main() -> int: selected_dumps[key] = dump_path for dump_path in sorted(selected_dumps.values()): + runtime_dump_count += 1 match = DUMP_RE.match(dump_path.name) assert match is not None @@ -199,16 +242,15 @@ def main() -> int: if len(candidates) == 1 else f"pair_c{compressed_size}_u{decompressed_size}" ) - container_dir = output_root / safe_name(base_label) - container_dir.mkdir(parents=True, exist_ok=True) - blob = dump_path.read_bytes() - children = parse_fhm(blob) - if not children: - raw_path = container_dir / dump_path.name - raw_path.write_bytes(blob) - extracted.append( + extracted.append( + extract_blob( + blob, + f"runtime_{base_label}", + output_root, + args.max_depth, { + "source": "runtime_dump", "dump": dump_path.name, "record_id": record_id, "codec_mode": codec_mode, @@ -216,31 +258,13 @@ def main() -> int: "decompressed_size": decompressed_size, "source_offset": source_offset, "candidate_indexes": [entry["index"] for entry in candidates], - "kind": "raw", - "path": str(raw_path.relative_to(output_root)).replace("\\", "/"), - } + }, ) - continue - - child_entries = extract_container(blob, container_dir, output_root, 0, args.max_depth) - - extracted.append( - { - "dump": dump_path.name, - "record_id": record_id, - "codec_mode": codec_mode, - "compressed_size": compressed_size, - "decompressed_size": decompressed_size, - "source_offset": source_offset, - "candidate_indexes": [entry["index"] for entry in candidates], - "kind": "fhm", - "child_count": len(child_entries), - "children": child_entries, - } ) manifest = { - "dump_dir": str(dump_dir), + "pac_files": str(pac_files), + "dump_dir": str(dump_dir) if args.include_runtime_dumps else None, "manifest": str(manifest_path), "output": str(output_root), "containers": extracted, @@ -250,6 +274,8 @@ def main() -> int: json.dumps( { "containers": len(extracted), + "offline_sources": sum(1 for item in extracted if item.get("source") == "offline_pac"), + "runtime_dumps": runtime_dump_count, "output": str(output_root), }, indent=2, diff --git a/tools/parse_ac6_swg.py b/tools/parse_ac6_swg.py index 20f56e98..f6381d9e 100644 --- a/tools/parse_ac6_swg.py +++ b/tools/parse_ac6_swg.py @@ -106,7 +106,10 @@ def main() -> int: output_root = args.output.resolve() output_root.mkdir(parents=True, exist_ok=True) - swg_files = [input_path] if input_path.is_file() else sorted(input_path.rglob("*_SWG_.bin")) + if input_path.is_file(): + swg_files = [input_path] + else: + swg_files = sorted({*input_path.rglob("*_SWG_.bin"), *input_path.rglob("*.swg")}) parsed = [] for swg_path in swg_files: result = parse_one(swg_path) diff --git a/tools/run_ac6_asset_pipeline.py b/tools/run_ac6_asset_pipeline.py index e1556dc6..aeff4bd9 100644 --- a/tools/run_ac6_asset_pipeline.py +++ b/tools/run_ac6_asset_pipeline.py @@ -13,6 +13,7 @@ DEFAULT_ASSET_ROOT = Path("out") / "build" / "win-amd64-relwithdebinfo" / "asset DEFAULT_RAW_OUT = Path("out") / "ac6_pac_extracted_raw" DEFAULT_DUMP_DIR = Path("out") / "ac6_pac_runtime_dump" DEFAULT_TYPED_OUT = Path("out") / "ac6_runtime_fhm_typed" +DEFAULT_MDLP_OUT = Path("out") / "ac6_mdlp_parts" DEFAULT_SWG_OUT = Path("out") / "ac6_runtime_swg_parsed" DEFAULT_NTXR_OUT = Path("out") / "ac6_runtime_ntxr_exported" @@ -33,13 +34,13 @@ def count_fhm_containers(manifest: dict | None) -> int | None: return None containers = manifest.get("containers") if isinstance(containers, list): - return len(containers) + return sum(1 for item in containers if item.get("kind") == "fhm") return None def main() -> int: parser = argparse.ArgumentParser( - description="Run the AC6 asset extraction pipeline over PAC archives and collected runtime decode dumps." + description="Run the AC6 asset extraction pipeline over offline-decoded PAC archives." ) parser.add_argument( "--asset-root", @@ -57,7 +58,7 @@ def main() -> int: "--dump-dir", type=Path, default=DEFAULT_DUMP_DIR, - help="Directory containing runtime PAC decode dumps", + help="Directory containing optional runtime PAC decode dumps", ) parser.add_argument( "--typed-out", @@ -71,6 +72,12 @@ def main() -> int: default=DEFAULT_SWG_OUT, help="Output directory for parse_ac6_swg.py", ) + parser.add_argument( + "--mdlp-out", + type=Path, + default=DEFAULT_MDLP_OUT, + help="Output directory for extract_ac6_mdlp_parts.py", + ) parser.add_argument( "--ntxr-out", type=Path, @@ -82,6 +89,16 @@ def main() -> int: action="store_true", help="Pass --raw-only to extract_ac6_pac.py", ) + parser.add_argument( + "--no-decompress", + action="store_true", + help="Do not pass --decompress to extract_ac6_pac.py", + ) + parser.add_argument( + "--include-runtime-dumps", + action="store_true", + help="Also process entry_* runtime dumps from --dump-dir when present", + ) parser.add_argument( "--skip-pac-extract", action="store_true", @@ -94,6 +111,7 @@ def main() -> int: raw_out = args.raw_out.resolve() dump_dir = args.dump_dir.resolve() typed_out = args.typed_out.resolve() + mdlp_out = args.mdlp_out.resolve() swg_out = args.swg_out.resolve() ntxr_out = args.ntxr_out.resolve() @@ -104,24 +122,36 @@ def main() -> int: pac_args = [str(THIS_DIR / "extract_ac6_pac.py"), str(asset_root), "--output", str(raw_out)] if args.raw_only: pac_args.append("--raw-only") + elif not args.no_decompress: + pac_args.append("--decompress") run_step(pac_args, repo_root) elif not raw_manifest.exists(): raise SystemExit(f"asset root does not exist and no prior raw manifest is available: {asset_root}") - if not dump_dir.exists(): - raise SystemExit(f"runtime dump directory does not exist: {dump_dir}") - fhm_args = [ str(THIS_DIR / "extract_ac6_runtime_fhm.py"), - "--dump-dir", - str(dump_dir), + "--manifest", + str(raw_manifest), + "--pac-files", + str(raw_out / "files"), "--output", str(typed_out), ] - if raw_manifest.exists(): - fhm_args.extend(["--manifest", str(raw_manifest)]) + if args.include_runtime_dumps: + fhm_args.extend(["--include-runtime-dumps", "--dump-dir", str(dump_dir)]) run_step(fhm_args, repo_root) + run_step( + [ + str(THIS_DIR / "extract_ac6_mdlp_parts.py"), + "--input", + str(typed_out), + "--output", + str(mdlp_out), + ], + repo_root, + ) + run_step( [ str(THIS_DIR / "parse_ac6_swg.py"), @@ -147,14 +177,16 @@ def main() -> int: summary = { "asset_root": str(asset_root), "raw_manifest": str(raw_manifest) if raw_manifest.exists() else None, - "dump_dir": str(dump_dir), + "dump_dir": str(dump_dir) if args.include_runtime_dumps else None, "typed_manifest": str(typed_out / "manifest.json"), + "mdlp_manifest": str(mdlp_out / "manifest.json"), "swg_manifest": str(swg_out / "manifest.json"), "ntxr_manifest": str(ntxr_out / "manifest.json"), } raw_data = read_json(raw_manifest) typed_data = read_json(typed_out / "manifest.json") + mdlp_data = read_json(mdlp_out / "manifest.json") swg_data = read_json(swg_out / "manifest.json") ntxr_data = read_json(ntxr_out / "manifest.json") @@ -162,9 +194,12 @@ def main() -> int: summary["pac_entries"] = raw_data.get("entry_count") summary["pac_extracted"] = raw_data.get("extracted_count") summary["pac_skipped"] = raw_data.get("skipped_count") + summary["pac_decompressed"] = raw_data.get("decompressed_count") fhm_containers = count_fhm_containers(typed_data) if fhm_containers is not None: summary["fhm_containers"] = fhm_containers + if mdlp_data: + summary["mdlp_packages"] = mdlp_data.get("package_count") if swg_data: summary["swg_files"] = swg_data.get("parsed_count") if ntxr_data: diff --git a/tools/unpack_ac6_fhm.py b/tools/unpack_ac6_fhm.py index 011b8459..3d04174b 100644 --- a/tools/unpack_ac6_fhm.py +++ b/tools/unpack_ac6_fhm.py @@ -3,54 +3,11 @@ from __future__ import annotations import argparse import json -import struct +import sys from pathlib import Path - -MAGIC_EXT = { - "FHM ": ".fhm", "NTXR": ".ntxr", "NDXR": ".ndxr", "NSXR": ".nsxr", - "MDLP": ".mdlp", "PLAD": ".plad", "MATE": ".mate", "NFIC": ".nfic", - "NFH\x00": ".nfh", "CAPT": ".capt", "Scen": ".scen", "ACE6": ".ace6", - "RIFF": ".wav", "SWG\x00": ".swg", -} - - -def parse_fhm(blob: bytes): - """Return list of (index, offset, child_bytes) or None if not an FHM.""" - if len(blob) < 0x1C or blob[:4] != b"FHM ": - return None - count = struct.unpack_from(">I", blob, 0x10)[0] - if count == 0 or count > 100000: - return [] - offs_base = 0x14 - size_base = offs_base + count * 4 - if size_base + count * 4 > len(blob): - return [] - offsets = [struct.unpack_from(">I", blob, offs_base + i * 4)[0] for i in range(count)] - sizes = [struct.unpack_from(">I", blob, size_base + i * 4)[0] for i in range(count)] - out = [] - for i, (off, sz) in enumerate(zip(offsets, sizes)): - if off == 0 or off >= len(blob): - continue - end = off + sz - if end > len(blob) or end <= off: - nxt = offsets[i + 1] if i + 1 < count else len(blob) - end = min(nxt, len(blob)) - if end > off: - out.append((i, off, blob[off:end])) - return out - - -def magic_of(blob: bytes) -> str: - return blob[:4].decode("latin-1") if len(blob) >= 4 else "" - - -def ext_for(blob: bytes) -> str: - return MAGIC_EXT.get(magic_of(blob), ".bin") - - -def safe_tag(magic: str) -> str: - return "".join(c if c.isalnum() else "_" for c in magic) or "raw" +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ac6_fhm import ext_for_blob, magic_of, parse_fhm, safe_tag def unpack(blob: bytes, out_dir: Path, root: Path, depth: int, max_depth: int) -> list[dict]: @@ -59,15 +16,18 @@ def unpack(blob: bytes, out_dir: Path, root: Path, depth: int, max_depth: int) - return [] recs = [] out_dir.mkdir(parents=True, exist_ok=True) - for idx, off, child in children: - magic = magic_of(child) - name = f"{idx:04d}_{safe_tag(magic)}{ext_for(child)}" + for child in children: + magic = child.magic + name = f"{child.index:04d}_{safe_tag(magic)}{ext_for_blob(child.data)}" path = out_dir / name - path.write_bytes(child) - rec = {"index": idx, "offset": off, "size": len(child), "magic": magic, - "path": str(path.relative_to(root)).replace("\\", "/")} - if depth < max_depth and child[:4] == b"FHM ": - nested = unpack(child, out_dir / f"{idx:04d}_FHM", root, depth + 1, max_depth) + path.write_bytes(child.data) + rec = {"index": child.index, "offset": child.offset, + "declared_size": child.declared_size, "size": child.size, + "magic": magic, "path": str(path.relative_to(root)).replace("\\", "/")} + if child.notes: + rec["parser_notes"] = child.notes + if depth < max_depth and child.data[:4] == b"FHM ": + nested = unpack(child.data, out_dir / f"{child.index:04d}_FHM", root, depth + 1, max_depth) if nested: rec["children"] = nested recs.append(rec) @@ -97,7 +57,7 @@ def main() -> int: stem = src.stem.split(".")[0] if blob[:4] != b"FHM ": # Top-level non-FHM (raw entry): copy through as a leaf. - dst = out_root / f"{stem}{ext_for(blob)}" + dst = out_root / f"{stem}{ext_for_blob(blob)}" dst.write_bytes(blob) manifest.append({"source": src.name, "kind": "leaf", "magic": magic_of(blob), "path": str(dst.relative_to(out_root)).replace("\\", "/")})