From 121c910b5722c3c40ccee167e7a69669b530d0c8 Mon Sep 17 00:00:00 2001 From: salh Date: Fri, 24 Apr 2026 17:01:15 +0300 Subject: [PATCH] Add AC6 PAC extraction and texture export tooling --- CMakeLists.txt | 1 + docs/TEXTURE_SWAPS.txt | 7 + docs/TEXTURE_SWAP_MODDING_GUIDE.txt | 27 + docs/ac6_asset_pipeline.md | 149 +++ src/ac6_pac_decode_dump.cpp | 71 ++ .../src/graphics/d3d12/command_processor.cpp | 241 ++++ .../src/graphics/d3d12/pipeline_cache.cpp | 12 +- .../graphics/d3d12/primitive_processor.cpp | 12 +- .../src/graphics/d3d12/texture_cache.cpp | 101 ++ .../pipeline/shader/dxbc_translator.cpp | 17 +- .../src/kernel/xboxkrnl/xboxkrnl_io.cpp | 81 ++ tools/export_ac6_ntxr.py | 1005 +++++++++++++++++ tools/extract_ac6_pac.py | 163 +++ tools/extract_ac6_runtime_fhm.py | 262 +++++ tools/launch_ac6_with_pac_dump.ps1 | 14 + tools/pac_probe_lzx.cpp | 333 ++++++ tools/parse_ac6_swg.py | 139 +++ tools/run_ac6_asset_pipeline.py | 179 +++ 18 files changed, 2795 insertions(+), 19 deletions(-) create mode 100644 docs/ac6_asset_pipeline.md create mode 100644 src/ac6_pac_decode_dump.cpp create mode 100644 tools/export_ac6_ntxr.py create mode 100644 tools/extract_ac6_pac.py create mode 100644 tools/extract_ac6_runtime_fhm.py create mode 100644 tools/launch_ac6_with_pac_dump.ps1 create mode 100644 tools/pac_probe_lzx.cpp create mode 100644 tools/parse_ac6_swg.py create mode 100644 tools/run_ac6_asset_pipeline.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ed2c4aa0..a61bfc41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ endif() # Sources set(AC6RECOMP_SOURCES src/main.cpp + src/ac6_pac_decode_dump.cpp src/d3d_hooks.cpp src/render_hooks.cpp src/ac6_texture_overrides.cpp diff --git a/docs/TEXTURE_SWAPS.txt b/docs/TEXTURE_SWAPS.txt index a934d593..7748eb0b 100644 --- a/docs/TEXTURE_SWAPS.txt +++ b/docs/TEXTURE_SWAPS.txt @@ -102,6 +102,13 @@ First pass limitations: - cube textures are skipped - unsupported DXGI formats fall back to the original guest texture +Current AC6 workflow notes: + - for uncompressed replacements, DX10 DDS with 8.8.8.8 RGBA has been the + safest authoring path + - for specular-style replacements, BC1 has been the most reliable format so far + - hot reload is useful for iteration, but a full game restart is still the + reliable check when cache reuse makes results look stale or mixed + The fallback path is always the original guest texture load. A bad or missing replacement will not block rendering. diff --git a/docs/TEXTURE_SWAP_MODDING_GUIDE.txt b/docs/TEXTURE_SWAP_MODDING_GUIDE.txt index d96b5c68..f22e56f7 100644 --- a/docs/TEXTURE_SWAP_MODDING_GUIDE.txt +++ b/docs/TEXTURE_SWAP_MODDING_GUIDE.txt @@ -184,12 +184,39 @@ wrong, or a higher-priority override is winning. What to do: - confirm the filename or manifest rule matches the dumped stable key - keep the same format, size, array/depth, and mip count + - for uncompressed replacements, prefer DX10 DDS with 8.8.8.8 RGBA + - for specular-style replacements, BC1 has been the most reliable format so far + - if you use a manifest, place it directly in override/textures or + mods//textures and use [[swap]] entries with source paths + relative to that textures folder - test with override/textures first Problem: The wrong texture is replaced. Cause: A wildcard manifest rule is too broad. What to do: Narrow the stable_key_globs rule or switch to exact stable_keys. +Problem: Auto reload works, but the result flips between skins or looks stale. +Cause: Texture cache reuse is currently best-effort for hot reload, and AC6 can +reuse texture layouts across appearances. +What to do: + - treat full game restart as the reliable validation path + - use exact stable_keys before trying wildcard manifest rules + - if a result looks "50/50", verify with a cold launch before assuming the + dump or replacement is wrong + +Current AC6 authoring notes + +These are not hard engine guarantees, but they match the current round of AC6 +experiments: + + - DX10 DDS plus 8.8.8.8 RGBA has been the safest choice for general texture + replacements + - BC1 has been the most reliable choice for specular replacements + - hangar and in-mission variants may not always use the same dumped texture, + even when the art looks related + - some captured textures may still need more investigation if the dump looks + incomplete or visibly corrupted + Practical advice - Use the current session folder first, not the full dump folder. diff --git a/docs/ac6_asset_pipeline.md b/docs/ac6_asset_pipeline.md new file mode 100644 index 00000000..fb7a0681 --- /dev/null +++ b/docs/ac6_asset_pipeline.md @@ -0,0 +1,149 @@ +# AC6 Asset Pipeline + +This pipeline extracts and converts the AC6 assets we currently understand. + +It automates these stages: + +1. `DATA.TBL` + `DATA00.PAC` / `DATA01.PAC` index extraction +2. Runtime PAC decode dump parsing +3. Recursive `FHM` extraction +4. `SWG` UI metadata parsing +5. `NTXR` texture export + +## Important Limitation + +The pipeline can process **all PAC entries and all runtime dumps you already have**, but it **cannot yet offline-decompress every compressed PAC entry in the archive** by itself. + +That means: + +- Raw PAC entries are handled in one pass. +- Runtime-decoded assets are handled in one pass. +- If a compressed asset has never been decoded by the game and never appeared in `out/ac6_pac_runtime_dump`, this pipeline cannot currently materialize it. + +So the pipeline is "all at once" for the **current corpus**, not yet "decode the entire PAC archive from scratch with no runtime help". + +If you want truly complete one-shot extraction of every compressed PAC asset without launching the game, the remaining missing piece is an offline implementation of AC6's mode-1 decompressor. + +## Prerequisites + +- The game assets exist at: + - `C:\ext\New folder\AC6_recomp\out\build\win-amd64-relwithdebinfo\assets` +- If you want runtime-decoded content included, you must already have: + - `C:\ext\New folder\AC6_recomp\out\ac6_pac_runtime_dump` + +To collect runtime dumps in future runs: + +```powershell +$env:AC6_DUMP_PAC_DECODED='1' +& 'C:\ext\New folder\AC6_recomp\out\build\win-amd64-relwithdebinfo\ac6recomp.exe' +``` + +Or use the launcher helper: + +```powershell +powershell -ExecutionPolicy Bypass -File .\tools\launch_ac6_with_pac_dump.ps1 +``` + +That script sets `AC6_DUMP_PAC_DECODED=1` and launches `ac6recomp.exe` for you. + +## One-Command Usage + +From the repo root: + +```powershell +python .\tools\run_ac6_asset_pipeline.py +``` + +This uses the default paths: + +- Asset root: `out\build\win-amd64-relwithdebinfo\assets` +- Raw PAC output: `out\ac6_pac_extracted_raw` +- Runtime dump input: `out\ac6_pac_runtime_dump` +- Typed FHM output: `out\ac6_runtime_fhm_typed` +- SWG output: `out\ac6_runtime_swg_parsed` +- Texture output: `out\ac6_runtime_ntxr_exported` + +The wrapper prints a final JSON summary with the current corpus totals, including: + +- PAC entries extracted +- runtime `FHM` container count +- parsed `SWG` files +- exported/skipped `NTXR` textures + +## Useful Variants + +Use a custom asset root: + +```powershell +python .\tools\run_ac6_asset_pipeline.py --asset-root 'C:\path\to\assets' +``` + +Skip PAC re-extraction and only process existing dumps: + +```powershell +python .\tools\run_ac6_asset_pipeline.py --skip-pac-extract +``` + +Recommended workflow after a new play session: + +```powershell +powershell -ExecutionPolicy Bypass -File .\tools\launch_ac6_with_pac_dump.ps1 +python .\tools\run_ac6_asset_pipeline.py --skip-pac-extract +``` + +Extract only PAC entries marked raw: + +```powershell +python .\tools\run_ac6_asset_pipeline.py --raw-only +``` + +## Output Folders + +- Raw PAC extraction: + - `C:\ext\New folder\AC6_recomp\out\ac6_pac_extracted_raw` +- Parsed runtime FHM corpus: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_fhm_typed` +- Parsed SWG metadata: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_swg_parsed` +- Exported textures: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_ntxr_exported` + +## What To Open + +- Main texture manifest: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_ntxr_exported\manifest.json` +- SWG manifest: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_swg_parsed\manifest.json` +- Example exported textures: + - `C:\ext\New folder\AC6_recomp\out\ac6_runtime_ntxr_exported` + +## Notes About Texture Types + +Not every texture is a normal color image. + +The exporter now handles: + +- Standard tiled `R8` atlases +- Standard tiled `RGBA8` atlases +- `BC1` textures +- `BC3` textures +- Packed-mip `BC3` families +- Six-face `BC1` assets using contact-sheet previews +- Several support textures stored as grayscale-in-ARGB + +Some outputs are still valid but are things like: + +- masks +- glyph sheets +- lookup textures +- normal maps +- cubemap-style faces + +## Current Status + +At the time this guide was written, the texture exporter was producing: + +- `845` exported textures +- `0` skipped textures + +That count applies to the currently available runtime dump corpus. diff --git a/src/ac6_pac_decode_dump.cpp b/src/ac6_pac_decode_dump.cpp new file mode 100644 index 00000000..bf79b0ff --- /dev/null +++ b/src/ac6_pac_decode_dump.cpp @@ -0,0 +1,71 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool DumpingEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("AC6_DUMP_PAC_DECODED"); + return value && value[0] && std::string_view(value) != "0"; + }(); + return enabled; +} + +std::filesystem::path DumpRoot() { + return std::filesystem::path("out") / "ac6_pac_runtime_dump"; +} + +std::mutex& DumpMutex() { + static std::mutex mutex; + return mutex; +} + +} // namespace + +void Ac6DumpPacDecodedEntry(uint16_t entry_index, uint8_t codec_mode, uint32_t compressed_size, + uint32_t decompressed_size, uint32_t source_offset, + const uint8_t* host_data) { + if (!DumpingEnabled() || !host_data || decompressed_size == 0) { + return; + } + + std::scoped_lock lock(DumpMutex()); + + std::error_code ec; + const auto root = DumpRoot(); + std::filesystem::create_directories(root, ec); + if (ec) { + REXFS_ERROR("[AC6 PAC] failed to create dump directory {}: {}", root.string(), ec.message()); + return; + } + + std::ostringstream name; + name << "entry_" << entry_index << "_mode" << static_cast(codec_mode) << "_c" << compressed_size + << "_u" << decompressed_size << "_off" << std::hex << source_offset << std::dec << ".bin"; + const auto path = root / name.str(); + + std::ofstream file(path, std::ios::binary | std::ios::trunc); + if (!file) { + REXFS_ERROR("[AC6 PAC] failed to open decoded dump {}", path.string()); + return; + } + + file.write(reinterpret_cast(host_data), static_cast(decompressed_size)); + if (!file) { + REXFS_ERROR("[AC6 PAC] failed to write decoded dump {}", path.string()); + return; + } + + REXFS_INFO( + "[AC6 PAC] dumped decoded entry index={} mode={} compressed=0x{:x} decompressed=0x{:x} " + "source_offset=0x{:x} path={}", + entry_index, static_cast(codec_mode), compressed_size, decompressed_size, source_offset, + path.string()); +} diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/command_processor.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/command_processor.cpp index 45f22372..9695fd77 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/command_processor.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/command_processor.cpp @@ -10,9 +10,11 @@ */ #include +#include #include #include #include +#include #include #include @@ -51,6 +53,97 @@ REXCVAR_DEFINE_BOOL(d3d12_submit_on_primary_buffer_end, true, "GPU/D3D12", namespace rex::graphics::d3d12 { +namespace { + +constexpr size_t kSuspiciousSpriteStateHistorySize = 16; + +struct SuspiciousSpriteStateWrite { + uint32_t reg_index = 0; + uint32_t value = 0; +}; + +std::array + g_suspicious_sprite_state_history; + uint32_t g_suspicious_sprite_state_history_count = 0; + size_t g_suspicious_sprite_state_history_pos = 0; + +bool IsSuspiciousSpriteStateRegister(uint32_t index) { + switch (index) { + case XE_GPU_REG_RB_MODECONTROL: + case XE_GPU_REG_RB_SURFACE_INFO: + case XE_GPU_REG_RB_COLORCONTROL: + case XE_GPU_REG_RB_COLOR_INFO: + case XE_GPU_REG_RB_COLOR_MASK: + case XE_GPU_REG_RB_BLENDCONTROL0: + case XE_GPU_REG_RB_DEPTHCONTROL: + case XE_GPU_REG_RB_DEPTH_INFO: + case XE_GPU_REG_PA_SU_POINT_SIZE: + case XE_GPU_REG_PA_SU_POINT_MINMAX: + return true; + default: + return false; + } +} + +const char* GetSuspiciousSpriteStateRegisterName(uint32_t index) { + switch (index) { + case XE_GPU_REG_RB_MODECONTROL: + return "RB_MODECONTROL"; + case XE_GPU_REG_RB_SURFACE_INFO: + return "RB_SURFACE_INFO"; + case XE_GPU_REG_RB_COLORCONTROL: + return "RB_COLORCONTROL"; + case XE_GPU_REG_RB_COLOR_INFO: + return "RB_COLOR_INFO0"; + case XE_GPU_REG_RB_COLOR_MASK: + return "RB_COLOR_MASK"; + case XE_GPU_REG_RB_BLENDCONTROL0: + return "RB_BLENDCONTROL0"; + case XE_GPU_REG_RB_DEPTHCONTROL: + return "RB_DEPTHCONTROL"; + case XE_GPU_REG_RB_DEPTH_INFO: + return "RB_DEPTH_INFO"; + case XE_GPU_REG_PA_SU_POINT_SIZE: + return "PA_SU_POINT_SIZE"; + case XE_GPU_REG_PA_SU_POINT_MINMAX: + return "PA_SU_POINT_MINMAX"; + default: + return "UNKNOWN"; + } +} + +void RecordSuspiciousSpriteStateWrite(uint32_t index, uint32_t value) { + if (!IsSuspiciousSpriteStateRegister(index)) { + return; + } + g_suspicious_sprite_state_history[g_suspicious_sprite_state_history_pos] = {index, value}; + g_suspicious_sprite_state_history_pos = + (g_suspicious_sprite_state_history_pos + 1) % kSuspiciousSpriteStateHistorySize; + g_suspicious_sprite_state_history_count = std::min( + g_suspicious_sprite_state_history_count + 1, kSuspiciousSpriteStateHistorySize); +} + +std::string FormatSuspiciousSpriteStateHistory() { + std::ostringstream stream; + stream << "TrailStateHistory:"; + if (!g_suspicious_sprite_state_history_count) { + stream << " "; + return stream.str(); + } + size_t start = (g_suspicious_sprite_state_history_pos + kSuspiciousSpriteStateHistorySize - + g_suspicious_sprite_state_history_count) % + kSuspiciousSpriteStateHistorySize; + stream << std::hex << std::uppercase; + for (uint32_t i = 0; i < g_suspicious_sprite_state_history_count; ++i) { + const SuspiciousSpriteStateWrite& write = + g_suspicious_sprite_state_history[(start + i) % kSuspiciousSpriteStateHistorySize]; + stream << ' ' << GetSuspiciousSpriteStateRegisterName(write.reg_index) << "=0x" << write.value; + } + return stream.str(); +} + +} // namespace + // Generated with `xb buildshaders`. namespace shaders { #include "../shaders/bytecode/d3d12_5_1/apply_gamma_pwl_cs.h" @@ -1847,6 +1940,7 @@ void D3D12CommandProcessor::ShutdownContext() { void D3D12CommandProcessor::WriteRegister(uint32_t index, uint32_t value) { CommandProcessor::WriteRegister(index, value); + RecordSuspiciousSpriteStateWrite(index, value); if (index >= XE_GPU_REG_SHADER_CONSTANT_000_X && index <= XE_GPU_REG_SHADER_CONSTANT_511_W) { if (frame_open_) { @@ -2579,6 +2673,124 @@ bool D3D12CommandProcessor::IssueDraw(xenos::PrimitiveType primitive_type, uint3 uint32_t normalized_color_mask = pixel_shader ? draw_util::GetNormalizedColorMask(regs, pixel_shader->writes_color_targets()) : 0; + bool forced_ac6_trail_color_mask = false; + if (pixel_shader && !normalized_color_mask && + primitive_processing_result.guest_primitive_type == xenos::PrimitiveType::kPointList && + regs.Get().edram_mode == xenos::EdramMode::kColorDepth && + vertex_shader->ucode_data_hash() == UINT64_C(0xC049A8C9E556F129) && + pixel_shader->ucode_data_hash() == UINT64_C(0x2E372EA28CC404B7)) { + xenos::ColorRenderTargetFormat color_format = + regs.Get(reg::RB_COLOR_INFO::rt_register_indices[0]).color_format; + uint32_t format_component_count = + xenos::GetColorRenderTargetFormatComponentCount(color_format); + if (format_component_count) { + uint32_t format_component_mask = (uint32_t(1) << format_component_count) - 1; + normalized_color_mask = format_component_mask | (uint32_t(0b1111) & ~format_component_mask); + forced_ac6_trail_color_mask = true; + static bool logged_forced_ac6_trail_color_mask = false; + if (!logged_forced_ac6_trail_color_mask) { + logged_forced_ac6_trail_color_mask = true; + REXGPU_WARN( + "Forcing RT0 color writes for AC6 trail point-list pass VS {:016X} / PS {:016X}", + vertex_shader->ucode_data_hash(), pixel_shader->ucode_data_hash()); + } + } + } + bool trace_sprite_draw = + primitive_processing_result.guest_primitive_type == xenos::PrimitiveType::kPointList || + primitive_processing_result.guest_primitive_type == xenos::PrimitiveType::kRectangleList || + primitive_processing_result.host_primitive_type == xenos::PrimitiveType::kPointList || + primitive_processing_result.host_primitive_type == xenos::PrimitiveType::kRectangleList; + if (trace_sprite_draw) { + uint64_t vertex_shader_hash = vertex_shader->ucode_data_hash(); + uint64_t pixel_shader_hash = pixel_shader ? pixel_shader->ucode_data_hash() : 0; + auto pa_su_point_size = regs.Get(); + auto pa_su_point_minmax = regs.Get(); + auto rb_modecontrol = regs.Get(); + auto sq_program_cntl = regs.Get(); + auto sq_context_misc = regs.Get(); + auto rb_colorcontrol = regs.Get(); + auto rt0_blendcontrol = + regs.Get(reg::RB_BLENDCONTROL::rt_register_indices[0]); + uint32_t raw_rb_color_mask = regs[XE_GPU_REG_RB_COLOR_MASK]; + uint32_t edram_mode = uint32_t(rb_modecontrol.edram_mode); + uint32_t sq_param_gen = sq_program_cntl.param_gen; + uint32_t sq_param_gen_pos = sq_context_misc.param_gen_pos; + uint32_t ps_param_gen_enable = pixel_shader_modification.pixel.param_gen_enable; + uint32_t ps_param_gen_interpolator = + pixel_shader_modification.pixel.param_gen_interpolator; + uint32_t ps_param_gen_point = pixel_shader_modification.pixel.param_gen_point; + uint32_t ps_writes_color_targets = pixel_shader ? pixel_shader->writes_color_targets() : 0; + uint32_t ps_writes_depth = pixel_shader ? pixel_shader->writes_depth() : 0; + uint32_t vs_output_point_size = vertex_shader_modification.vertex.output_point_size; + uint32_t alpha_test_enable = rb_colorcontrol.alpha_test_enable; + uint32_t alpha_to_mask_enable = rb_colorcontrol.alpha_to_mask_enable; + uint32_t depth_enable = normalized_depth_control.z_enable; + uint32_t depth_write_enable = normalized_depth_control.z_write_enable; + uint32_t point_width = pa_su_point_size.width; + uint32_t point_height = pa_su_point_size.height; + uint32_t point_min_size = pa_su_point_minmax.min_size; + uint32_t point_max_size = pa_su_point_minmax.max_size; + uint64_t trace_key = vertex_shader_hash; + trace_key ^= pixel_shader_hash * UINT64_C(0x9E3779B185EBCA87); + trace_key ^= uint64_t(primitive_processing_result.guest_primitive_type) << 1; + trace_key ^= uint64_t(primitive_processing_result.host_primitive_type) << 5; + trace_key ^= uint64_t(primitive_processing_result.host_vertex_shader_type) << 9; + trace_key ^= uint64_t(pa_su_point_size.value) << 32; + trace_key ^= uint64_t(pa_su_point_minmax.value); + trace_key ^= uint64_t(normalized_depth_control.value) << 16; + trace_key ^= uint64_t(raw_rb_color_mask) << 20; + trace_key ^= uint64_t(normalized_color_mask) << 48; + trace_key ^= uint64_t(rb_colorcontrol.value); + trace_key ^= uint64_t(ps_writes_color_targets) << 24; + trace_key ^= uint64_t(ps_writes_depth) << 28; + trace_key ^= uint64_t(edram_mode) << 60; + trace_key ^= uint64_t(ps_param_gen_enable) << 13; + trace_key ^= uint64_t(ps_param_gen_point) << 14; + trace_key ^= uint64_t(ps_param_gen_interpolator) << 15; + static std::unordered_set logged_trace_keys; + if (logged_trace_keys.emplace(trace_key).second) { + REXGPU_INFO( + "SpriteTrace: guest_prim={} host_prim={} host_vs_type={} VS={:016X}/{}b " + "PS={:016X}/{}b interp_mask={:04X} ps_param_gen_pos={} sq_param_gen={} " + "sq_param_gen_pos={} ps_param_gen_enable={} ps_param_gen_interp={} " + "ps_param_gen_point={} vs_output_point_size={} vs_point_writes={} " + "point_size={}x{} point_minmax={}..{} rb_color_mask={:04X} " + "ps_writes_color_targets={:X} ps_writes_depth={} color_mask={:04X} " + "forced_color_mask={} " + "edram_mode={} alpha_test={} a2c={} z_enable={} z_write={} zfunc={} " + "rt0_blend=({},{},{},{},{},{})", + uint32_t(primitive_processing_result.guest_primitive_type), + uint32_t(primitive_processing_result.host_primitive_type), + uint32_t(primitive_processing_result.host_vertex_shader_type), vertex_shader_hash, + vertex_shader->ucode_dword_count() * sizeof(uint32_t), pixel_shader_hash, + pixel_shader ? (pixel_shader->ucode_dword_count() * sizeof(uint32_t)) : 0, + interpolator_mask, ps_param_gen_pos == UINT32_MAX ? -1 : int32_t(ps_param_gen_pos), + sq_param_gen, sq_param_gen_pos, ps_param_gen_enable, ps_param_gen_interpolator, + ps_param_gen_point, vs_output_point_size, + vertex_shader->writes_point_size_edge_flag_kill_vertex(), point_width, point_height, + point_min_size, point_max_size, raw_rb_color_mask, ps_writes_color_targets, + ps_writes_depth, normalized_color_mask, forced_ac6_trail_color_mask, edram_mode, + alpha_test_enable, + alpha_to_mask_enable, depth_enable, + depth_write_enable, uint32_t(normalized_depth_control.zfunc), + uint32_t(rt0_blendcontrol.color_srcblend), + uint32_t(rt0_blendcontrol.color_destblend), + uint32_t(rt0_blendcontrol.color_comb_fcn), + uint32_t(rt0_blendcontrol.alpha_srcblend), + uint32_t(rt0_blendcontrol.alpha_destblend), + uint32_t(rt0_blendcontrol.alpha_comb_fcn)); + } + bool trace_trail_state_history = + raw_rb_color_mask == 0 && + (vertex_shader_hash == UINT64_C(0xC049A8C9E556F129) || + pixel_shader_hash == UINT64_C(0x2E372EA28CC404B7)); + static std::unordered_set logged_trace_state_history_keys; + if (trace_trail_state_history && + logged_trace_state_history_keys.emplace(trace_key ^ UINT64_C(0xA5A5A5A5F00DFACE)).second) { + REXGPU_INFO("{}", FormatSuspiciousSpriteStateHistory()); + } + } if (!render_target_cache_->Update(is_rasterization_done, normalized_depth_control, normalized_color_mask, *vertex_shader)) { return false; @@ -2604,6 +2816,35 @@ bool D3D12CommandProcessor::IssueDraw(xenos::PrimitiveType primitive_type, uint3 bound_depth_and_color_render_target_formats); } else { bound_depth_and_color_render_target_bits = 0; + std::memset(bound_depth_and_color_render_target_formats, 0, + sizeof(bound_depth_and_color_render_target_formats)); + } + if (trace_sprite_draw) { + uint64_t vertex_shader_hash = vertex_shader->ucode_data_hash(); + uint64_t pixel_shader_hash = pixel_shader ? pixel_shader->ucode_data_hash() : 0; + uint64_t trace_rt_key = vertex_shader_hash; + trace_rt_key ^= pixel_shader_hash * UINT64_C(0xD6E8FEB86659FD93); + trace_rt_key ^= uint64_t(primitive_processing_result.guest_primitive_type) << 1; + trace_rt_key ^= uint64_t(primitive_processing_result.host_primitive_type) << 5; + trace_rt_key ^= uint64_t(normalized_depth_control.value) << 16; + trace_rt_key ^= uint64_t(normalized_color_mask) << 48; + trace_rt_key ^= uint64_t(bound_depth_and_color_render_target_bits) << 24; + for (uint32_t i = 0; i < 1 + xenos::kMaxColorRenderTargets; ++i) { + trace_rt_key ^= + uint64_t(bound_depth_and_color_render_target_formats[i]) << ((i * 11) & 63); + } + static std::unordered_set logged_trace_rt_keys; + if (logged_trace_rt_keys.emplace(trace_rt_key).second) { + REXGPU_INFO( + "SpriteTraceRT: host_rt_path={} bound_bits={:02X} formats=[{:X},{:X},{:X},{:X},{:X}] " + "normalized_color_mask={:04X}", + uint32_t(render_target_cache_->GetPath()), bound_depth_and_color_render_target_bits, + bound_depth_and_color_render_target_formats[0], + bound_depth_and_color_render_target_formats[1], + bound_depth_and_color_render_target_formats[2], + bound_depth_and_color_render_target_formats[3], + bound_depth_and_color_render_target_formats[4], normalized_color_mask); + } } void* pipeline_handle; ID3D12RootSignature* root_signature; diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp index 35c40ba4..23a1ea9f 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp @@ -2013,13 +2013,9 @@ void PipelineCache::CreateDxbcGeometryShader(GeometryShaderKey key, // Names (after the parameters). name_ptr = uint32_t((shader_out.size() - osgn_position_dwords) * sizeof(uint32_t)); uint32_t osgn_name_ptr_texcoord = name_ptr; - if (key.interpolator_count) { + if (key.interpolator_count || key.has_point_coordinates) { name_ptr += dxbc::AppendAlignedString(shader_out, "TEXCOORD"); } - uint32_t osgn_name_ptr_xespritetexcoord = name_ptr; - if (key.has_point_coordinates) { - name_ptr += dxbc::AppendAlignedString(shader_out, "XESPRITETEXCOORD"); - } uint32_t osgn_name_ptr_sv_position = name_ptr; name_ptr += dxbc::AppendAlignedString(shader_out, "SV_Position"); uint32_t osgn_name_ptr_sv_clip_distance = name_ptr; @@ -2059,13 +2055,15 @@ void PipelineCache::CreateDxbcGeometryShader(GeometryShaderKey key, } } - // Point coordinates (XESPRITETEXCOORD). + // Point coordinates use the next TEXCOORD semantic after guest interpolators + // so the DXBC->DXIL path doesn't depend on custom GS->PS semantic names. if (key.has_point_coordinates) { output_register_point_coordinates = output_register_index; assert_true(osgn_parameter_index < osgn_parameter_count); dxbc::SignatureParameterForGS& osgn_point_coordinates = osgn_parameters[osgn_parameter_index++]; - osgn_point_coordinates.semantic_name_ptr = osgn_name_ptr_xespritetexcoord; + osgn_point_coordinates.semantic_name_ptr = osgn_name_ptr_texcoord; + osgn_point_coordinates.semantic_index = key.interpolator_count; osgn_point_coordinates.component_type = dxbc::SignatureRegisterComponentType::kFloat32; osgn_point_coordinates.register_index = output_register_index++; osgn_point_coordinates.mask = 0b0011; diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/primitive_processor.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/primitive_processor.cpp index 82295cff..00da28b0 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/primitive_processor.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/primitive_processor.cpp @@ -24,10 +24,11 @@ #include REXCVAR_DEFINE_BOOL( - d3d12_expand_point_sprites_in_vs, true, "GPU/D3D12", - "Expand Xbox point lists as triangle strips in the vertex shader instead of the host " - "point-list + geometry-shader path. Fixes missing trails/particles on some titles and " - "drivers where native point expansion is culled or rasterized incorrectly.") + d3d12_expand_point_sprites_in_vs, false, "GPU/D3D12", + "Experimental: expand Xbox point lists as triangle strips in the vertex shader instead of " + "the host point-list + geometry-shader path. Disabled by default on D3D12 because the DXBC " + "fallback path is not yet reliable enough for AC6 point-sprite effects such as missile " + "trails.") .lifecycle(rex::cvar::Lifecycle::kInitOnly); namespace rex::graphics::d3d12 { @@ -37,6 +38,9 @@ D3D12PrimitiveProcessor::~D3D12PrimitiveProcessor() { } bool D3D12PrimitiveProcessor::Initialize() { + // Keep the DXBC VS-expansion fallback opt-in for debugging, but prefer the + // native point-list + geometry-shader path by default until the D3D12 + // translator matches the Vulkan point-sprite fallback behavior. const bool point_sprites_native_without_vs = !REXCVAR_GET(d3d12_expand_point_sprites_in_vs); if (!InitializeCommon(true, false, false, true, point_sprites_native_without_vs, true)) { Shutdown(); diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/texture_cache.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/texture_cache.cpp index 9328f49f..62b861c5 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/texture_cache.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/texture_cache.cpp @@ -14,7 +14,12 @@ #include #include #include +#include #include +#include +#include +#include +#include #include #include @@ -37,6 +42,10 @@ namespace rex::graphics::d3d12 { +REXCVAR_DEFINE_BOOL(d3d12_log_bc1_diagnostics, false, "GPU/D3D12", + "Log detailed diagnostics for BC1 texture loads and dumps") + .lifecycle(rex::cvar::Lifecycle::kHotReload); + // Generated with `xb buildshaders`. namespace shaders { #include "../shaders/bytecode/d3d12_5_1/texture_load_128bpb_cs.h" @@ -99,6 +108,9 @@ namespace { constexpr D3D12_FORMAT_SUPPORT1 kLinearFilterSupport = D3D12_FORMAT_SUPPORT1_SHADER_SAMPLE; +std::mutex g_bc1_diagnostic_mutex; +std::unordered_set g_bc1_diagnostic_logged_keys; + bool IsFormatSampleFilterable(ID3D12Device* device, DXGI_FORMAT format) { if (format == DXGI_FORMAT_UNKNOWN) { return false; @@ -112,6 +124,55 @@ bool IsFormatSampleFilterable(ID3D12Device* device, DXGI_FORMAT format) { return (support.Support1 & kLinearFilterSupport) == kLinearFilterSupport; } +bool ShouldLogBc1Diagnostics(xenos::TextureFormat format, xenos::Endian endianness) { + return REXCVAR_GET(d3d12_log_bc1_diagnostics) && + GetBaseFormat(format) == xenos::TextureFormat::k_DXT1 && + endianness != xenos::Endian::kNone; +} + +bool MarkBc1DiagnosticKeyLogged(std::string_view stable_key) { + std::lock_guard lock(g_bc1_diagnostic_mutex); + return g_bc1_diagnostic_logged_keys.insert(std::string(stable_key)).second; +} + +const char* GetEndianDebugName(xenos::Endian endianness) { + switch (endianness) { + case xenos::Endian::kNone: + return "kNone"; + case xenos::Endian::k8in16: + return "k8in16"; + case xenos::Endian::k8in32: + return "k8in32"; + case xenos::Endian::k16in32: + return "k16in32"; + default: + return "unknown"; + } +} + +const char* GetLoadShaderDebugName(uint32_t index) { + switch (index) { + case 3: + return "kLoadShaderIndex64bpb"; + case 24: + return "kLoadShaderIndexDXT1ToRGBA8"; + default: + return "other"; + } +} + +std::string FormatByteSample(const uint8_t* data, size_t count) { + std::ostringstream stream; + stream << std::hex << std::uppercase << std::setfill('0'); + for (size_t i = 0; i < count; ++i) { + if (i != 0) { + stream << ' '; + } + stream << std::setw(2) << uint32_t(data[i]); + } + return stream.str(); +} + } // namespace const D3D12TextureCache::HostFormat D3D12TextureCache::kBestHostFormats[64] = { @@ -1849,6 +1910,35 @@ bool D3D12TextureCache::LoadTextureDataFromResidentMemoryImpl(Texture& texture, (is_base ? host_slice_size_base : host_slice_sizes_mips[level]) = level_host_slice_size; copy_buffer_size += level_host_slice_size * array_size; } + const bool log_bc1_diagnostics = ShouldLogBc1Diagnostics(guest_format, texture_key.endianness); + std::string bc1_diagnostic_stable_key; + if (log_bc1_diagnostics) { + const uint64_t texture_key_hash = XXH3_64bits(&texture_key, sizeof(texture_key)); + bc1_diagnostic_stable_key = ac6::textures::BuildTextureStableKey( + texture_key_hash, texture_key.base_page, texture_key.mip_page, uint32_t(texture_key.dimension), + width, height, depth_or_array_size, texture_key.mip_max_level + 1, uint32_t(texture_key.format), + uint32_t(texture_key.endianness), texture_key.tiled != 0, texture_key.packed_mips != 0, + texture_key.signed_separate != 0, texture_key.scaled_resolve != 0); + if (MarkBc1DiagnosticKeyLogged(bc1_diagnostic_stable_key)) { + REXGPU_INFO( + "BC1 diagnostic {}: shader={} host_copy_format={} sample_format={} tiled={} packed={} " + "scaled={} endian={} base=0x{:08X} mip=0x{:08X} size={}x{}x{} mips={} " + "guest_base_row_pitch={} guest_base_z_rows={} guest_base_slice_stride=0x{:X} " + "guest_base_extent=0x{:X} guest_mips_extent=0x{:X} host_base_row_pitch={} " + "host_base_width={} host_base_height={} host_base_depth={} copy_buffer_size=0x{:X}", + bc1_diagnostic_stable_key, GetLoadShaderDebugName(load_shader), + ac6::textures::DescribeDxgiFormat(host_copy_format), + ac6::textures::DescribeDxgiFormat(host_sample_format), texture_key.tiled ? 1 : 0, + texture_key.packed_mips ? 1 : 0, texture_resolution_scaled ? 1 : 0, + GetEndianDebugName(texture_key.endianness), texture_key.base_page << 12, texture_key.mip_page << 12, + width, height, depth_or_array_size, texture_key.mip_max_level + 1, guest_layout.base.row_pitch_bytes, + guest_layout.base.z_slice_stride_block_rows, guest_layout.base.array_slice_stride_bytes, + guest_layout.base.level_data_extent_bytes, guest_layout.mips_total_extent_bytes, + host_slice_layout_base.Footprint.RowPitch, host_slice_layout_base.Footprint.Width, + host_slice_layout_base.Footprint.Height, host_slice_layout_base.Footprint.Depth, + uint32_t(copy_buffer_size)); + } + } D3D12_RESOURCE_STATES copy_buffer_state = D3D12_RESOURCE_STATE_UNORDERED_ACCESS; ID3D12Resource* copy_buffer = command_processor_.RequestScratchGPUBuffer(uint32_t(copy_buffer_size), copy_buffer_state); @@ -2313,6 +2403,17 @@ void D3D12TextureCache::ProcessCompletedTextureTransfers() { continue; } + if (ShouldLogBc1Diagnostics(xenos::TextureFormat(it->guest_format), xenos::Endian(it->endianness)) && + !dds_image.subresources.empty() && !dds_image.subresources.front().data.empty()) { + const ac6::textures::DdsSubresource& subresource = dds_image.subresources.front(); + const size_t sample_size = std::min(subresource.data.size(), 8); + REXGPU_INFO( + "BC1 diagnostic {}: dump_format={} first_block={} row_pitch={} slice_pitch={} payload_size={}", + it->stable_key, ac6::textures::DescribeDxgiFormat(dds_image.format), + FormatByteSample(subresource.data.data(), sample_size), subresource.row_pitch, + subresource.slice_pitch, subresource.data.size()); + } + ac6::textures::TextureDumpMetadata metadata; metadata.stable_key = it->stable_key; metadata.texture_key_hash = it->texture_key_hash; diff --git a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator.cpp b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator.cpp index f6432bd1..dd97cb1a 100644 --- a/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/pipeline/shader/dxbc_translator.cpp @@ -2602,7 +2602,8 @@ void DxbcShaderTranslator::WriteInputSignature() { } } - // Point coordinates for PsParamGen (XESPRITETEXCOORD). + // Point coordinates for PsParamGen. Use the next TEXCOORD semantic after + // guest interpolators for better DXBC->DXIL linkage reliability on D3D12. size_t point_coordinates_position = shader_object_.size(); if (in_reg_ps_point_coordinates_ != UINT32_MAX) { shader_object_.resize(shader_object_.size() + kParameterDwords); @@ -2610,6 +2611,7 @@ void DxbcShaderTranslator::WriteInputSignature() { { auto& point_coordinates = *reinterpret_cast( shader_object_.data() + point_coordinates_position); + point_coordinates.semantic_index = interpolator_count; point_coordinates.component_type = dxbc::SignatureRegisterComponentType::kFloat32; point_coordinates.register_index = in_reg_ps_point_coordinates_; point_coordinates.mask = 0b0011; @@ -2665,20 +2667,19 @@ void DxbcShaderTranslator::WriteInputSignature() { // Semantic names. uint32_t semantic_offset = uint32_t((shader_object_.size() - blob_position) * sizeof(uint32_t)); - if (interpolator_count) { + if (interpolator_count || in_reg_ps_point_coordinates_ != UINT32_MAX) { auto interpolators = reinterpret_cast(shader_object_.data() + interpolator_position); for (uint32_t i = 0; i < interpolator_count; ++i) { interpolators[i].semantic_name_ptr = semantic_offset; } + if (in_reg_ps_point_coordinates_ != UINT32_MAX) { + auto& point_coordinates = *reinterpret_cast( + shader_object_.data() + point_coordinates_position); + point_coordinates.semantic_name_ptr = semantic_offset; + } semantic_offset += dxbc::AppendAlignedString(shader_object_, "TEXCOORD"); } - if (in_reg_ps_point_coordinates_ != UINT32_MAX) { - auto& point_coordinates = *reinterpret_cast( - shader_object_.data() + point_coordinates_position); - point_coordinates.semantic_name_ptr = semantic_offset; - semantic_offset += dxbc::AppendAlignedString(shader_object_, "XESPRITETEXCOORD"); - } { auto& position = *reinterpret_cast(shader_object_.data() + position_position); diff --git a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_io.cpp b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_io.cpp index 81dd8aee..517b1389 100644 --- a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_io.cpp +++ b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_io.cpp @@ -6,6 +6,9 @@ // Disable warnings about unused parameters for kernel functions #pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#include + #include #include #include @@ -14,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +30,36 @@ namespace rex::kernel::xboxkrnl { using namespace rex::system; +namespace { + +bool AsciiCaseInsensitiveEquals(char left, char right) { + return std::tolower(static_cast(left)) == + std::tolower(static_cast(right)); +} + +bool ContainsAsciiInsensitive(std::string_view haystack, std::string_view needle) { + return std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end(), + AsciiCaseInsensitiveEquals) != haystack.end(); +} + +bool IsFocusedAc6PacPath(std::string_view path) { + return ContainsAsciiInsensitive(path, "DATA00.PAC") || + ContainsAsciiInsensitive(path, "DATA01.PAC") || + ContainsAsciiInsensitive(path, "DATA.TBL"); +} + +uint32_t CurrentGuestCallerAddress() { + auto* thread_state = runtime::ThreadState::Get(); + if (!thread_state || !thread_state->context()) { + return 0; + } + + const uint32_t lr = static_cast(thread_state->context()->lr); + return lr >= 4 ? (lr - 4) : 0; +} + +} // namespace + struct CreateOptions { // https://processhacker.sourceforge.io/doc/ntioapi_8h.html static const uint32_t FILE_DIRECTORY_FILE = 0x00000001; @@ -122,6 +156,14 @@ ppc_u32_result_t NtCreateFile_entry(ppc_pu32_t handle_out, ppc_u32_t desired_acc "NtCreateFile", "path={} access={:#x} attrs={:#x} share={:#x} disp={:#x} options={:#x}", target_path, (uint32_t)desired_access, (uint32_t)file_attributes, (uint32_t)share_access, (uint32_t)creation_disposition, (uint32_t)create_options); + if (IsFocusedAc6PacPath(target_path)) { + REXKRNL_INFO( + "[AC6 PAC] NtCreateFile caller={:08X} path={} access={:#x} attrs={:#x} share={:#x} " + "disp={:#x} options={:#x}", + CurrentGuestCallerAddress(), target_path, (uint32_t)desired_access, + (uint32_t)file_attributes, (uint32_t)share_access, (uint32_t)creation_disposition, + (uint32_t)create_options); + } // Enforce that the path is ASCII. if (!IsValidPath(target_path, false)) { @@ -206,6 +248,16 @@ ppc_u32_result_t NtReadFile_entry(ppc_u32_t file_handle, ppc_u32_t event_handle, result = X_STATUS_INVALID_HANDLE; } + const bool focused_pac_read = file && IsFocusedAc6PacPath(file->path()); + if (focused_pac_read) { + REXKRNL_INFO( + "[AC6 PAC] NtReadFile request caller={:08X} thid={} path={} handle={:#x} len={:#x} " + "offset={} sync={}", + CurrentGuestCallerAddress(), XThread::GetCurrentThreadId(), file->path(), + (uint32_t)file_handle, (uint32_t)buffer_length, byte_offset_ptr ? (int64_t)byte_offset : -1, + file->is_synchronous()); + } + if (XSUCCEEDED(result)) { uint32_t bytes_read = 0; result = file->Read(buffer.guest_address(), buffer_length, @@ -239,6 +291,15 @@ ppc_u32_result_t NtReadFile_entry(ppc_u32_t file_handle, ppc_u32_t event_handle, result = X_STATUS_PENDING; } signal_event = true; + + if (focused_pac_read) { + REXKRNL_INFO( + "[AC6 PAC] NtReadFile result caller={:08X} path={} status={:#x} bytes_read={:#x} " + "iosb_status={:#x} iosb_info={:#x}", + CurrentGuestCallerAddress(), file->path(), result, bytes_read, + io_status_block ? (uint32_t)io_status_block->status : 0xFFFFFFFFu, + io_status_block ? (uint32_t)io_status_block->information : 0xFFFFFFFFu); + } } if (XFAILED(result) && io_status_block) { @@ -285,6 +346,17 @@ ppc_u32_result_t NtReadFileScatter_entry(ppc_u32_t file_handle, ppc_u32_t event_ result = X_STATUS_INVALID_HANDLE; } + const bool focused_pac_read = file && IsFocusedAc6PacPath(file->path()); + if (focused_pac_read) { + const uint64_t byte_offset = byte_offset_ptr ? static_cast(*byte_offset_ptr) : 0; + REXKRNL_INFO( + "[AC6 PAC] NtReadFileScatter request caller={:08X} thid={} path={} handle={:#x} " + "len={:#x} offset={} sync={}", + CurrentGuestCallerAddress(), XThread::GetCurrentThreadId(), file->path(), + (uint32_t)file_handle, (uint32_t)length, byte_offset_ptr ? (int64_t)byte_offset : -1, + file->is_synchronous()); + } + if (XSUCCEEDED(result)) { uint32_t bytes_read = 0; result = file->ReadScatter(segment_array.guest_address(), length, @@ -305,6 +377,15 @@ ppc_u32_result_t NtReadFileScatter_entry(ppc_u32_t file_handle, ppc_u32_t event_ result = X_STATUS_PENDING; } signal_event = true; + + if (focused_pac_read) { + REXKRNL_INFO( + "[AC6 PAC] NtReadFileScatter result caller={:08X} path={} status={:#x} bytes_read={:#x} " + "iosb_status={:#x} iosb_info={:#x}", + CurrentGuestCallerAddress(), file->path(), result, bytes_read, + io_status_block ? (uint32_t)io_status_block->status : 0xFFFFFFFFu, + io_status_block ? (uint32_t)io_status_block->information : 0xFFFFFFFFu); + } } if (XFAILED(result) && io_status_block) { diff --git a/tools/export_ac6_ntxr.py b/tools/export_ac6_ntxr.py new file mode 100644 index 00000000..e3206f88 --- /dev/null +++ b/tools/export_ac6_ntxr.py @@ -0,0 +1,1005 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import struct +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image, ImageFilter + + +DDS_MAGIC = 0x20534444 +DDS_PF_ALPHAPIXELS = 0x00000001 +DDS_PF_RGB = 0x00000040 +DDS_HEADER_FLAGS_TEXTURE = 0x00001007 +DDS_HEADER_FLAGS_PITCH = 0x00000008 +DDS_HEADER_FLAGS_MIPMAP = 0x00020000 +DDS_CAPS_TEXTURE = 0x00001000 +DDS_CAPS_COMPLEX = 0x00000008 +DDS_CAPS_MIPMAP = 0x00400000 + + +def be16(blob: bytes, offset: int) -> int: + return struct.unpack_from(">H", blob, offset)[0] + + +def be32(blob: bytes, offset: int) -> int: + return struct.unpack_from(">I", blob, offset)[0] + + +def align_up(value: int, alignment: int) -> int: + return ((value + alignment - 1) // alignment) * alignment + + +def safe_ascii(blob: bytes) -> str: + return "".join(chr(b) if 32 <= b < 127 else "." for b in blob) + + +@dataclass +class ExportPlan: + layout: str + format_name: str + visible_width: int + visible_height: int + storage_width: int + storage_height: int + mip_count: int + payload_offset: int + payload_size: int + mip_sizes: list[int] + notes: list[str] + + +def bc_mip0_storage_size(width: int, height: int, bytes_per_block: int) -> int: + width_blocks = (width + 3) // 4 + height_blocks = (height + 3) // 4 + return align_up(width_blocks, 32) * align_up(height_blocks, 32) * bytes_per_block + + +def looks_like_gray_argb(payload: bytes) -> bool: + if len(payload) < 4 or len(payload) % 4 != 0: + return False + for i in range(0, len(payload), 4): + if payload[i] != 0xFF: + return False + if payload[i + 1] != payload[i + 2] or payload[i + 2] != payload[i + 3]: + return False + return True + + +def classify_ntxr(blob: bytes) -> tuple[ExportPlan | None, str | None]: + if len(blob) < 0x60 or blob[:4] != b"NTXR": + return None, "not_ntxr" + + variant_a = be16(blob, 0x20) + variant_b = be16(blob, 0x22) + width = be16(blob, 0x24) + height = be16(blob, 0x26) + declared_payload_size = be32(blob, 0x18) + + if width == 0 or height == 0 or declared_payload_size == 0: + return None, "invalid_dimensions" + + explicit_mip_sizes = [] + if variant_a > 1 and len(blob) >= 0x40 + (variant_a * 4): + explicit_mip_sizes = [be32(blob, 0x40 + (i * 4)) for i in range(variant_a)] + + if ( + variant_a > 1 + and explicit_mip_sizes + and all(size > 0 for size in explicit_mip_sizes) + and sum(explicit_mip_sizes) == declared_payload_size + and len(blob) >= 0x70 + declared_payload_size + and variant_b == 2 + ): + first_mip_size = explicit_mip_sizes[0] + if first_mip_size == width * height * 4: + format_name = "rgba8" + elif first_mip_size == width * height: + format_name = "r8" + else: + return None, f"unsupported_explicit_mip_size_{first_mip_size}" + return ( + ExportPlan( + layout=f"{format_name}_mipped_0x70", + format_name=format_name, + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=variant_a, + payload_offset=0x70, + payload_size=declared_payload_size, + mip_sizes=explicit_mip_sizes, + notes=["explicit mip sizes at 0x40"], + ), + None, + ) + + if ( + variant_a > 1 + and variant_b in (1, 2) + and len(explicit_mip_sizes) >= 2 + and len(blob) >= 0x1000 + declared_payload_size + ): + bc3_mip0_size = bc_mip0_storage_size(width, height, 16) + if explicit_mip_sizes[0] == bc3_mip0_size and explicit_mip_sizes[0] + explicit_mip_sizes[1] == declared_payload_size: + return ( + ExportPlan( + layout="bc3_swap16_mipped_0x1000", + format_name="bc3_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=variant_a, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=explicit_mip_sizes, + notes=["packed BC3 mip chain", "16-bit endian-swapped blocks", "preview exports mip 0"], + ), + None, + ) + + if variant_a == 1 and variant_b == 19 and declared_payload_size == width * height * 4: + # Some variant_b=19 textures store a short header only, while others + # have a larger metadata/padding block and the texel payload starts at + # 0x1000. Prefer the larger offset when present - it fixes the torn + # aircraft/body atlases and still produces sane results for the simpler + # UI textures. + if len(blob) >= 0x1000 + declared_payload_size: + payload_offset = 0x1000 + layout = "rgba8_single_0x1000" + notes = ["variant_b=19", "ARGB payload at 0x1000"] + elif len(blob) >= 0x60 + declared_payload_size: + payload_offset = 0x60 + layout = "rgba8_single_0x60" + notes = ["variant_b=19", "ARGB payload at 0x60"] + else: + return None, "truncated_rgba_single" + return ( + ExportPlan( + layout=layout, + format_name="rgba8", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=notes, + ), + None, + ) + + if variant_b == 0 and len(blob) >= 0x1000 + declared_payload_size: + bc1_mip0_size = bc_mip0_storage_size(width, height, 8) + bc3_mip0_size = bc_mip0_storage_size(width, height, 16) + if variant_a == 1 and declared_payload_size == bc1_mip0_size * 6: + return ( + ExportPlan( + layout="bc1_swap16_cube6_0x1000", + format_name="bc1_swap16_cube6", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=[bc1_mip0_size] * 6, + notes=["six BC1 faces", "16-bit endian-swapped blocks", "preview exports contact sheet"], + ), + None, + ) + if variant_a == 1 and declared_payload_size == bc1_mip0_size: + return ( + ExportPlan( + layout="bc1_swap16_single_0x1000", + format_name="bc1_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["BC1 texture", "16-bit endian-swapped blocks"], + ), + None, + ) + if variant_a == 1 and declared_payload_size == bc3_mip0_size: + return ( + ExportPlan( + layout="bc3_swap16_single_0x1000", + format_name="bc3_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["BC3 texture", "16-bit endian-swapped blocks"], + ), + None, + ) + if ( + variant_a > 1 + and len(explicit_mip_sizes) >= 2 + and explicit_mip_sizes[0] == bc1_mip0_size + and explicit_mip_sizes[0] + explicit_mip_sizes[1] == declared_payload_size + ): + return ( + ExportPlan( + layout="bc1_swap16_mipped_0x1000", + format_name="bc1_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=variant_a, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=explicit_mip_sizes, + notes=["packed BC1 mip chain", "16-bit endian-swapped blocks", "preview exports mip 0"], + ), + None, + ) + + if variant_b == 0: + bc1_mip0_size = bc_mip0_storage_size(width, height, 8) + bc3_mip0_size = bc_mip0_storage_size(width, height, 16) + payload_offset = None + if len(blob) >= 0x1000 + declared_payload_size: + payload_offset = 0x1000 + elif len(blob) >= 0x70 + declared_payload_size: + payload_offset = 0x70 + elif len(blob) >= 0x60 + declared_payload_size: + payload_offset = 0x60 + if payload_offset is not None: + if variant_a == 1 and declared_payload_size == bc1_mip0_size * 6: + return ( + ExportPlan( + layout=f"bc1_swap16_cube6_{payload_offset:#x}", + format_name="bc1_swap16_cube6", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=[bc1_mip0_size] * 6, + notes=["six BC1 faces", "16-bit endian-swapped blocks", "preview exports contact sheet"], + ), + None, + ) + if variant_a == 1 and declared_payload_size == bc1_mip0_size: + return ( + ExportPlan( + layout=f"bc1_swap16_single_{payload_offset:#x}", + format_name="bc1_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["BC1 texture", "16-bit endian-swapped blocks"], + ), + None, + ) + if variant_a == 1 and declared_payload_size == bc3_mip0_size: + return ( + ExportPlan( + layout=f"bc3_swap16_single_{payload_offset:#x}", + format_name="bc3_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["BC3 texture", "16-bit endian-swapped blocks"], + ), + None, + ) + if ( + variant_a > 1 + and len(explicit_mip_sizes) >= 2 + and explicit_mip_sizes[0] == bc1_mip0_size + and explicit_mip_sizes[0] + explicit_mip_sizes[1] == declared_payload_size + ): + return ( + ExportPlan( + layout=f"bc1_swap16_mipped_{payload_offset:#x}", + format_name="bc1_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=variant_a, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=explicit_mip_sizes, + notes=["packed BC1 mip chain", "16-bit endian-swapped blocks", "preview exports mip 0"], + ), + None, + ) + + if variant_a == 1 and variant_b == 1 and len(blob) >= 0x1000 + declared_payload_size: + bc3_mip0_size = bc_mip0_storage_size(width, height, 16) + if declared_payload_size == bc3_mip0_size: + return ( + ExportPlan( + layout="bc3_swap16_single_0x1000_b1", + format_name="bc3_swap16", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["variant_b=1", "BC3 texture", "16-bit endian-swapped blocks"], + ), + None, + ) + + if variant_a == 1 and variant_b == 20 and declared_payload_size == width * height * 4: + payload_offset = 0x60 + if len(blob) >= payload_offset + declared_payload_size: + payload = blob[payload_offset : payload_offset + declared_payload_size] + if looks_like_gray_argb(payload): + return ( + ExportPlan( + layout="gray_argb_linear_0x60", + format_name="gray_argb_linear", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=payload_offset, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["variant_b=20", "linear grayscale stored as opaque ARGB"], + ), + None, + ) + + if variant_a == 1 and variant_b == 2: + storage_width = align_up(width, 128) + storage_height = align_up(height, 128) + if declared_payload_size == storage_width * storage_height and len(blob) >= 0x1000 + declared_payload_size: + return ( + ExportPlan( + layout="r8_single_aligned_0x1000", + format_name="r8", + visible_width=width, + visible_height=height, + storage_width=storage_width, + storage_height=storage_height, + mip_count=1, + payload_offset=0x1000, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["128-aligned backing rectangle"], + ), + None, + ) + + if variant_a == 1 and variant_b == 1 and declared_payload_size == width * height: + if len(blob) >= 0x9000 + declared_payload_size: + return ( + ExportPlan( + layout="r8_single_0x9000", + format_name="r8", + visible_width=width, + visible_height=height, + storage_width=width, + storage_height=height, + mip_count=1, + payload_offset=0x9000, + payload_size=declared_payload_size, + mip_sizes=[declared_payload_size], + notes=["0x9000 payload offset"], + ), + None, + ) + + return None, f"unsupported_variant_a{variant_a}_b{variant_b}" + + +def build_legacy_bgra_dds(width: int, height: int, mip_payloads_bgra: list[bytes]) -> bytes: + header_flags = DDS_HEADER_FLAGS_TEXTURE | DDS_HEADER_FLAGS_PITCH + caps = DDS_CAPS_TEXTURE + if len(mip_payloads_bgra) > 1: + header_flags |= DDS_HEADER_FLAGS_MIPMAP + caps |= DDS_CAPS_COMPLEX | DDS_CAPS_MIPMAP + + pitch = width * 4 + header_values = [ + 124, + header_flags, + height, + width, + pitch, + 0, + max(len(mip_payloads_bgra), 1), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 32, + DDS_PF_RGB | DDS_PF_ALPHAPIXELS, + 0, + 32, + 0x00FF0000, + 0x0000FF00, + 0x000000FF, + 0xFF000000, + caps, + 0, + 0, + 0, + 0, + ] + assert len(header_values) == 31 + header = struct.pack("<31I", *header_values) + return struct.pack(" bytes: + bgra = bytearray(len(rgba)) + for i in range(0, len(rgba), 4): + r = rgba[i + 0] + g = rgba[i + 1] + b = rgba[i + 2] + a = rgba[i + 3] + bgra[i + 0] = b + bgra[i + 1] = g + bgra[i + 2] = r + bgra[i + 3] = a + return bytes(bgra) + + +def argb_to_rgba(argb: bytes) -> bytes: + rgba = bytearray(len(argb)) + for i in range(0, len(argb), 4): + a = argb[i + 0] + r = argb[i + 1] + g = argb[i + 2] + b = argb[i + 3] + rgba[i + 0] = r + rgba[i + 1] = g + rgba[i + 2] = b + rgba[i + 3] = a + return bytes(rgba) + + +def gray_to_rgba(gray: bytes) -> bytes: + rgba = bytearray(len(gray) * 4) + out = 0 + for value in gray: + rgba[out + 0] = value + rgba[out + 1] = value + rgba[out + 2] = value + rgba[out + 3] = 255 + out += 4 + return bytes(rgba) + + +def swap16(data: bytes) -> bytes: + out = bytearray(len(data)) + for i in range(0, len(data), 2): + out[i : i + 2] = data[i : i + 2][::-1] + return bytes(out) + + +def rgb565(color: int) -> tuple[int, int, int]: + return ( + ((color >> 11) & 31) * 255 // 31, + ((color >> 5) & 63) * 255 // 63, + (color & 31) * 255 // 31, + ) + + +def decode_bc4_block(block: bytes) -> list[int]: + alpha0 = block[0] + alpha1 = block[1] + bits = int.from_bytes(block[2:8], "little") + values = [alpha0, alpha1] + if alpha0 > alpha1: + values += [ + (6 * alpha0 + 1 * alpha1) // 7, + (5 * alpha0 + 2 * alpha1) // 7, + (4 * alpha0 + 3 * alpha1) // 7, + (3 * alpha0 + 4 * alpha1) // 7, + (2 * alpha0 + 5 * alpha1) // 7, + (1 * alpha0 + 6 * alpha1) // 7, + ] + else: + values += [ + (4 * alpha0 + 1 * alpha1) // 5, + (3 * alpha0 + 2 * alpha1) // 5, + (2 * alpha0 + 3 * alpha1) // 5, + (1 * alpha0 + 4 * alpha1) // 5, + 0, + 255, + ] + return [values[(bits >> (3 * i)) & 7] for i in range(16)] + + +def decode_bc3_to_rgba(data: bytes, width: int, height: int) -> bytes: + blocks_w = (width + 3) // 4 + blocks_h = (height + 3) // 4 + out = bytearray(width * height * 4) + cursor = 0 + for block_y in range(blocks_h): + for block_x in range(blocks_w): + alpha = decode_bc4_block(data[cursor : cursor + 8]) + color0 = int.from_bytes(data[cursor + 8 : cursor + 10], "little") + color1 = int.from_bytes(data[cursor + 10 : cursor + 12], "little") + color_bits = int.from_bytes(data[cursor + 12 : cursor + 16], "little") + cursor += 16 + + r0, g0, b0 = rgb565(color0) + r1, g1, b1 = rgb565(color1) + colors = [(r0, g0, b0), (r1, g1, b1)] + if color0 > color1: + colors += [ + ((2 * r0 + r1) // 3, (2 * g0 + g1) // 3, (2 * b0 + b1) // 3), + ((r0 + 2 * r1) // 3, (g0 + 2 * g1) // 3, (b0 + 2 * b1) // 3), + ] + else: + colors += [ + ((r0 + r1) // 2, (g0 + g1) // 2, (b0 + b1) // 2), + (0, 0, 0), + ] + + for py in range(4): + for px in range(4): + x = block_x * 4 + px + y = block_y * 4 + py + if x >= width or y >= height: + continue + color_index = (color_bits >> (2 * (py * 4 + px))) & 3 + r, g, b = colors[color_index] + a = alpha[py * 4 + px] + out_index = (y * width + x) * 4 + out[out_index : out_index + 4] = bytes((r, g, b, a)) + return bytes(out) + + +def decode_bc1_to_rgba(data: bytes, width: int, height: int) -> bytes: + blocks_w = (width + 3) // 4 + blocks_h = (height + 3) // 4 + out = bytearray(width * height * 4) + cursor = 0 + for block_y in range(blocks_h): + for block_x in range(blocks_w): + color0 = int.from_bytes(data[cursor : cursor + 2], "little") + color1 = int.from_bytes(data[cursor + 2 : cursor + 4], "little") + color_bits = int.from_bytes(data[cursor + 4 : cursor + 8], "little") + cursor += 8 + + r0, g0, b0 = rgb565(color0) + r1, g1, b1 = rgb565(color1) + colors = [(r0, g0, b0, 255), (r1, g1, b1, 255)] + if color0 > color1: + colors += [ + ((2 * r0 + r1) // 3, (2 * g0 + g1) // 3, (2 * b0 + b1) // 3, 255), + ((r0 + 2 * r1) // 3, (g0 + 2 * g1) // 3, (b0 + 2 * b1) // 3, 255), + ] + else: + colors += [ + ((r0 + r1) // 2, (g0 + g1) // 2, (b0 + b1) // 2, 255), + (0, 0, 0, 0), + ] + + for py in range(4): + for px in range(4): + x = block_x * 4 + px + y = block_y * 4 + py + if x >= width or y >= height: + continue + color = colors[(color_bits >> (2 * (py * 4 + px))) & 3] + out_index = (y * width + x) * 4 + out[out_index : out_index + 4] = bytes(color) + return bytes(out) + + +def tiled_row(y: int, width: int, log2_bpp: int) -> int: + macro = ((y // 32) * (width // 32)) << (log2_bpp + 7) + micro = ((y & 6) << 2) << log2_bpp + return macro + ((micro & ~0xF) << 1) + (micro & 0xF) + ((y & 8) << (3 + log2_bpp)) + ( + (y & 1) << 4 + ) + + +def tiled_col(x: int, y: int, log2_bpp: int, base_offset: int) -> int: + macro = (x // 32) << (log2_bpp + 7) + micro = (x & 7) << log2_bpp + offset = base_offset + (macro + ((micro & ~0xF) << 1) + (micro & 0xF)) + return ( + ((offset & ~0x1FF) << 3) + + ((offset & 0x1C0) << 2) + + (offset & 0x3F) + + ((y & 16) << 7) + + (((((y & 8) >> 2) + (x >> 3)) & 3) << 6) + ) + + +def untile_blocks(src: bytes, width: int, height: int, pitch: int, bytes_per_block: int) -> bytes: + log2_bpp = (bytes_per_block // 4) + ((bytes_per_block // 2) >> (bytes_per_block // 4)) + out = bytearray(width * height * bytes_per_block) + for y in range(height): + base = tiled_row(y, pitch, log2_bpp) + row_off = y * width * bytes_per_block + for x in range(width): + off = tiled_col(x, y, log2_bpp, base) >> log2_bpp + src_off = off * bytes_per_block + dst_off = row_off + x * bytes_per_block + out[dst_off:dst_off + bytes_per_block] = src[src_off:src_off + bytes_per_block] + return bytes(out) + + +def write_tga_rgba(path: Path, width: int, height: int, rgba: bytes) -> None: + header = struct.pack( + " None: + header = struct.pack( + " None: + Image.frombytes("RGBA", (width, height), rgba).save(path) + + +def write_png_gray(path: Path, width: int, height: int, gray: bytes) -> None: + Image.frombytes("L", (width, height), gray).save(path) + + +def write_png_r8_alpha(path: Path, width: int, height: int, gray: bytes) -> None: + alpha = Image.frombytes("L", (width, height), gray) + rgba = Image.new("RGBA", (width, height), (255, 255, 255, 0)) + rgba.putalpha(alpha) + rgba.save(path) + + +def write_png_r8_preview(path: Path, width: int, height: int, gray: bytes) -> None: + # These one-channel atlases are often glyph / mask data. A lightly smoothed + # composited preview is easier to inspect than raw dithered luma. + alpha = Image.frombytes("L", (width, height), gray).filter(ImageFilter.BoxBlur(0.5)) + preview = Image.new("RGBA", (width, height), (20, 20, 24, 255)) + fg = Image.new("RGBA", (width, height), (245, 245, 245, 255)) + fg.putalpha(alpha) + preview.alpha_composite(fg) + preview.save(path) + + +def extract_r8_visible(payload: bytes, storage_width: int, visible_width: int, visible_height: int) -> bytes: + rows = [] + for row in range(visible_height): + start = row * storage_width + rows.append(payload[start:start + visible_width]) + return b"".join(rows) + + +def export_ntxr(input_path: Path, output_root: Path, source_root: Path) -> dict: + blob = input_path.read_bytes() + plan, reason = classify_ntxr(blob) + output_base = output_root / input_path.relative_to(source_root) + entry = { + "source": str(input_path.relative_to(source_root)).replace("\\", "/"), + "size": len(blob), + "header": { + "variant_a": be16(blob, 0x20) if len(blob) >= 0x22 else None, + "variant_b": be16(blob, 0x22) if len(blob) >= 0x24 else None, + "width": be16(blob, 0x24) if len(blob) >= 0x26 else None, + "height": be16(blob, 0x26) if len(blob) >= 0x28 else None, + "declared_payload_size": be32(blob, 0x18) if len(blob) >= 0x1C else None, + "tag_0x40": safe_ascii(blob[0x40:0x44]) if len(blob) >= 0x44 else None, + "tag_0x50": safe_ascii(blob[0x50:0x54]) if len(blob) >= 0x54 else None, + }, + } + if plan is None: + stale_paths = ( + output_base.with_suffix(".dds"), + output_base.with_suffix(".tga"), + output_base.with_suffix(".png"), + output_base.with_suffix(".json"), + output_base.with_name(output_base.stem + ".raw.png"), + output_base.with_name(output_base.stem + ".alpha.png"), + ) + for stale_path in stale_paths: + if stale_path.exists(): + stale_path.unlink() + for stale_face in output_base.parent.glob(output_base.stem + ".face*.png"): + stale_face.unlink() + entry["status"] = "skipped" + entry["reason"] = reason + return entry + + payload = blob[plan.payload_offset:plan.payload_offset + plan.payload_size] + output_base.parent.mkdir(parents=True, exist_ok=True) + + preview_path = output_base.with_suffix(".tga") + png_path = output_base.with_suffix(".png") + dds_path = output_base.with_suffix(".dds") + json_path = output_base.with_suffix(".json") + raw_png_path = output_base.with_name(output_base.stem + ".raw.png") + alpha_png_path = output_base.with_name(output_base.stem + ".alpha.png") + + dds_rgba_payloads: list[bytes] = [] + preview_rgba: bytes | None = None + preview_gray: bytes | None = None + preview_width = plan.visible_width + preview_height = plan.visible_height + face_pngs: list[str] = [] + + if plan.format_name == "rgba8": + cursor = 0 + for mip_index, mip_size in enumerate(plan.mip_sizes): + mip_blob = payload[cursor:cursor + mip_size] + if len(mip_blob) != mip_size: + entry["status"] = "skipped" + entry["reason"] = "truncated_mip_payload" + return entry + untiled_mip_blob = untile_blocks( + mip_blob, + max(plan.visible_width >> mip_index, 1), + max(plan.visible_height >> mip_index, 1), + max(plan.storage_width >> mip_index, 1), + 4, + ) + rgba_mip = argb_to_rgba(untiled_mip_blob) + dds_rgba_payloads.append(rgba_mip) + if mip_index == 0: + preview_rgba = rgba_mip[: plan.visible_width * plan.visible_height * 4] + cursor += mip_size + assert preview_rgba is not None + write_tga_rgba(preview_path, plan.visible_width, plan.visible_height, preview_rgba) + write_png_rgba(png_path, plan.visible_width, plan.visible_height, preview_rgba) + elif plan.format_name == "r8": + cursor = 0 + mip_width = plan.visible_width + mip_height = plan.visible_height + for mip_index, mip_size in enumerate(plan.mip_sizes): + mip_blob = payload[cursor:cursor + mip_size] + if len(mip_blob) != mip_size: + entry["status"] = "skipped" + entry["reason"] = "truncated_mip_payload" + return entry + storage_width = max(plan.storage_width >> mip_index, 1) + untiled_gray = untile_blocks(mip_blob, mip_width, mip_height, storage_width, 1) + expected_size = mip_width * mip_height + visible_gray = untiled_gray[:expected_size] + dds_rgba_payloads.append(gray_to_rgba(visible_gray)) + if mip_index == 0: + preview_gray = visible_gray + cursor += mip_size + mip_width = max(mip_width >> 1, 1) + mip_height = max(mip_height >> 1, 1) + assert preview_gray is not None + write_tga_gray(preview_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_r8_preview(png_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_gray(raw_png_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_r8_alpha(alpha_png_path, plan.visible_width, plan.visible_height, preview_gray) + elif plan.format_name == "bc3_swap16": + width_blocks = (plan.visible_width + 3) // 4 + height_blocks = (plan.visible_height + 3) // 4 + pitch_blocks = align_up(width_blocks, 32) + mip0_size = plan.mip_sizes[0] + mip0_blob = payload[:mip0_size] + untiled = untile_blocks(mip0_blob, width_blocks, height_blocks, pitch_blocks, 16) + preview_rgba = decode_bc3_to_rgba(swap16(untiled), plan.visible_width, plan.visible_height) + dds_rgba_payloads.append(preview_rgba) + write_tga_rgba(preview_path, plan.visible_width, plan.visible_height, preview_rgba) + write_png_rgba(png_path, plan.visible_width, plan.visible_height, preview_rgba) + elif plan.format_name == "bc1_swap16": + width_blocks = (plan.visible_width + 3) // 4 + height_blocks = (plan.visible_height + 3) // 4 + pitch_blocks = align_up(width_blocks, 32) + mip0_size = plan.mip_sizes[0] + mip0_blob = payload[:mip0_size] + untiled = untile_blocks(mip0_blob, width_blocks, height_blocks, pitch_blocks, 8) + preview_rgba = decode_bc1_to_rgba(swap16(untiled), plan.visible_width, plan.visible_height) + dds_rgba_payloads.append(preview_rgba) + write_tga_rgba(preview_path, plan.visible_width, plan.visible_height, preview_rgba) + write_png_rgba(png_path, plan.visible_width, plan.visible_height, preview_rgba) + elif plan.format_name == "bc1_swap16_cube6": + width_blocks = (plan.visible_width + 3) // 4 + height_blocks = (plan.visible_height + 3) // 4 + pitch_blocks = align_up(width_blocks, 32) + face_size = plan.mip_sizes[0] + faces: list[bytes] = [] + for face_index in range(6): + face_blob = payload[face_index * face_size : (face_index + 1) * face_size] + untiled = untile_blocks(face_blob, width_blocks, height_blocks, pitch_blocks, 8) + face_rgba = decode_bc1_to_rgba(swap16(untiled), plan.visible_width, plan.visible_height) + faces.append(face_rgba) + face_path = output_base.with_name(output_base.stem + f".face{face_index}.png") + write_png_rgba(face_path, plan.visible_width, plan.visible_height, face_rgba) + face_pngs.append(str(face_path.relative_to(output_root)).replace("\\", "/")) + dds_rgba_payloads.append(faces[0]) + preview_width = plan.visible_width * 3 + preview_height = plan.visible_height * 2 + preview_image = Image.new("RGBA", (preview_width, preview_height), (0, 0, 0, 255)) + for face_index, face_rgba in enumerate(faces): + x = (face_index % 3) * plan.visible_width + y = (face_index // 3) * plan.visible_height + preview_image.paste(Image.frombytes("RGBA", (plan.visible_width, plan.visible_height), face_rgba), (x, y)) + preview_rgba = preview_image.tobytes() + write_tga_rgba(preview_path, preview_width, preview_height, preview_rgba) + write_png_rgba(png_path, preview_width, preview_height, preview_rgba) + elif plan.format_name == "gray_argb_linear": + gray = payload[1::4] + preview_gray = gray + dds_rgba_payloads.append(gray_to_rgba(gray)) + write_tga_gray(preview_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_r8_preview(png_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_gray(raw_png_path, plan.visible_width, plan.visible_height, preview_gray) + write_png_r8_alpha(alpha_png_path, plan.visible_width, plan.visible_height, preview_gray) + else: + entry["status"] = "skipped" + entry["reason"] = f"unhandled_format_{plan.format_name}" + return entry + + dds_bytes = build_legacy_bgra_dds( + width=plan.visible_width, + height=plan.visible_height, + mip_payloads_bgra=[rgba_to_bgra(mip) for mip in dds_rgba_payloads], + ) + dds_path.write_bytes(dds_bytes) + json_path.write_text( + json.dumps( + { + "source": entry["source"], + "layout": plan.layout, + "format": plan.format_name, + "visible_width": plan.visible_width, + "visible_height": plan.visible_height, + "storage_width": plan.storage_width, + "storage_height": plan.storage_height, + "mip_count": plan.mip_count, + "payload_offset": plan.payload_offset, + "payload_size": plan.payload_size, + "dds_encoding": "legacy_a8r8g8b8_view", + "notes": plan.notes, + "preview_png": png_path.name, + "raw_png": raw_png_path.name if plan.format_name in ("r8", "gray_argb_linear") else None, + "alpha_png": alpha_png_path.name if plan.format_name in ("r8", "gray_argb_linear") else None, + "preview_width": preview_width, + "preview_height": preview_height, + "face_pngs": face_pngs if face_pngs else None, + }, + indent=2, + ), + encoding="utf-8", + ) + + entry["status"] = "exported" + entry["layout"] = plan.layout + entry["format"] = plan.format_name + entry["dds"] = str(dds_path.relative_to(output_root)).replace("\\", "/") + entry["preview"] = str(preview_path.relative_to(output_root)).replace("\\", "/") + entry["preview_png"] = str(png_path.relative_to(output_root)).replace("\\", "/") + if plan.format_name in ("r8", "gray_argb_linear"): + entry["raw_preview_png"] = str(raw_png_path.relative_to(output_root)).replace("\\", "/") + entry["alpha_preview_png"] = str(alpha_png_path.relative_to(output_root)).replace("\\", "/") + if face_pngs: + entry["face_pngs"] = face_pngs + entry["metadata"] = str(json_path.relative_to(output_root)).replace("\\", "/") + entry["visible_width"] = plan.visible_width + entry["visible_height"] = plan.visible_height + entry["storage_width"] = plan.storage_width + entry["storage_height"] = plan.storage_height + entry["mip_count"] = plan.mip_count + entry["notes"] = plan.notes + return entry + + +def main() -> int: + parser = argparse.ArgumentParser(description="Export known AC6 NTXR textures to DDS and TGA.") + parser.add_argument( + "--input", + type=Path, + default=Path("out") / "ac6_runtime_fhm_typed", + help="Root directory containing extracted .ntxr files", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("out") / "ac6_runtime_ntxr_exported", + help="Output directory for exported textures", + ) + args = parser.parse_args() + + source_root = args.input.resolve() + output_root = args.output.resolve() + output_root.mkdir(parents=True, exist_ok=True) + + exported = [] + skipped = [] + for input_path in sorted(source_root.rglob("*.ntxr")): + result = export_ntxr(input_path, output_root, source_root) + if result["status"] == "exported": + exported.append(result) + else: + skipped.append(result) + + manifest = { + "input": str(source_root), + "output": str(output_root), + "exported_count": len(exported), + "skipped_count": len(skipped), + "exported": exported, + "skipped": skipped, + } + (output_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + print( + json.dumps( + { + "exported_count": len(exported), + "skipped_count": len(skipped), + "output": str(output_root), + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/extract_ac6_pac.py b/tools/extract_ac6_pac.py new file mode 100644 index 00000000..3ecd69d5 --- /dev/null +++ b/tools/extract_ac6_pac.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import struct +from pathlib import Path + + +HEADER_SIZE = 8 +ENTRY_SIZE = 16 + + +def parse_tbl(path: Path) -> list[dict]: + data = path.read_bytes() + if len(data) < HEADER_SIZE: + raise ValueError("DATA.TBL is too small") + + entry_count, pack_count = struct.unpack_from(">II", data, 0) + expected_size = HEADER_SIZE + (entry_count * ENTRY_SIZE) + if len(data) != expected_size: + raise ValueError(f"unexpected DATA.TBL size: got {len(data)}, expected {expected_size}") + + entries = [] + for index in range(entry_count): + group, offset, compressed_size, decompressed_size = struct.unpack_from( + ">4I", data, HEADER_SIZE + (index * ENTRY_SIZE) + ) + pac_name = "DATA01.PAC" if (group & 0x01000000) else "DATA00.PAC" + storage_kind = "raw" if (group & 0x00020000) else "compressed" + entries.append( + { + "index": index, + "group": group, + "group_hex": f"0x{group:08x}", + "pac_name": pac_name, + "storage_kind": storage_kind, + "offset": offset, + "compressed_size": compressed_size, + "decompressed_size": decompressed_size, + } + ) + return entries + + +def sha256_path(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], include_compressed: bool) -> dict: + pac_bytes = { + "DATA00.PAC": asset_root.joinpath("DATA00.PAC").read_bytes(), + "DATA01.PAC": asset_root.joinpath("DATA01.PAC").read_bytes(), + } + pac_sizes = {name: len(data) for name, data in pac_bytes.items()} + + output_root.mkdir(parents=True, exist_ok=True) + files_dir = output_root / "files" + files_dir.mkdir(exist_ok=True) + + manifest_entries = [] + extracted_count = 0 + skipped_count = 0 + + for entry in entries: + if entry["storage_kind"] == "compressed" and not include_compressed: + skipped_count += 1 + manifest_entries.append({**entry, "extracted": False, "reason": "compressed entry skipped"}) + continue + + pac_name = entry["pac_name"] + pac_size = pac_sizes[pac_name] + start = entry["offset"] + end = start + entry["compressed_size"] + if end > pac_size: + raise ValueError( + f"entry {entry['index']} exceeds {pac_name}: offset=0x{start:x}, size=0x{entry['compressed_size']:x}" + ) + + blob = pac_bytes[pac_name][start:end] + subdir = files_dir / pac_name.replace(".PAC", "") / entry["storage_kind"] + subdir.mkdir(parents=True, exist_ok=True) + out_path = subdir / f"{entry['index']:04d}.bin" + out_path.write_bytes(blob) + + manifest_entries.append( + { + **entry, + "extracted": True, + "path": str(out_path.relative_to(output_root)).replace("\\", "/"), + "sha256": hashlib.sha256(blob).hexdigest(), + "head_hex": blob[:32].hex(), + } + ) + extracted_count += 1 + + manifest = { + "asset_root": str(asset_root), + "output_root": str(output_root), + "entry_count": len(entries), + "extracted_count": extracted_count, + "skipped_count": skipped_count, + "include_compressed": include_compressed, + "archives": { + name: { + "size": size, + "sha256": sha256_path(asset_root / name), + } + for name, size in pac_sizes.items() + }, + "entries": manifest_entries, + } + + (output_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description="Extract indexed records from Ace Combat 6 DATA00/01.PAC using DATA.TBL.") + parser.add_argument("asset_root", type=Path, help="Directory containing DATA.TBL, DATA00.PAC, and DATA01.PAC") + parser.add_argument( + "--output", + type=Path, + default=Path("out") / "ac6_pac_extracted_raw", + help="Output directory for the manifest and extracted records", + ) + parser.add_argument( + "--raw-only", + action="store_true", + help="Extract only entries marked raw in DATA.TBL and skip compressed entries", + ) + args = parser.parse_args() + + asset_root = args.asset_root.resolve() + output_root = args.output.resolve() + entries = parse_tbl(asset_root / "DATA.TBL") + manifest = extract_entries(asset_root, output_root, entries, include_compressed=not args.raw_only) + + print( + json.dumps( + { + "entry_count": manifest["entry_count"], + "extracted_count": manifest["extracted_count"], + "skipped_count": manifest["skipped_count"], + "output_root": manifest["output_root"], + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/extract_ac6_runtime_fhm.py b/tools/extract_ac6_runtime_fhm.py new file mode 100644 index 00000000..f76d8645 --- /dev/null +++ b/tools/extract_ac6_runtime_fhm.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import struct +from collections import defaultdict +from pathlib import Path + + +DUMP_RE = re.compile( + r"^entry_(?P\d+)_mode(?P\d+)_c(?P\d+)_u(?P\d+)" + r"(?:_off(?P[0-9a-fA-F]+))?\.bin$" +) + + +def load_manifest_entries(path: Path) -> dict[tuple[int, int], list[dict]]: + manifest = json.loads(path.read_text(encoding="utf-8")) + by_pair: dict[tuple[int, int], list[dict]] = defaultdict(list) + for entry in manifest["entries"]: + if entry["storage_kind"] != "compressed": + continue + by_pair[(entry["compressed_size"], entry["decompressed_size"])].append(entry) + return by_pair + + +def parse_fhm(blob: bytes) -> list[dict]: + if len(blob) < 0x1C or blob[:4] != b"FHM ": + return [] + + count = struct.unpack_from(">I", blob, 0x10)[0] + if count == 0: + return [] + + table_base = 0x14 + offsets_base = table_base + sizes_base = offsets_base + (count * 4) + if sizes_base + (count * 4) > len(blob): + return [] + + offsets = [struct.unpack_from(">I", blob, offsets_base + (i * 4))[0] for i in range(count)] + sizes = [struct.unpack_from(">I", blob, sizes_base + (i * 4))[0] for i in range(count)] + + entries = [] + for index, (offset, size) in enumerate(zip(offsets, sizes)): + if offset >= len(blob): + continue + + end = offset + size + if end > len(blob): + next_offset = offsets[index + 1] if index + 1 < len(offsets) else len(blob) + end = min(next_offset, len(blob)) + if end <= offset: + continue + + child = blob[offset:end] + entries.append( + { + "index": index, + "offset": offset, + "size": len(child), + "magic": child[:4].decode("ascii", errors="replace"), + "data": child, + } + ) + return entries + + +def safe_name(name: str) -> str: + return "".join(ch if ch.isalnum() or ch in ("-", "_", ".") else "_" for ch in name) + + +def magic_extension(magic: str) -> str: + normalized = magic.strip().upper() + mapping = { + "FHM": ".fhm", + "NTXR": ".ntxr", + "NSXR": ".nsxr", + "MDLP": ".mdlp", + "PLAD": ".plad", + "BFX": ".bfx", + "BSN": ".bsn", + "ACE6": ".ace6", + "NFH": ".nfh", + } + return mapping.get(normalized, ".bin") + + +def extract_container(blob: bytes, container_dir: Path, output_root: Path, depth: int, + max_depth: int) -> list[dict]: + children = parse_fhm(blob) + if not children: + return [] + + child_entries = [] + for child in children: + safe_magic = safe_name(child["magic"]) + child_name = f"{child['index']:03d}_{safe_magic}{magic_extension(child['magic'])}" + child_path = container_dir / child_name + child_path.write_bytes(child["data"]) + + child_entry = { + "index": child["index"], + "offset": child["offset"], + "size": child["size"], + "magic": child["magic"], + "path": str(child_path.relative_to(output_root)).replace("\\", "/"), + } + + if depth < max_depth and child["data"][:4] == b"FHM ": + nested_dir = container_dir / f"{child['index']:03d}_{safe_magic}" + nested_dir.mkdir(parents=True, exist_ok=True) + nested_children = extract_container(child["data"], nested_dir, output_root, depth + 1, + max_depth) + if nested_children: + child_entry["nested"] = nested_children + + child_entries.append(child_entry) + + return child_entries + + +def main() -> int: + parser = argparse.ArgumentParser(description="Extract child payloads from runtime-dumped AC6 FHM containers.") + parser.add_argument( + "--dump-dir", + type=Path, + default=Path("out") / "ac6_pac_runtime_dump", + help="Directory containing runtime PAC decode dumps", + ) + parser.add_argument( + "--manifest", + type=Path, + default=Path("out") / "ac6_pac_extracted_raw" / "manifest.json", + help="Manifest produced by extract_ac6_pac.py", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("out") / "ac6_runtime_fhm_extracted", + help="Output directory for parsed FHM containers and child payloads", + ) + parser.add_argument( + "--max-depth", + type=int, + default=4, + help="Maximum nested FHM recursion depth", + ) + args = parser.parse_args() + + dump_dir = args.dump_dir.resolve() + manifest_path = args.manifest.resolve() + output_root = args.output.resolve() + output_root.mkdir(parents=True, exist_ok=True) + + by_pair = load_manifest_entries(manifest_path) + extracted = [] + selected_dumps: dict[tuple[int, int, int, int], Path] = {} + + for dump_path in sorted(dump_dir.glob("*.bin")): + match = DUMP_RE.match(dump_path.name) + if not match: + continue + + meta = match.groupdict() + key = ( + int(meta["record_id"]), + int(meta["mode"]), + int(meta["compressed_size"]), + int(meta["decompressed_size"]), + ) + current = selected_dumps.get(key) + if current is None: + selected_dumps[key] = dump_path + continue + + current_match = DUMP_RE.match(current.name) + assert current_match is not None + current_has_offset = current_match.groupdict()["source_offset"] is not None + new_has_offset = meta["source_offset"] is not None + if new_has_offset and not current_has_offset: + selected_dumps[key] = dump_path + + for dump_path in sorted(selected_dumps.values()): + match = DUMP_RE.match(dump_path.name) + assert match is not None + + meta = match.groupdict() + compressed_size = int(meta["compressed_size"]) + decompressed_size = int(meta["decompressed_size"]) + codec_mode = int(meta["mode"]) + record_id = int(meta["record_id"]) + source_offset = int(meta["source_offset"], 16) if meta["source_offset"] else None + candidates = by_pair.get((compressed_size, decompressed_size), []) + + base_label = ( + f"idx_{candidates[0]['index']:04d}" + if len(candidates) == 1 + else f"pair_c{compressed_size}_u{decompressed_size}" + ) + container_dir = output_root / safe_name(base_label) + container_dir.mkdir(parents=True, exist_ok=True) + + blob = dump_path.read_bytes() + children = parse_fhm(blob) + if not children: + raw_path = container_dir / dump_path.name + raw_path.write_bytes(blob) + extracted.append( + { + "dump": dump_path.name, + "record_id": record_id, + "codec_mode": codec_mode, + "compressed_size": compressed_size, + "decompressed_size": decompressed_size, + "source_offset": source_offset, + "candidate_indexes": [entry["index"] for entry in candidates], + "kind": "raw", + "path": str(raw_path.relative_to(output_root)).replace("\\", "/"), + } + ) + continue + + child_entries = extract_container(blob, container_dir, output_root, 0, args.max_depth) + + extracted.append( + { + "dump": dump_path.name, + "record_id": record_id, + "codec_mode": codec_mode, + "compressed_size": compressed_size, + "decompressed_size": decompressed_size, + "source_offset": source_offset, + "candidate_indexes": [entry["index"] for entry in candidates], + "kind": "fhm", + "child_count": len(child_entries), + "children": child_entries, + } + ) + + manifest = { + "dump_dir": str(dump_dir), + "manifest": str(manifest_path), + "output": str(output_root), + "containers": extracted, + } + (output_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + print( + json.dumps( + { + "containers": len(extracted), + "output": str(output_root), + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/launch_ac6_with_pac_dump.ps1 b/tools/launch_ac6_with_pac_dump.ps1 new file mode 100644 index 00000000..55dba555 --- /dev/null +++ b/tools/launch_ac6_with_pac_dump.ps1 @@ -0,0 +1,14 @@ +$ErrorActionPreference = 'Stop' + +$repoRoot = Split-Path -Parent $PSScriptRoot +$exePath = Join-Path $repoRoot 'out\build\win-amd64-relwithdebinfo\ac6recomp.exe' + +if (-not (Test-Path -LiteralPath $exePath)) { + throw "ac6recomp.exe not found at $exePath" +} + +$env:AC6_DUMP_PAC_DECODED = '1' +Write-Host "AC6_DUMP_PAC_DECODED=1" +Write-Host "Launching $exePath" + +Start-Process -FilePath $exePath -WorkingDirectory (Split-Path -Parent $exePath) diff --git a/tools/pac_probe_lzx.cpp b/tools/pac_probe_lzx.cpp new file mode 100644 index 00000000..ec37a359 --- /dev/null +++ b/tools/pac_probe_lzx.cpp @@ -0,0 +1,333 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "mspack/lzx.h" +#include "mspack/mspack.h" +} + +namespace { + +constexpr uint32_t kEntrySize = 16; +constexpr uint32_t kHeaderSize = 8; +constexpr int kInputBufferSize = 1 << 15; + +struct TblEntry { + uint32_t group; + uint32_t offset; + uint32_t compressed_size; + uint32_t decompressed_size; +}; + +struct ProbeInput { + std::vector data; + size_t pos = 0; +}; + +struct ProbeOutput { + std::vector data; +}; + +struct ProbeFile { + mspack_file base{}; + ProbeInput* input = nullptr; + ProbeOutput* output = nullptr; +}; + +struct ProbeResult { + int window_bits = 0; + int reset_interval = 0; + int status = MSPACK_ERR_ARGS; + std::vector output; +}; + +uint32_t ReadBE32(const uint8_t* bytes) { + return (uint32_t(bytes[0]) << 24) | (uint32_t(bytes[1]) << 16) | + (uint32_t(bytes[2]) << 8) | uint32_t(bytes[3]); +} + +std::string HexPrefix(const std::vector& data, size_t count) { + std::ostringstream out; + out << std::hex << std::setfill('0'); + const size_t limit = std::min(count, data.size()); + for (size_t i = 0; i < limit; ++i) { + out << std::setw(2) << unsigned(data[i]); + } + return out.str(); +} + +std::string AsciiPrefix(const std::vector& data, size_t count) { + std::string out; + const size_t limit = std::min(count, data.size()); + out.reserve(limit); + for (size_t i = 0; i < limit; ++i) { + const unsigned char c = data[i]; + out.push_back(std::isprint(c) ? char(c) : '.'); + } + return out; +} + +std::optional> ReadFile(const std::string& path) { + std::ifstream file(path, std::ios::binary); + if (!file) { + return std::nullopt; + } + file.seekg(0, std::ios::end); + const auto size = file.tellg(); + if (size < 0) { + return std::nullopt; + } + file.seekg(0, std::ios::beg); + std::vector data(static_cast(size)); + if (!data.empty()) { + file.read(reinterpret_cast(data.data()), static_cast(data.size())); + if (!file) { + return std::nullopt; + } + } + return data; +} + +std::optional> ParseTbl(const std::string& path) { + const auto bytes = ReadFile(path); + if (!bytes || bytes->size() < kHeaderSize) { + return std::nullopt; + } + + const uint32_t count = ReadBE32(bytes->data()); + const uint32_t pack_count = ReadBE32(bytes->data() + 4); + (void)pack_count; + if (bytes->size() != kHeaderSize + (size_t(count) * kEntrySize)) { + return std::nullopt; + } + + std::vector entries; + entries.reserve(count); + for (uint32_t i = 0; i < count; ++i) { + const uint8_t* p = bytes->data() + kHeaderSize + (i * kEntrySize); + entries.push_back(TblEntry{ + ReadBE32(p + 0), + ReadBE32(p + 4), + ReadBE32(p + 8), + ReadBE32(p + 12), + }); + } + return entries; +} + +int ProbeRead(mspack_file* file, void* buffer, int bytes) { + auto* handle = reinterpret_cast(file); + if (!handle || !handle->input || bytes < 0) { + return -1; + } + const size_t remaining = handle->input->data.size() - handle->input->pos; + const size_t to_read = std::min(remaining, static_cast(bytes)); + if (to_read > 0) { + std::memcpy(buffer, handle->input->data.data() + handle->input->pos, to_read); + handle->input->pos += to_read; + } + return static_cast(to_read); +} + +int ProbeWrite(mspack_file* file, void* buffer, int bytes) { + auto* handle = reinterpret_cast(file); + if (!handle || !handle->output || bytes < 0) { + return -1; + } + const auto* src = reinterpret_cast(buffer); + handle->output->data.insert(handle->output->data.end(), src, src + bytes); + return bytes; +} + +int ProbeSeek(mspack_file* file, off_t offset, int mode) { + auto* handle = reinterpret_cast(file); + if (!handle || !handle->input) { + return -1; + } + + size_t base = 0; + switch (mode) { + case MSPACK_SYS_SEEK_START: + base = 0; + break; + case MSPACK_SYS_SEEK_CUR: + base = handle->input->pos; + break; + case MSPACK_SYS_SEEK_END: + base = handle->input->data.size(); + break; + default: + return -1; + } + + if (offset < 0 && static_cast(-offset) > base) { + return -1; + } + + const size_t next = offset >= 0 ? base + static_cast(offset) + : base - static_cast(-offset); + if (next > handle->input->data.size()) { + return -1; + } + handle->input->pos = next; + return 0; +} + +off_t ProbeTell(mspack_file* file) { + auto* handle = reinterpret_cast(file); + if (!handle || !handle->input) { + return off_t(-1); + } + return static_cast(handle->input->pos); +} + +void ProbeMessage(mspack_file*, const char* format, ...) { + std::va_list args; + va_start(args, format); + std::vfprintf(stderr, format, args); + std::fputc('\n', stderr); + va_end(args); +} + +void* ProbeAlloc(mspack_system*, size_t bytes) { + return std::malloc(bytes); +} + +void ProbeFree(void* ptr) { + std::free(ptr); +} + +void ProbeCopy(void* src, void* dest, size_t bytes) { + std::memcpy(dest, src, bytes); +} + +ProbeResult TryLzx(const std::vector& compressed, uint32_t expected_size, + int window_bits, int reset_interval) { + ProbeInput input{compressed, 0}; + ProbeOutput output; + ProbeFile in_file{}; + ProbeFile out_file{}; + in_file.input = &input; + out_file.output = &output; + + mspack_system system{}; + system.open = nullptr; + system.close = nullptr; + system.read = &ProbeRead; + system.write = &ProbeWrite; + system.seek = &ProbeSeek; + system.tell = &ProbeTell; + system.message = &ProbeMessage; + system.alloc = &ProbeAlloc; + system.free = &ProbeFree; + system.copy = &ProbeCopy; + system.null_ptr = nullptr; + + ProbeResult result; + result.window_bits = window_bits; + result.reset_interval = reset_interval; + + lzxd_stream* lzx = lzxd_init(&system, &in_file.base, &out_file.base, window_bits, + reset_interval, kInputBufferSize, expected_size, 0); + if (!lzx) { + result.status = MSPACK_ERR_NOMEMORY; + return result; + } + + result.status = lzxd_decompress(lzx, expected_size); + result.output = std::move(output.data); + lzxd_free(lzx); + return result; +} + +std::string PacPathForGroup(const std::string& asset_root, uint32_t group) { + const bool is_data01 = (group & 0x01000000u) != 0; + return asset_root + "\\" + (is_data01 ? "DATA01.PAC" : "DATA00.PAC"); +} + +void Usage() { + std::cerr << "usage: pac_probe_lzx \n"; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { + Usage(); + return 1; + } + + const std::string asset_root = argv[1]; + const int entry_index = std::atoi(argv[2]); + if (entry_index < 0) { + std::cerr << "invalid entry index\n"; + return 1; + } + + const auto entries = ParseTbl(asset_root + "\\DATA.TBL"); + if (!entries) { + std::cerr << "failed to parse DATA.TBL\n"; + return 1; + } + if (static_cast(entry_index) >= entries->size()) { + std::cerr << "entry index out of range\n"; + return 1; + } + + const TblEntry& entry = (*entries)[entry_index]; + const auto pac_bytes = ReadFile(PacPathForGroup(asset_root, entry.group)); + if (!pac_bytes) { + std::cerr << "failed to read PAC file\n"; + return 1; + } + if (size_t(entry.offset) + size_t(entry.compressed_size) > pac_bytes->size()) { + std::cerr << "entry is out of bounds for PAC file\n"; + return 1; + } + + const std::vector compressed( + pac_bytes->begin() + entry.offset, + pac_bytes->begin() + entry.offset + entry.compressed_size); + + std::cout << "entry=" << entry_index << " group=0x" << std::hex << entry.group + << " offset=0x" << entry.offset << " csize=0x" << entry.compressed_size + << " usize=0x" << entry.decompressed_size << std::dec << "\n"; + std::cout << "compressed_head_hex=" << HexPrefix(compressed, 32) << "\n"; + + std::array reset_candidates{0, 1, 2, 4, 8, 16, 32}; + bool found = false; + for (int window_bits = 15; window_bits <= 21; ++window_bits) { + for (int reset_interval : reset_candidates) { + ProbeResult result = TryLzx(compressed, entry.decompressed_size, window_bits, reset_interval); + if (result.status == MSPACK_ERR_OK && result.output.size() == entry.decompressed_size) { + found = true; + std::cout << "OK window_bits=" << window_bits + << " reset_interval=" << reset_interval + << " out_head_hex=" << HexPrefix(result.output, 32) + << " out_head_ascii=" << AsciiPrefix(result.output, 32) << "\n"; + } else { + std::cout << "FAIL window_bits=" << window_bits + << " reset_interval=" << reset_interval + << " status=" << result.status + << " produced=" << result.output.size() << "\n"; + } + } + } + + return found ? 0 : 2; +} diff --git a/tools/parse_ac6_swg.py b/tools/parse_ac6_swg.py new file mode 100644 index 00000000..20f56e98 --- /dev/null +++ b/tools/parse_ac6_swg.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import struct +from pathlib import Path + + +def be32(blob: bytes, offset: int) -> int: + return struct.unpack_from(">I", blob, offset)[0] + + +def read_c_string(blob: bytes, offset: int) -> str: + end = blob.find(b"\0", offset) + if end < 0: + end = len(blob) + return blob[offset:end].decode("ascii", errors="replace") + + +def extract_ascii_strings(blob: bytes) -> list[dict[str, int | str]]: + results = [] + for match in re.finditer(rb"[ -~]{4,}", blob): + text = match.group().decode("ascii", errors="replace") + if text.count("?") > len(text) // 2: + continue + results.append({"offset": match.start(), "text": text}) + return results + + +def detect_texture_table(blob: bytes, ntxr_count: int) -> tuple[int, list[dict[str, int]]] | tuple[None, list]: + if ntxr_count <= 0: + return None, [] + + max_start = max(0, len(blob) - ntxr_count * 12) + for start in range(0, max_start + 1, 4): + entries = [] + ok = True + for index in range(ntxr_count): + off = start + index * 12 + tex_id = be32(blob, off + 0) + width, height = struct.unpack_from(">HH", blob, off + 4) + flags = be32(blob, off + 8) + if tex_id != index: + ok = False + break + if width <= 0 or height <= 0 or width > 8192 or height > 8192: + ok = False + break + entries.append( + { + "texture_id": tex_id, + "width": width, + "height": height, + "flags": flags, + } + ) + if ok: + return start, entries + return None, [] + + +def parse_one(swg_path: Path) -> dict: + blob = swg_path.read_bytes() + if blob[:4] != b"SWG\0": + raise ValueError(f"{swg_path} is not an SWG blob") + + sibling_ntxrs = sorted(swg_path.parent.glob("*_NTXR.ntxr")) + table_offset, texture_table = detect_texture_table(blob, len(sibling_ntxrs)) + if texture_table: + for entry in texture_table: + candidate = swg_path.parent / f"{entry['texture_id'] + 1:03d}_NTXR.ntxr" + entry["candidate_ntxr"] = candidate.name if candidate.exists() else None + + strings = extract_ascii_strings(blob) + return { + "source": str(swg_path), + "magic": blob[:4].decode("ascii", errors="replace"), + "widget_name": read_c_string(blob, 0x08) if len(blob) > 0x08 else "", + "size": len(blob), + "sibling_ntxr_count": len(sibling_ntxrs), + "texture_table_offset": table_offset, + "texture_table": texture_table, + "strings": strings, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Parse AC6 SWG UI metadata blobs.") + parser.add_argument( + "--input", + type=Path, + default=Path("out") / "ac6_runtime_fhm_typed", + help="SWG file or directory to scan", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("out") / "ac6_runtime_swg_parsed", + help="Output directory for parsed SWG json files", + ) + args = parser.parse_args() + + input_path = args.input.resolve() + output_root = args.output.resolve() + output_root.mkdir(parents=True, exist_ok=True) + + swg_files = [input_path] if input_path.is_file() else sorted(input_path.rglob("*_SWG_.bin")) + parsed = [] + for swg_path in swg_files: + result = parse_one(swg_path) + relative = swg_path.relative_to(input_path.parent if input_path.is_file() else input_path) + out_path = output_root / relative.with_suffix(".json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(result, indent=2), encoding="utf-8") + parsed.append( + { + "source": str(relative).replace("\\", "/"), + "output": str(out_path.relative_to(output_root)).replace("\\", "/"), + "widget_name": result["widget_name"], + "texture_table_offset": result["texture_table_offset"], + "texture_count": len(result["texture_table"]), + } + ) + + manifest = { + "input": str(input_path), + "output": str(output_root), + "parsed_count": len(parsed), + "files": parsed, + } + (output_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + print(json.dumps({"parsed_count": len(parsed), "output": str(output_root)}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_ac6_asset_pipeline.py b/tools/run_ac6_asset_pipeline.py new file mode 100644 index 00000000..e1556dc6 --- /dev/null +++ b/tools/run_ac6_asset_pipeline.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +THIS_DIR = Path(__file__).resolve().parent +DEFAULT_ASSET_ROOT = Path("out") / "build" / "win-amd64-relwithdebinfo" / "assets" +DEFAULT_RAW_OUT = Path("out") / "ac6_pac_extracted_raw" +DEFAULT_DUMP_DIR = Path("out") / "ac6_pac_runtime_dump" +DEFAULT_TYPED_OUT = Path("out") / "ac6_runtime_fhm_typed" +DEFAULT_SWG_OUT = Path("out") / "ac6_runtime_swg_parsed" +DEFAULT_NTXR_OUT = Path("out") / "ac6_runtime_ntxr_exported" + + +def run_step(args: list[str], cwd: Path) -> None: + cmd = [sys.executable, *args] + subprocess.run(cmd, cwd=cwd, check=True) + + +def read_json(path: Path) -> dict | None: + if not path.exists(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def count_fhm_containers(manifest: dict | None) -> int | None: + if not manifest: + return None + containers = manifest.get("containers") + if isinstance(containers, list): + return len(containers) + return None + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run the AC6 asset extraction pipeline over PAC archives and collected runtime decode dumps." + ) + parser.add_argument( + "--asset-root", + type=Path, + default=DEFAULT_ASSET_ROOT, + help="Directory containing DATA.TBL, DATA00.PAC, and DATA01.PAC", + ) + parser.add_argument( + "--raw-out", + type=Path, + default=DEFAULT_RAW_OUT, + help="Output directory for extract_ac6_pac.py", + ) + parser.add_argument( + "--dump-dir", + type=Path, + default=DEFAULT_DUMP_DIR, + help="Directory containing runtime PAC decode dumps", + ) + parser.add_argument( + "--typed-out", + type=Path, + default=DEFAULT_TYPED_OUT, + help="Output directory for extract_ac6_runtime_fhm.py", + ) + parser.add_argument( + "--swg-out", + type=Path, + default=DEFAULT_SWG_OUT, + help="Output directory for parse_ac6_swg.py", + ) + parser.add_argument( + "--ntxr-out", + type=Path, + default=DEFAULT_NTXR_OUT, + help="Output directory for export_ac6_ntxr.py", + ) + parser.add_argument( + "--raw-only", + action="store_true", + help="Pass --raw-only to extract_ac6_pac.py", + ) + parser.add_argument( + "--skip-pac-extract", + action="store_true", + help="Skip extract_ac6_pac.py even if the PAC archives are available", + ) + args = parser.parse_args() + + repo_root = THIS_DIR.parent + asset_root = args.asset_root.resolve() + raw_out = args.raw_out.resolve() + dump_dir = args.dump_dir.resolve() + typed_out = args.typed_out.resolve() + swg_out = args.swg_out.resolve() + ntxr_out = args.ntxr_out.resolve() + + raw_manifest = raw_out / "manifest.json" + + if not args.skip_pac_extract: + if asset_root.exists(): + pac_args = [str(THIS_DIR / "extract_ac6_pac.py"), str(asset_root), "--output", str(raw_out)] + if args.raw_only: + pac_args.append("--raw-only") + run_step(pac_args, repo_root) + elif not raw_manifest.exists(): + raise SystemExit(f"asset root does not exist and no prior raw manifest is available: {asset_root}") + + if not dump_dir.exists(): + raise SystemExit(f"runtime dump directory does not exist: {dump_dir}") + + fhm_args = [ + str(THIS_DIR / "extract_ac6_runtime_fhm.py"), + "--dump-dir", + str(dump_dir), + "--output", + str(typed_out), + ] + if raw_manifest.exists(): + fhm_args.extend(["--manifest", str(raw_manifest)]) + run_step(fhm_args, repo_root) + + run_step( + [ + str(THIS_DIR / "parse_ac6_swg.py"), + "--input", + str(typed_out), + "--output", + str(swg_out), + ], + repo_root, + ) + + run_step( + [ + str(THIS_DIR / "export_ac6_ntxr.py"), + "--input", + str(typed_out), + "--output", + str(ntxr_out), + ], + repo_root, + ) + + summary = { + "asset_root": str(asset_root), + "raw_manifest": str(raw_manifest) if raw_manifest.exists() else None, + "dump_dir": str(dump_dir), + "typed_manifest": str(typed_out / "manifest.json"), + "swg_manifest": str(swg_out / "manifest.json"), + "ntxr_manifest": str(ntxr_out / "manifest.json"), + } + + raw_data = read_json(raw_manifest) + typed_data = read_json(typed_out / "manifest.json") + swg_data = read_json(swg_out / "manifest.json") + ntxr_data = read_json(ntxr_out / "manifest.json") + + if raw_data: + summary["pac_entries"] = raw_data.get("entry_count") + summary["pac_extracted"] = raw_data.get("extracted_count") + summary["pac_skipped"] = raw_data.get("skipped_count") + fhm_containers = count_fhm_containers(typed_data) + if fhm_containers is not None: + summary["fhm_containers"] = fhm_containers + if swg_data: + summary["swg_files"] = swg_data.get("parsed_count") + if ntxr_data: + summary["textures_exported"] = ntxr_data.get("exported_count") + summary["textures_skipped"] = ntxr_data.get("skipped_count") + + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())