From 144839f24b6e64f0e56b33fc0fbd1e4fed7f5357 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 16 Jun 2026 15:39:52 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-13):=20T1a/T1b=20=E2=80=94=20SC01/07?= =?UTF-8?q?7=20onboarded=20+=20all-asm=20byte-match=20(d19c9580)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ov_SC01_077 scaffolded: config/overlays.mk (OVERLAY_BINARIES + var block), config/splat.ov_SC01_077.yaml (from template), check.sha, empty symbols, + ov_SC01_077 entry & sentinel anchor in the 4 Python BINARIES dicts - GATE T1b: make check BINARY=ov_SC01_077 -> d19c9580 BYTE-IDENTICAL @ 100% INCLUDE_ASM; clean rebuild leaves main 143dbb89 + resident 8e17e02f unregressed - reusable non-word-aligned-overlay handling (~75% of fleet) added to the template + Makefile: (1) [word_floor, bin, trailing] carve for the final 1-3 bytes spimdisasm drops; (2) build/assets/%.o incbin rule (+ .data align=1) for splat bin assets (asset_path scoped per-alias); (3) objcopy end-align TRIM (shrink-only, <=3 B) removing the .ld's segment-end ALIGN(.,4) pad - .gitignore /assets/ (regenerable splat output); clean removes assets/ --- .gitignore | 1 + Makefile | 32 +++++++++++++-- config/check.ov_SC01_077.sha | 1 + config/overlays.mk | 34 +++++++++++++++ config/splat.ov_SC01_077.yaml | 59 +++++++++++++++++++++++++++ config/splat.us.overlay.template.yaml | 4 ++ config/symbols.ov_SC01_077.txt | 6 +++ diff_settings.py | 4 ++ phase-ends/CURRENT_PHASE.md | 3 +- tools/difficulty.py | 3 ++ tools/dup_report.py | 2 + tools/progress.py | 3 ++ 12 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 config/check.ov_SC01_077.sha create mode 100644 config/overlays.mk create mode 100644 config/splat.ov_SC01_077.yaml create mode 100644 config/symbols.ov_SC01_077.txt diff --git a/.gitignore b/.gitignore index 3cdcb16245..6a60845bd5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ # config/splat.us.exe.yaml + config/symbols.us.txt + the EXE. The durable RE record # is the symbol file, not these (build scaffolding, not committed work). /asm/ +/assets/ /include/include_asm.h /include/macro.inc /include/labels.inc diff --git a/Makefile b/Makefile index 68543aa852..b92afb67ec 100644 --- a/Makefile +++ b/Makefile @@ -366,6 +366,14 @@ ASM_SRCS := $(shell find $(ASM_DIR) -name '*.s' -not -path '$(ASM_DIR)/nonmatchi C_SRCS := $(shell find $(SRC_DIR) -name '*.c' $(SRC_PRUNE) 2>/dev/null) OBJS := $(ASM_SRCS:%.s=build/%.o) $(C_SRCS:%.c=build/%.o) +# splat `bin` subsegs (raw byte regions — e.g. an overlay's trailing non-word-aligned bytes that +# spimdisasm's data path drops, since it won't emit a <4-byte partial word). splat extracts them to +# assets//*.bin and references build/assets//*.o in the .ld; wrap each raw .bin into a +# linkable object (bytes verbatim in .data). Per-binary: asset_path is scoped to assets/ so +# overlays' same-named `trailing.bin` never collide; main/resident have no assets -> empty. +ASSET_BINS := $(shell find assets/$(BINARY) -name '*.bin' 2>/dev/null) +ASSET_OBJS := $(ASSET_BINS:assets/%.bin=build/assets/%.o) + # extract: splat split -> asm/, the linker script, include/ macros, undefined_*_auto.txt. extract: @mkdir -p $(OUT_DIR) @@ -391,6 +399,17 @@ build/asm/%.o: asm/%.s @echo " AS $@" @$(AS) $(ASFLAGS) -o $@ $< +# Wrap a splat `bin` asset (raw bytes) into a linkable object: assemble a one-line stub that +# .incbin's the raw file into .data (format-safe — same mipsel-as as everything else, no objcopy +# -I binary arch guessing). The .ld pulls it by path. The --set-section-alignment forces .data to +# 1-byte align (as defaults it to 16) so ld places the 1-3 trailing bytes at the exact word-floor +# offset and does NOT pad the image up to a 16/8-byte boundary (that added a stray byte otherwise). +build/assets/%.o: assets/%.bin + @mkdir -p $(dir $@) + @echo " INCBIN $@" + @printf '.section .data\n.incbin "%s"\n' "$<" | $(AS) $(ASFLAGS) -o $@ + @$(OBJCOPY) --set-section-alignment .data=1 $@ + # C path (Phase 6): modern cpp -> vintage cc1 -> maspsx -> modern as. Each src/*.c is # splat-generated INCLUDE_ASM stubs (file-scope __asm__ .include of the per-function # asm/nonmatchings//.s); as we match, stubs are replaced by real C. The flags @@ -412,7 +431,7 @@ build/src/%.o: src/%.c build/src/boot.o: CC1FLAGS := -quiet -O0 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker # link (the .ld pulls in the .o by path) + objcopy to the raw PS-X EXE image. -$(OUT): $(OBJS) $(LD_SCRIPT) +$(OUT): $(OBJS) $(ASSET_OBJS) $(LD_SCRIPT) @set -e mkdir -p $(dir $@) # PsyQ SDK library integrations are EXE-only (libgs/libgte/sound/apicard are @@ -473,6 +492,13 @@ endif $(LD) -T $(LD_SCRIPT) -T $(UNDEF_SYMS) -T $(UNDEF_FUNCS) $$SYMS --no-check-sections -Map $(MAPFILE) -o $(ELF) echo " OBJCOPY $@" $(OBJCOPY) -O binary $(ELF) $@ + # Trim the linker's end-of-segment 4-align pad: splat's .ld does `. = ALIGN(., 4)` after the + # data section, over-emitting up to 3 zero bytes when the payload size isn't 4-aligned (most + # overlays; the EXE + resident are 4-aligned so this never fires for them). Shrink-ONLY and + # capped at 3 bytes -> it can never hide a real shortfall (build < target fails the SHA) nor + # extend the image. The target's true byte length is the matched payload $(EXE). + tsz=$$(stat -c%s "$(EXE)"); osz=$$(stat -c%s "$@"); d=$$((osz - tsz)) + if [ "$$d" -gt 0 ] && [ "$$d" -le 3 ]; then truncate -s "$$tsz" "$@"; echo " TRIM $@ (-$$d B linker end-align pad)"; fi # build = produce $(OUT) and verify its SHA1 (check pulls in $(OUT)). build: check @@ -509,6 +535,6 @@ expected: build # clean: remove ALL regenerable outputs (build/ + the splat tree) so a config change # is followed by a stale-free `make clean && make extract && make build` (H3). clean: - @rm -rf build expected asm undefined_syms_auto.txt undefined_funcs_auto.txt + @rm -rf build expected asm assets undefined_syms_auto.txt undefined_funcs_auto.txt @rm -f include/include_asm.h include/macro.inc include/labels.inc include/gte_macros.inc - @echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, include macros, undefined_*_auto.txt)." + @echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, assets/, include macros, undefined_*_auto.txt)." diff --git a/config/check.ov_SC01_077.sha b/config/check.ov_SC01_077.sha new file mode 100644 index 0000000000..4617663604 --- /dev/null +++ b/config/check.ov_SC01_077.sha @@ -0,0 +1 @@ +d19c9580a02dc63ba1f0e7e0c770f3b10de35635 ov_SC01_077 diff --git a/config/overlays.mk b/config/overlays.mk new file mode 100644 index 0000000000..64dc60ed74 --- /dev/null +++ b/config/overlays.mk @@ -0,0 +1,34 @@ +# config/overlays.mk — GENERATED overlay-binary registry (Phase 13). +# ============================================================================= +# Holds OVERLAY_BINARIES + the per- Makefile variable blocks for every onboarded +# location overlay. `-include`d by the Makefile BEFORE its `BINARIES :=` line so the +# aliases enter the build. Regenerated idempotently by tools/new_overlay.sh (T4); the +# hand-maintained Makefile body is never edited to add an overlay. Committed (tiny +# config, not ROM-derived). Each block mirrors the resident_* block (the flat-blob +# recipe): EXE -> the extracted 0.4.dec, VRAM_BASE/TEXT_LO = the shared slot 0x80128158, +# nested asm/ + src/ + build/, per-binary undefined_*_auto + GHIDRA_PROG. +# ============================================================================= +OVERLAY_BINARIES := ov_SC01_077 + +# --- ov_SC01_077 (location overlay SC01/FILE_077, vram 0x80128158 — Phase 13 T1) --- +# The milestone overlay (load byte-proven, Phase 3 T6b). 731,607 B (0xB29D7) -> end vram +# 0x801DAB2F. Decompressed 0.4.dec payload (LZSS repack is Gen3, not in the verify path). +ov_SC01_077_EXE := extracted/retail/SC01.CD.dir/FILE_077.dir/0.4.dec +ov_SC01_077_NAME := ov_SC01_077 +ov_SC01_077_OUT_DIR := build/ov_SC01_077 +ov_SC01_077_OUT := $(ov_SC01_077_OUT_DIR)/$(ov_SC01_077_NAME) +ov_SC01_077_ELF := $(ov_SC01_077_OUT).elf +ov_SC01_077_MAPFILE := $(ov_SC01_077_OUT).map +ov_SC01_077_LD_SCRIPT := $(ov_SC01_077_OUT).ld +ov_SC01_077_SPLAT_YAML := config/splat.ov_SC01_077.yaml +ov_SC01_077_CHECK_SHA := config/check.ov_SC01_077.sha +ov_SC01_077_SYMBOLS := config/symbols.ov_SC01_077.txt +ov_SC01_077_SIG := .run/sig.ov_SC01_077.jsonl +ov_SC01_077_GHIDRA_PROG := ov_SC01_077 +ov_SC01_077_VRAM_BASE := 0x80128158 +ov_SC01_077_TEXT_LO := 0x80128158 +ov_SC01_077_TEXT_HI := 0x801DAB2F +ov_SC01_077_ASM_DIR := asm/ov_SC01_077 +ov_SC01_077_SRC_DIR := src/ov_SC01_077 +ov_SC01_077_UNDEF_SYMS := build/ov_SC01_077/undefined_syms_auto.txt +ov_SC01_077_UNDEF_FUNCS := build/ov_SC01_077/undefined_funcs_auto.txt diff --git a/config/splat.ov_SC01_077.yaml b/config/splat.ov_SC01_077.yaml new file mode 100644 index 0000000000..77ea1f3f8a --- /dev/null +++ b/config/splat.ov_SC01_077.yaml @@ -0,0 +1,59 @@ +# splat config — Brave Fencer Musashi location overlay ov_SC01_077 (SC01.CD/FILE_077/0.4.dec) +# Phase 13 T1: the milestone overlay (load byte-proven, Phase 3 T6b). Instantiated from +# config/splat.us.overlay.template.yaml. Flat decompressed blob @ vram 0x80128158 (the shared +# overlay slot); code at file 0x0; data tail from 0x5E978 (end of the last sig_image function, +# vram 0x80186AD0) to EOF 0xB29D7 (731,607 B -> end vram 0x801DAB2F). The text/data split is +# iterated vs `make check BINARY=ov_SC01_077` (Phase-5/10 precedent). Matches the DECOMPRESSED +# 0.4.dec (sha1 d19c9580…); LZSS repack is Gen3 (not in the verify path). +name: ov_SC01_077 +sha1: d19c9580a02dc63ba1f0e7e0c770f3b10de35635 +options: + basename: ov_SC01_077 + target_path: extracted/retail/SC01.CD.dir/FILE_077.dir/0.4.dec + elf_path: build/ov_SC01_077/ov_SC01_077.elf + base_path: .. + platform: psx + compiler: PSYQ + + asm_path: asm/ov_SC01_077 + src_path: src/ov_SC01_077 + asset_path: assets/ov_SC01_077 # `bin` subsegs (the trailing carve) -> assets/ov_SC01_077/*.bin + build_path: build + ld_script_path: build/ov_SC01_077/ov_SC01_077.ld + ld_dependencies: True + + undefined_syms_auto_path: build/ov_SC01_077/undefined_syms_auto.txt + undefined_funcs_auto_path: build/ov_SC01_077/undefined_funcs_auto.txt + + find_file_boundaries: False + + o_as_suffix: True + use_legacy_include_asm: False + + section_order: [".rodata", ".text", ".data", ".bss"] + + symbol_addrs_path: + - config/symbols.us.txt + - config/symbols.resident.txt + - config/symbols.ov_SC01_077.txt + + subalign: 2 + + string_encoding: ASCII + data_string_encoding: ASCII + rodata_string_guesser_level: 2 + data_string_guesser_level: 2 + +segments: + - name: ov_SC01_077 + type: code + start: 0x0 + vram: 0x80128158 + align: 4 + subsegments: + - [0x0, c, ov_SC01_077] # code: vram 0x80128158..0x80186AD0 (file 0x0..0x5E978) + - [0x5E978, data, tail] # data tail (word-aligned bulk): file 0x5E978..0xB29D4 + - [0xB29D4, bin, trailing] # final 3 bytes (EOF 0xB29D7 not word-aligned; spimdisasm drops + # a trailing partial word and a <4-byte `data` carve emits nothing, + # so use `bin` = raw .incbin, byte-exact). Word-aligned overlays omit this. + - [0xB29D7] # EOF (731,607 B) -> end vram 0x801DAB2F diff --git a/config/splat.us.overlay.template.yaml b/config/splat.us.overlay.template.yaml index 8e8790e7cb..e6a3008cd5 100644 --- a/config/splat.us.overlay.template.yaml +++ b/config/splat.us.overlay.template.yaml @@ -38,6 +38,10 @@ options: asm_path: asm/@ALIAS@ src_path: src/@ALIAS@ + # asset_path scoped per-alias so a `bin` subseg (the trailing-partial-word carve, below) lands at + # assets/@ALIAS@/*.bin and never collides with another overlay's same-named asset. The Makefile's + # build/assets/%.o rule incbin's it into a linkable object. + asset_path: assets/@ALIAS@ # build_path = build (NOT build/): splat writes the .ld's object paths under # $(build_path), and the Makefile's pattern rules build them at build/asm/** + build/src/**. # Only the elf/ld/output live in build//. diff --git a/config/symbols.ov_SC01_077.txt b/config/symbols.ov_SC01_077.txt new file mode 100644 index 0000000000..b4a921eb75 --- /dev/null +++ b/config/symbols.ov_SC01_077.txt @@ -0,0 +1,6 @@ +// config/symbols.ov_SC01_077.txt — overlay SC01/FILE_077 local symbols (Phase 13) +// R13/R15: overlay-REGION ONLY (vram 0x80128158..0x801DAB2F); NEVER merged into +// config/symbols.us.txt. Seeded in T1c from sig_image (.run/sig.ov_SC01_077.jsonl) + +// docs/duplicates.cross.md to fix conservative splat carves (jump-table-in-rodata, §8/§11) +// and attach stable names. The all-asm byte-match (T1b) does not depend on this file. +// count=0 diff --git a/diff_settings.py b/diff_settings.py index fe7e219ed9..bf9b4be9a9 100644 --- a/diff_settings.py +++ b/diff_settings.py @@ -20,6 +20,10 @@ BINARIES = { "resident": dict(baseimg="expected/build/resident/resident.elf", myimg="build/resident/resident.elf", mapfile="build/resident/resident.map"), + "ov_SC01_077": dict(baseimg="expected/build/ov_SC01_077/ov_SC01_077.elf", + myimg="build/ov_SC01_077/ov_SC01_077.elf", + mapfile="build/ov_SC01_077/ov_SC01_077.map"), + # <<< overlays: tools/new_overlay.sh inserts ov_* entries above this line (Phase 13) >>> } diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 1079400c4e..92a60f60a0 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -23,7 +23,8 @@ All load @ vram 0x80128158. SC01/077 starts with CODE at offset 0x0 (no leading ## Tasks - [x] **T0** — Overlay splat template (`config/splat.us.overlay.template.yaml`) + Makefile `-include config/overlays.mk` indirection. ✅ GATE: clean rebuild main `143dbb89…` + resident `8e17e02f…` BYTE-IDENTICAL (indirection proven a no-op; `overlays.mk` absent → silent `-include` → `BINARIES` unchanged). -- [ ] **T1** — SC01/077 end-to-end. T1a scaffold · T1b all-asm byte-match (`d19c9580…`, milestone) · T1c seed boundaries + Ghidra import · T1d dedup-credit · T1e unique-remainder harvest (Ultracode §12 — prompt R26). +- [~] **T1** — SC01/077 end-to-end. ✅ T1a scaffold (overlays.mk + splat.ov_SC01_077.yaml + check.sha + symbols + 4 Python dicts w/ sentinel) · ✅ T1b all-asm byte-match `d19c9580…` BYTE-IDENTICAL (milestone; main+resident unregressed) · ☐ T1c seed boundaries + Ghidra import · ☐ T1d dedup-credit · ☐ T1e unique-remainder harvest (Ultracode §12 — prompt R26). + - **Reusable finding (→ cookbook §13 + PhaseEnd):** non-word-aligned overlays (≈75% of fleet) need 3 template/Makefile additions: (1) a `[, bin, trailing]` carve for the final 1-3 bytes spimdisasm's data path drops; (2) a `build/assets/%.o` incbin rule (+ `--set-section-alignment .data=1`) wiring splat `bin` assets (scoped `asset_path: assets/`); (3) an objcopy end-align **trim** (shrink-only, ≤3 B) removing the splat `.ld`'s `. = ALIGN(., 4)` segment-end pad. EXE/resident are 4-aligned so untouched. - [ ] **T2** — Dup-pair collapse SC01/005≡006 (both `56760dbe…` from one shared source). - [ ] **T3** — Free-roam SC03/001 (`f8fd92f5…`); recipe scales to a large overlay. - [ ] **T4** — `tools/new_overlay.sh` one-command onboarding (gen `overlays.mk`; sentinel-insert 4 dicts; idempotent). GATE: onboard a fresh overlay in one command. diff --git a/tools/difficulty.py b/tools/difficulty.py index de88ecb0ff..62f80e670e 100644 --- a/tools/difficulty.py +++ b/tools/difficulty.py @@ -18,6 +18,9 @@ BINARIES = { "main": dict(src="src", asm="asm/nonmatchings", md="docs/difficulty.md", csv=".run/difficulty.csv"), "resident": dict(src="src/resident", asm="asm/resident/nonmatchings", md="docs/difficulty.resident.md", csv=".run/difficulty.resident.csv"), + "ov_SC01_077": dict(src="src/ov_SC01_077", asm="asm/ov_SC01_077/nonmatchings", + md="docs/difficulty.ov_SC01_077.md", csv=".run/difficulty.ov_SC01_077.csv"), + # <<< overlays: tools/new_overlay.sh inserts ov_* entries above this line (Phase 13) >>> } SRCS = ASM_ROOT = MD = CSV = None # set by main() from --binary diff --git a/tools/dup_report.py b/tools/dup_report.py index 9e74000220..d1025f5734 100644 --- a/tools/dup_report.py +++ b/tools/dup_report.py @@ -26,6 +26,8 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent BINARIES = { "main": dict(sig=".run/sig.SLUS_007.26.jsonl", md="docs/duplicates.md"), "resident": dict(sig=".run/sig.resident.jsonl", md="docs/duplicates.resident.md"), + "ov_SC01_077": dict(sig=".run/sig.ov_SC01_077.jsonl", md="docs/duplicates.ov_SC01_077.md"), + # <<< overlays: tools/new_overlay.sh inserts ov_* entries above this line (Phase 13) >>> } CROSS_MD = "docs/duplicates.cross.md" diff --git a/tools/progress.py b/tools/progress.py index e3baca4a14..ad031dfb51 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -23,6 +23,9 @@ BINARIES = { src="src", asm="asm/nonmatchings", out="docs/progress.md"), "resident": dict(build="build/resident/resident", check="config/check.resident.sha", src="src/resident", asm="asm/resident/nonmatchings", out="docs/progress.resident.md"), + "ov_SC01_077": dict(build="build/ov_SC01_077/ov_SC01_077", check="config/check.ov_SC01_077.sha", + src="src/ov_SC01_077", asm="asm/ov_SC01_077/nonmatchings", out="docs/progress.ov_SC01_077.md"), + # <<< overlays: tools/new_overlay.sh inserts ov_* entries above this line (Phase 13) >>> } BINARY = next((sys.argv[i + 1] for i, x in enumerate(sys.argv) if x == "--binary" and i + 1 < len(sys.argv)), "main")