diff --git a/.gitignore b/.gitignore index 541cdcb15..d23f46e88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,32 @@ # ============================================================ -# ROM-content policy — see PROJECT_CONTEXT.md rule H1 +# ROM-content policy — PROJECT_CONTEXT.md rule H1, IN FORCE (Phase 33, the public flip). # -# While the repo is PRIVATE, ROM-derived content MAY be committed -# (asm/, assets/, extracted/, decompressed .CD output). Compliance is -# handled by a pre-public history scrub + a rom->decoder regeneration -# tool. Two things stay ignored regardless: -# 1. the raw multi-GB disc dump (GitHub rejects >100 MB files; it is -# trivially reproducible from the disc), and -# 2. pure regenerated build churn / large binary caches. -# Re-add asm/ assets/ extracted/ to this list before going public. +# The repository ships NO ROM-derived bytes and NO proprietary SDK: the game disc, the +# extracted payloads (`extracted/`, regenerated from YOUR dump by `make disc-extract` and +# verified against the committed manifest), the splat disassembly (`asm/`, `assets/`), the +# RAM-dump corpus (`dumps/*.bin`), the Ghidra project (`ghidra/`, whose database embeds the +# program bytes — regenerable from text via tools/ghidra_rebuild.sh), Sony's PsyQ SDK +# (`tools/psyq/`, optional and user-supplied), the session-transcript archive, and +# redistributed third-party binaries. Symbol names, addresses, hashes, configs and the +# decompiled C are what a decomp publishes (sotn precedent). The private history that +# carried these while the repo was private (rule R1, 2026-06-10 .. 2026-09-06) lives in the +# archive repo only. # ============================================================ -# Raw disc dump — never committed (size + reproducible) +# Raw disc dump — never committed (size + reproducible; your own dump goes here) /Brave Fencer Musashi (USA)/ /disks/ -# RAM-dump corpus (dumps/*.bin) is COMMITTED while private (H1 relaxed; static, not regenerable; -# off-machine backup). Joins the pre-public ROM-content scrub. INDEX.md documents each. +# RAM-dump corpus — 28 × 2 MB images of the running game (local-only; dumps/INDEX.md + +# dumps/CHECKSUMS.sha1 stay tracked and describe them) +/dumps/*.bin + +# The Claude Code session-transcript archive (private; the archive repo holds it) +/session archive/ + +# Redistributed third-party binaries: download per docs/SETUP.md (sha256 recorded there) +/tools/ghidra-ext/*.zip +/tools/brave-CUE/brave.exe # Project-local runtime scratch (MCP server log + stop sentinel) — replaces /tmp use. # Contents-exclude form (the /tools/bin/*.sha256 precedent) so the IRREPLACEABLE recon @@ -115,27 +125,20 @@ /expected/ # Regenerated splat outputs — recreated by `make extract` from the committed -# config/splat.us.exe.yaml + config/symbols.us.txt + the EXE. The durable RE record -# is the symbol file, not these (build scaffolding, not committed work). +# config/splat.*.yaml + config/symbols*.txt + the extracted payloads. The durable RE record +# is the symbol file, not these (build scaffolding, not committed work). The four splat preset +# headers (include/include_asm.h, macro.inc, labels.inc, gte_macros.inc) are TRACKED since P33 B1: +# generic presets, identical for every binary, ROM-free — the compile-only CI needs them. /asm/ /assets/ -/include/include_asm.h -/include/macro.inc -/include/labels.inc -/include/gte_macros.inc /undefined_syms_auto.txt /undefined_funcs_auto.txt -# Ghidra project — COMMITTED as the irreplaceable RE-work backup (private repo; rule R20, -# Drew 2026-06-15). NOT regenerable by hand (annotations/types/structs beyond symbols.us.txt). -# Track the project; exclude ONLY live-session transients so a running/restarted MCP server -# doesn't dirty the tree. Re-commit after significant RE + a Ghidra save. (Ghidra pinned at -# 12.1 -> the committed .rep restores directly.) Joins the pre-public ROM-content scrub list. -/ghidra/*.lock -/ghidra/**/*.lock -/ghidra/*.lock~ -/ghidra/**/*.lock~ -/ghidra/**/tmp*.ps +# Ghidra project — its database embeds the program bytes (ROM-derived), so it is not tracked +# (P33 B1; it was committed while private under rule R20). The RE work it holds is exported to +# text (config/ghidra/*.jsonl) and the project is rebuilt from that + the disc by +# tools/ghidra_rebuild.sh. NEVER `git clean -x` here — that deletes the working database. +/ghidra/ # Toolchain downloads & caches (not source) /.venv/ @@ -145,14 +148,12 @@ __pycache__/ # pattern). Contents-exclude form (not /tools/bin/) so the !re-include can apply. /tools/bin/* !/tools/bin/*.sha256 -# PsyQ SDK — Sony-copyrighted (stays PRIVATE; excluded from the public mirror, like ROM content). -# Back up the SMALL gathered working artifacts (lib40/ *.LIB, lib40_elf/ *.a, the PsyQ *.EXE tools, -# psyq-obj-parser, conv47/) — hard to re-source if the upstream archives go down (rule R20). Ignore -# ONLY the >100MB raw source archives (GitHub rejects them; the working libs are already extracted -# from them — the disc-dump-equivalent, re-sourceable from redump if ever truly needed). -/tools/psyq/psyq40usa.zip -/tools/psyq/*.bin -/tools/psyq/*.cue +# PsyQ SDK — Sony-copyrighted; OPTIONAL and user-supplied (tools/fetch_psyq.sh), never distributed. +# The whole directory is ignored (P33 B1; the small working artifacts were committed while private +# under rule R20 and live in the archive repo). The build is byte-identical without it (the +# INCLUDE_ASM fallback tiles); with it, main links Sony's real objects (`make sdk-dual` proves both). +# The checksums of the archives a user must supply are tracked at tools/psyq_CHECKSUMS.sha256. +/tools/psyq/ /tools/pcsx-redux/ *.tar.gz # ...EXCEPT the old-gcc cc1 compiler tarballs — back them up (gathered compilers; small; verified by @@ -177,19 +178,17 @@ Thumbs.db # incl. FMV) and NOT the prototype EXEs. Per-ROM subfolders under extracted/: # extracted/retail/ — retail SLUS-00726 disc extraction (3 files committed) # extracted/proto/ — prototype main EXEs (ROM-derived; fully ignored) -# Nested allowlist: ignore everything under extracted/, re-include only the 3 -# retail committed files (EXE + manifest + manifest hash — always safe; sotn -# precedent). This keeps the "zero ROM-derived bulk staged" guarantee. +# Nested allowlist: ignore everything under extracted/, re-include only the 2 manifest +# files (hashes, never bytes — the oracle `make disc-extract` verifies YOUR extraction +# against). The EXE itself left the tree at P33 B1 (H1): it is regenerated from your disc. /extracted/* !/extracted/retail/ /extracted/retail/* -!/extracted/retail/SLUS_007.26 !/extracted/retail/manifest.jsonl !/extracted/retail/manifest.sha1 -# NOTE (H1 relaxed, private phase): assets/ and other ROM-derived content stay -# committable while the repo is private. asm/ + the splat-generated include macros + -# undefined_*_auto.txt are IGNORED above: `make extract` regenerates them deterministically +# NOTE: asm/ + assets/ + undefined_*_auto.txt are IGNORED above: `make extract` regenerates +# them deterministically from the extracted payloads # from the committed yaml + symbols, so they are build scaffolding, not durable RE work. # (Phase 5 refined the earlier "commit asm/" stance once the regeneration flow existed; # the durable record is config/symbols.us.txt.) @@ -234,9 +233,6 @@ unsloth_compiled_cache/ !/.run/s45/*.jsonl !/.run/s45/*.tsv !/.run/s45/onboard*.log -/tools/psyq/lib46/ -/tools/psyq/lib42/ -/tools/psyq/lib421/ # P32 T3 (2026-09-05): the one-agent-per-function wave's ledger + drafts (R20 — the drafts are the only copy of # byte-verified MATCH/NEAR bodies outside the harness transcripts; scratch dirs stay ignored) diff --git a/Makefile b/Makefile index 1c6d92c04..3ace65418 100644 --- a/Makefile +++ b/Makefile @@ -144,19 +144,24 @@ CC1_SMOKE_FLAGS := -quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-lin BINUTILS_WARN_MAJOR := 2 BINUTILS_WARN_MINOR := 38 -.PHONY: help check-env extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health +.PHONY: help check-env disc-extract extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health # ----------------------------------------------------------------------------- help: - @echo "BFM-decomp — make targets:" - echo " make check-env Phase-4 toolchain preflight (the only live target)" - echo " make extract [Phase 5] splat split -> asm/ + linker scripts" - echo " make build [Phase 5] full pipeline -> build/us/SLUS_007.26 (+ SHA1 check)" - echo " make check [Phase 5] standalone SHA1 verification" - echo " make expected [Phase 5] snapshot build/us -> expected/ (asm-differ baseline)" - echo " make clean [Phase 5] remove build output" - echo " make report [Phase 7] regenerate docs/ progress+difficulty+duplicate digests" - echo " make sig-refresh [Phase 7] regenerate .run/sig.*.jsonl from Ghidra (MCP must be stopped)" + @echo "BFM-decomp — make targets (218 binaries: main + resident + 141 overlays + 75 modules; BINARY= scopes a target)" + echo " make check-env toolchain preflight (python/venv/cc1/maspsx/binutils; the extracted EXE if present)" + echo " make disc-extract regenerate extracted/ from YOUR redump dump in disks/ and verify it against the committed manifest (H1: no ROM in the repo)" + echo " make extract splat split one binary -> asm/, the linker script (runs disc-extract if its payload is absent)" + echo " make extract-all disc-extract + splat split every binary (main first, then parallel)" + echo " make check build one binary and compare its SHA1 to config/check..sha (build is an alias)" + echo " make check-all build + SHA1-check every binary in parallel (build-all is an alias)" + echo " make sdk-dual main byte-identical WITH and WITHOUT the (optional, user-supplied) PsyQ objects" + echo " make report regenerate docs/ progress + difficulty + duplicate digests (BINARY=main also the fleet roll-up)" + echo " make tools-health the pre-work ritual: fresh sigs, both boundary oracles, every audit, report, digest assertion" + echo " make sig-main | sig-main-oracle | sig-overlays | sig-resident | sig-modules the byte-derived function sigs (.run/sig.*.jsonl)" + echo " make audit-corpus | audit-binaries | audit-text-sources | audit-digest | audit-disc | audit-frontier the oracles" + echo " make clean remove build/, expected/, asm/, assets/ (fleet-wide; then: make extract-all && make check-all)" + echo "The R22 contract proof: make clean && make extract-all && make check-all (expects: check-all: 218 passed, 0 failed of 218)" # ----------------------------------------------------------------------------- # Phase 7 reports: deterministic, committable docs/ digests. progress/difficulty/dup_report @@ -577,7 +582,9 @@ check-env: echo "[PASS] mipsel binutils $$asver (< 2.38)" fi fi - # 6) committed EXE hash == EXPECTED_EXE_SHA1 (reused constant; fresh-clone-safe) + # 6) the extracted EXE hash == EXPECTED_EXE_SHA1 (reused constant). P33 B1: the EXE is no longer + # committed — it is regenerated from the user's disc by `make disc-extract`, so its absence is a + # WARN with the instruction, not a FAIL; a PRESENT but wrong EXE is still a FAIL. if [ -f "$(EXE)" ]; then want=$$($(PYTHON) -c 'from tools.bfm_extract.extract_exe import EXPECTED_EXE_SHA1 as h; print(h)' 2>/dev/null) got=$$(sha1sum "$(EXE)" | cut -d' ' -f1) @@ -587,8 +594,21 @@ check-env: echo "[FAIL] $(EXE) sha1 $${got:-none} != expected $${want:-unknown}"; fail=1 fi else - echo "[FAIL] $(EXE) missing (committed retail EXE)"; fail=1 + echo "[WARN] $(EXE) absent — run 'make disc-extract' with your redump dump in $(DISC_DIR)/ (docs/SETUP.md §4.4)" fi + # 7) the committed manifest oracle is self-consistent (manifest.sha1 == sha1(manifest.jsonl)); ms. + if [ -f "$(EXTRACT_ROOT)/manifest.jsonl" ] && [ -f "$(EXTRACT_ROOT)/manifest.sha1" ]; then + mgot=$$(sha1sum "$(EXTRACT_ROOT)/manifest.jsonl" | cut -d' ' -f1) + mwant=$$(cat "$(EXTRACT_ROOT)/manifest.sha1") + if [ "$$mgot" = "$$mwant" ]; then + echo "[PASS] $(EXTRACT_ROOT)/manifest.sha1 == sha1(manifest.jsonl) ($$mgot)" + else + echo "[FAIL] $(EXTRACT_ROOT)/manifest.sha1 ($$mwant) != sha1(manifest.jsonl) ($$mgot) — the committed oracle is inconsistent"; fail=1 + fi + else + echo "[FAIL] $(EXTRACT_ROOT)/manifest.jsonl + manifest.sha1 missing (the committed extraction oracle)"; fail=1 + fi + if [ -f "$(DISC_TRACK1)" ]; then echo "[INFO] disc dump present: '$(DISC_TRACK1)'"; else echo "[INFO] no disc dump under $(DISC_DIR)/ (needed only to (re)generate extracted/)"; fi echo if [ "$$fail" -ne 0 ]; then echo "check-env: FAIL — see the [FAIL] lines above." @@ -764,9 +784,48 @@ C_DEPS := $(C_SRCS:%.c=build/%.d) ASSET_BINS := $(shell find assets/$(BINARY) -name '*.bin' 2>/dev/null) ASSET_OBJS := $(ASSET_BINS:assets/%.bin=build/assets/%.o) +# ----------------------------------------------------------------------------- +# disc-extract (P33 B1): the rom->decoder step. The repository ships NO ROM bytes (H1): every +# payload under extracted/ — the EXE, the .CD archives, the PAC entries, the LZSS-decoded 0.4.dec +# overlays and module blobs — is regenerated from the USER'S OWN redump dump by +# tools/bfm_extract/extract.py, and the result is compared against the COMMITTED oracle +# extracted/retail/manifest.jsonl (1,801 rows; manifest.sha1 is its hash). The oracle is never +# written by a build step (--expect-manifest compares; a plain `extract.py` run is how the oracle +# was made). Idempotent: a tree that already matches the oracle is a ~5 s hash probe and a no-op. +# The oracle lists the 3 .DA audio files from Tracks 2-4, so the canonical input is the 4-track +# BIN/CUE; a Track-1-only dump is accepted with an explicit PARTIAL verdict (never a silent pass). +DISC_DIR ?= disks +DISC_TRACK1 := $(DISC_DIR)/Brave Fencer Musashi (USA) (Track 1).bin +DISC_TRACK2 := $(DISC_DIR)/Brave Fencer Musashi (USA) (Track 2).bin +EXTRACT_ROOT := extracted/retail +disc-extract: + @set -e + audio="" + if [ ! -f "$(DISC_TRACK2)" ]; then audio="--allow-missing-audio"; fi + if $(PYTHON) tools/bfm_extract/extract.py --verify --out "$(EXTRACT_ROOT)" $$audio >/dev/null 2>&1; then + echo "disc-extract: up to date — $(EXTRACT_ROOT)/ matches the committed manifest (sha1 $$(cat $(EXTRACT_ROOT)/manifest.sha1))" + exit 0 + fi + if [ ! -f "$(DISC_TRACK1)" ]; then + echo "[FAIL] disc-extract: no disc dump at '$(DISC_TRACK1)'" + echo " Stage your own redump dump of Brave Fencer Musashi (USA) — the 4-track BIN/CUE (Track 1 = data;" + echo " Tracks 2-4 = the .DA audio) — under $(DISC_DIR)/ (docs/SETUP.md §4.4). The repository ships no ROM bytes (H1)." + exit 2 + fi + # 1) the dump is the canonical one: full-track SHA1 + CRC32 vs redump (refuses a non-canonical dump) + $(PYTHON) tools/bfm_extract/extract_exe.py --bin "$(DISC_TRACK1)" --verify-disc + if [ -n "$$audio" ]; then echo "disc-extract: NOTE — '$(DISC_TRACK2)' absent: the 3 .DA audio files are not extracted; the result is PARTIAL"; fi + # 2) extract everything and COMPARE against the committed oracle (writes nothing on a match) + $(PYTHON) tools/bfm_extract/extract.py --bin "$(DISC_TRACK1)" --out "$(EXTRACT_ROOT)" --expect-manifest "$(EXTRACT_ROOT)/manifest.jsonl" $$audio + # 3) the idempotency probe of the tree that was just written + $(PYTHON) tools/bfm_extract/extract.py --verify --out "$(EXTRACT_ROOT)" $$audio + echo "disc-extract: OK — $(EXTRACT_ROOT)/ regenerated from '$(DISC_TRACK1)' and verified against the committed manifest (sha1 $$(cat $(EXTRACT_ROOT)/manifest.sha1))" + # extract: splat split -> asm/, the linker script, include/ macros, undefined_*_auto.txt. extract: @mkdir -p $(OUT_DIR) + # P33 B1: the payload is the user's — regenerate extracted/ from the disc when it is absent (H1). + if [ ! -f "$(EXE)" ]; then $(MAKE) --no-print-directory disc-extract; fi # A re-extract REWRITES every .s — and an object's assembly arrives through INCLUDE_ASM, which # expands to a `.include` consumed by maspsx/as AFTER cpp. So `.o <- .s` is NOT a dependency make # can see (-MMD tracks headers only), and an incremental build after an extract silently links @@ -1034,6 +1093,9 @@ JOBS ?= 16 # parallel binary builds/extracts (override: `make check-all JOBS= # (+ build/psyq) exist before the parallel fan-out; then extract the rest in parallel. extract-all: @mkdir -p .run; : > .run/extract-all.txt + # P33 B1: the payloads come from the user's disc — regenerate/verify extracted/ ONCE, serially, + # before anything reads it (a ~5 s no-op when the tree already matches the committed manifest). + $(MAKE) --no-print-directory disc-extract # Under the global `-e` a failing main extract now aborts here. It previously did NOT: its status # was swallowed by .ONESHELL, and the closing `! grep -q` then passed regardless — a seed failure # could sail through as green. @@ -1114,7 +1176,10 @@ clean: echo "clean: this removes asm/ for ALL binaries. Recover with 'make extract-all'."; \ fi @rm -rf build expected asm assets undefined_syms_auto.txt undefined_funcs_auto.txt - @rm -f include/include_asm.h include/macro.inc include/labels.inc include/gte_macros.inc - @echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, assets/, include macros, undefined_*_auto.txt)." + @# P33 B1: the four splat preset headers (include/include_asm.h, macro.inc, labels.inc, gte_macros.inc) + @# are TRACKED now — generic splat presets, identical for every binary and ROM-free, so a compile-only + @# CI can assemble without a disc. `make extract` rewrites them only if splat's presets change (a + @# visible diff), so clean no longer deletes them. + @echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, assets/, undefined_*_auto.txt)." diff --git a/docs/SETUP.md b/docs/SETUP.md index a6d228810..6070d3234 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1069,6 +1069,31 @@ fills fast). Nothing is leaking — but the host does not get the memory back on * `gate_main` **REFUSES a draft containing its own `INCLUDE_ASM`** (substituting it restores the stub, so the build passes for free and the function counts as banked), and counts banks from the SOURCE. +### P33 B1 (S86, 2026-09-06) — `make disc-extract`: the rom→decoder step, promoted into the build +- **`make disc-extract`** (`DISC_DIR ?= disks`): the repository ships no ROM bytes (H1 in force). The target (1) probes + `extracted/retail/` against the committed oracle (`extract.py --verify`, **0.7 s** when up to date → no-op), else (2) + requires `disks/Brave Fencer Musashi (USA) (Track 1).bin` (exit 2 with the staging instruction), (3) verifies the dump + is the canonical redump one (`extract_exe.py --verify-disc`: Track-1 SHA1 `b44f0f0a…` + CRC32 `c238191b`; a truncated + or foreign dump FAILS and nothing is written), (4) extracts everything and COMPARES against the committed + `extracted/retail/manifest.jsonl` (`extract.py --expect-manifest`: identical → nothing written; different → the actual + manifest goes to `.run/extract/` with the first 20 differences listed, exit 1), (5) re-verifies the tree. Measured: + **15.7 s wall** for the full 4-track extraction + manifest compare + verify (1,801 files, 759 MB). `extract`/`extract-all` + call it when a payload is absent / once up front, so the R22 chain on a fresh clone is `make disc-extract && make + extract-all && make check-all`. A Track-1-only dump is accepted with an explicit **PARTIAL** verdict (`--allow-missing- + audio`: the 3 `.DA` audio rows from Tracks 2–4 are excluded from the compare and the verify; without the flag a partial + extraction FAILS, R43). The committed oracle is never overwritten by a build step; regenerating it is a deliberate plain + `extract.py` run. `extracted/proto/` (the two prototype EXEs) is NOT produced here — it comes from the prototype discs via + `tools/bfm_extract/extract_proto_exe.py`. +- `make check-env`: the EXE's absence is now a `[WARN]` with the instruction (present-but-wrong stays `[FAIL]`); new step 7 + asserts the oracle is self-consistent (`manifest.sha1 == sha1(manifest.jsonl)`); `[INFO]` disc presence. `make help` + rewritten for the live targets. `make clean` no longer deletes the four splat preset headers — `include/include_asm.h`, + `macro.inc`, `labels.inc`, `gte_macros.inc` are TRACKED (generic splat presets, identical for every binary, ROM-free; the + compile-only CI needs them; a splat preset change shows as a diff). +- `.gitignore` re-tightened to H1: `/dumps/*.bin`, `/session archive/`, `/tools/ghidra-ext/*.zip`, `/tools/brave-CUE/brave.exe`, + `/ghidra/` (whole), `/tools/psyq/` (whole; checksums move to `tools/psyq_CHECKSUMS.sha256` in C3), the EXE re-include + dropped (only the two manifest files stay under `extracted/`), the header rewritten. Tracked files under those paths stay + tracked until the C3 `git rm --cached` commit (ignore rules bind untracked paths only). + ### P33 A4 (S86, 2026-09-06) — the family map carries its own coverage - `tools/family_hseq.py` → `.run/family_hseq.json` now records `"binaries"` (the binaries it SCANNED — 217: 141 overlays + 75 modules + the resident) and `"open_instances"`; `load()` returns `(instances, scanned)`. Reason: at 100% `families` diff --git a/include/gte_macros.inc b/include/gte_macros.inc new file mode 100644 index 000000000..9d724142b --- /dev/null +++ b/include/gte_macros.inc @@ -0,0 +1,415 @@ +.ifndef .L_GTE_MACRO_INC +.L_GTE_MACRO_INC: + +## GTE instruction macros +## These are meant for use with GAS and replace DMPSX + +.macro cop2op fake_op, op, gbg = 0, sf = 1, mx = 0, v = 0, cv = 0, lm = 0 + cop2 \fake_op << 20 | \gbg << 20 | \sf << 19 | \mx << 17 | \v << 15 | \cv << 13 | \lm << 10 | \op +.endm + +/* RTPS 15 0x4A180001 Perspective transform */ +.macro rtps + cop2op 0x01, 0x01 +.endm + +/* RTPT 23 0x4A280030 Perspective transform on 3 points */ +.macro rtpt + cop2op 0x02, 0x30 +.endm + +/* DPCL 8 0x4A680029 Depth Cue Color light */ +.macro dpcl + cop2op 0x06, 0x29 +.endm + +/* DPCS 8 0x4A780010 Depth Cueing */ +.macro dpcs + cop2op 0x07, 0x10 +.endm + +/* DPCT 17 0x4AF8002A Depth cue color RGB0,RGB1,RGB2 */ +.macro dpct + cop2op 0x0F, 0x2A +.endm + +/* INTPL 8 0x4A980011 Interpolation of vector and far color */ +.macro intpl + cop2op 0x09, 0x11 +.endm + +/* NCS 14 0x4AC8041E Normal color v0 */ +.macro ncs + cop2op 0x0C, 0x1E, lm = 1 +.endm + +/* NCT 30 0x4AD80420 Normal color v0, v1, v2 */ +.macro nct + cop2op 0x0D, 0x20, lm = 1 +.endm + +/* NCDS 19 0x4AE80413 Normal color depth cuev0 */ +.macro ncds + cop2op 0x0E, 0x13, lm = 1 +.endm + +/* NCDT 44 0x4AF80416 Normal color depth cue v0, v1, v2 */ +.macro ncdt + cop2op 0x0F, 0x16, lm = 1 +.endm + +/* NCCS 17 0x4B08041B Normal color col. v0 */ +.macro nccs + cop2op 0x10, 0x1B, lm = 1 +.endm + +/* NCCT 39 0x4B18043F Normal color col.v0, v1, v2 */ +.macro ncct + cop2op 0x11, 0x3F, lm = 1 +.endm + +/* CDP 13 0x4B280414 Color Depth Queue */ +.macro cdp + cop2op 0x12, 0x14, lm = 1 +.endm + +/* CC 11 0x4B38041C Color Col. */ +.macro cc + cop2op 0x13, 0x1C, lm = 1 +.endm + +/* NCLIP 8 0x4B400006 Normal clipping */ +.macro nclip + cop2op 0x14, 0x06, sf = 0 +.endm + +/* AVSZ3 5 0x4B58002D Average of three Z values */ +.macro avsz3 + cop2op 0x15, 0x2D +.endm + +/* AVSZ4 6 0x4B68002E Average of four Z values */ +.macro avsz4 + cop2op 0x16, 0x2E +.endm + + +## Instructions which take an argument +# gbg: arg is 5 bit wide +# sf : arg is 1 bit wide +# mx : arg is 2 bit wide +# v : arg is 2 bit wide +# cv : arg is 2 bit wide +# lm : arg is 1 bit wide + +/* mvmva 8 0x4A4nnn12 Multiply vector by matrix and vector addition. */ +.macro mvmva sf, mx, v, cv, lm + cop2op 0x04, 0x12, sf = \sf, mx = \mx, v = \v, cv = \cv, lm = \lm +.endm + +/* SQR 5 0x4AAn0428 Square of vector */ +.macro sqr sf + cop2op 0x0A, 0x28, sf = \sf, lm = 1 +.endm + +/* OP 6 0x4B7n000C Outer Product */ +.macro op sf + cop2op 0x17, 0x0C, sf = \sf +.endm + +/* GPF 6 0x4B9n003D General purpose interpolation */ +.macro gpf sf + cop2op 0x19, 0x3D, sf = \sf +.endm + +/* GPL 5 0x4BAn003E general purpose interpolation */ +.macro gpl sf + cop2op 0x1A, 0x3E, sf = \sf +.endm + + +## Convenience macros + +/* rtv0 - 0x4A486012 v0 * rotmatrix */ +.macro rtv0 + # .word 0x4A486012 + mvmva 1, 0, 0, 3, 0 +.endm + +/* rtv1 - 0x4A48E012 v1 * rotmatrix */ +.macro rtv1 + # .word 0x4A48E012 + mvmva 1, 0, 1, 3, 0 +.endm + +/* rtv2 - 0x4A496012 v2 * rotmatrix */ +.macro rtv2 + # .word 0x4A496012 + mvmva 1, 0, 2, 3, 0 +.endm + +/* rtir12 - 0x4A49E012 ir * rotmatrix */ +.macro rtir12 + # .word 0x4A49E012 + mvmva 1, 0, 3, 3, 0 +.endm + +/* rtir0 - 0x4A41E012 ir * rotmatrix */ +.macro rtir0 + # .word 0x4A41E012 + mvmva 0, 0, 3, 3, 0 +.endm + +/* rtv0tr - 0x4A480012 v0 * rotmatrix + tr vector */ +.macro rtv0tr + # .word 0x4A480012 + mvmva 1, 0, 0, 0, 0 +.endm + +/* rtv1tr - 0x4A488012 v1 * rotmatrix + tr vector */ +.macro rtv1tr + # .word 0x4A488012 + mvmva 1, 0, 1, 0, 0 +.endm + +/* rtv2tr - 0x4A490012 v2 * rotmatrix + tr vector */ +.macro rtv2tr + # .word 0x4A490012 + mvmva 1, 0, 2, 0, 0 +.endm + +/* rtirtr - 0x4A498012 ir * rotmatrix + tr vector */ +.macro rtirtr + # .word 0x4A498012 + mvmva 1, 0, 3, 0, 0 +.endm + +/* rtv0bk - 0x4A482012 v0 * rotmatrix + bk vector */ +.macro rtv0bk + # .word 0x4A482012 + mvmva 1, 0, 0, 1, 0 +.endm + +/* rtv1bk - 0x4A48A012 v1 * rotmatrix + bk vector */ +.macro rtv1bk + # .word 0x4A48A012 + mvmva 1, 0, 1, 1, 0 +.endm + +/* rtv2bk - 0x4A492012 v2 * rotmatrix + bk vector */ +.macro rtv2bk + # .word 0x4A492012 + mvmva 1, 0, 2, 1, 0 +.endm + +/* rtirbk - 0x4A49A012 ir * rotmatrix + bk vector */ +.macro rtirbk + # .word 0x4A49A012 + mvmva 1, 0, 3, 1, 0 +.endm + +/* ll - 0x4A4A6412 v0 * light matrix. Lower limit result to 0 */ +.macro ll + # .word 0x4A4A6412 + mvmva 1, 1, 0, 3, 1 +.endm + +/* llv0 - 0x4A4A6012 v0 * light matrix */ +.macro llv0 + # .word 0x4A4A6012 + mvmva 1, 1, 0, 3, 0 +.endm + +/* llv1 - 0x4A4AE012 v1 * light matrix */ +.macro llv1 + # .word 0x4A4AE012 + mvmva 1, 1, 1, 3, 0 +.endm + +/* llv2 - 0x4A4B6012 v2 * light matrix */ +.macro llv2 + # .word 0x4A4B6012 + mvmva 1, 1, 2, 3, 0 +.endm + +/* llvir - 0x4A4BE012 ir * light matrix */ +.macro llvir + # .word 0x4A4BE012 + mvmva 1, 1, 3, 3, 0 +.endm + +/* llv0tr - 0x4A4A0012 v0 * light matrix + tr vector */ +.macro llv0tr + # .word 0x4A4A0012 + mvmva 1, 1, 0, 0, 0 +.endm + +/* llv1tr - 0x4A4A8012 v1 * light matrix + tr vector */ +.macro llv1tr + # .word 0x4A4A8012 + mvmva 1, 1, 1, 0, 0 +.endm + +/* llv2tr - 0x4A4B0012 v2 * light matrix + tr vector */ +.macro llv2tr + # .word 0x4A4B0012 + mvmva 1, 1, 2, 0, 0 +.endm + +/* llirtr - 0x4A4B8012 ir * light matrix + tr vector */ +.macro llirtr + # .word 0x4A4B8012 + mvmva 1, 1, 3, 0, 0 +.endm + +/* llv0bk - 0x4A4A2012 v0 * light matrix + bk vector */ +.macro llv0bk + # .word 0x4A4A2012 + mvmva 1, 1, 0, 1, 0 +.endm + +/* llv1bk - 0x4A4AA012 v1 * light matrix + bk vector */ +.macro llv1bk + # .word 0x4A4AA012 + mvmva 1, 1, 1, 1, 0 +.endm + +/* llv2bk - 0x4A4B2012 v2 * light matrix + bk vector */ +.macro llv2bk + # .word 0x4A4B2012 + mvmva 1, 1, 2, 1, 0 +.endm + +/* llirbk - 0x4A4BA012 ir * light matrix + bk vector */ +.macro llirbk + # .word 0x4A4BA012 + mvmva 1, 1, 3, 1, 0 +.endm + +/* lc - 0x4A4DA412 v0 * color matrix, Lower limit clamped to 0 */ +.macro lc + # .word 0x4A4DA412 + mvmva 1, 2, 3, 1, 1 +.endm + +/* lcv0 - 0x4A4C6012 v0 * color matrix */ +.macro lcv0 + # .word 0x4A4C6012 + mvmva 1, 2, 0, 3, 0 +.endm + +/* lcv1 - 0x4A4CE012 v1 * color matrix */ +.macro lcv1 + # .word 0x4A4CE012 + mvmva 1, 2, 1, 3, 0 +.endm + +/* lcv2 - 0x4A4D6012 v2 * color matrix */ +.macro lcv2 + # .word 0x4A4D6012 + mvmva 1, 2, 2, 3, 0 +.endm + +/* lcvir - 0x4A4DE012 ir * color matrix */ +.macro lcvir + # .word 0x4A4DE012 + mvmva 1, 2, 3, 3, 0 +.endm + +/* lcv0tr - 0x4A4C0012 v0 * color matrix + tr vector */ +.macro lcv0tr + # .word 0x4A4C0012 + mvmva 1, 2, 0, 0, 0 +.endm + +/* lcv1tr - 0x4A4C8012 v1 * color matrix + tr vector */ +.macro lcv1tr + # .word 0x4A4C8012 + mvmva 1, 2, 1, 0, 0 +.endm + +/* lcv2tr - 0x4A4D0012 v2 * color matrix + tr vector */ +.macro lcv2tr + # .word 0x4A4D0012 + mvmva 1, 2, 2, 0, 0 +.endm + +/* lcirtr - 0x4A4D8012 ir * color matrix + tr vector */ +.macro lcirtr + # .word 0x4A4D8012 + mvmva 1, 2, 3, 0, 0 +.endm + +/* lev0bk - 0x4A4C2012 v0 * color matrix + bk vector */ +.macro lev0bk + # .word 0x4A4C2012 + mvmva 1, 2, 0, 1, 0 +.endm + +/* lev1bk - 0x4A4CA012 v1 * color matrix + bk vector */ +.macro lev1bk + # .word 0x4A4CA012 + mvmva 1, 2, 1, 1, 0 +.endm + +/* lev2bk - 0x4A4D2012 v2 * color matrix + bk vector */ +.macro lev2bk + # .word 0x4A4D2012 + mvmva 1, 2, 2, 1, 0 +.endm + +/* leirbk - 0x4A4DA012 ir * color matrix + bk vector */ +.macro leirbk + # .word 0x4A4DA012 + mvmva 1, 2, 3, 1, 0 +.endm + +/* sqr12 - 0x4AA80428 square of ir 1,19,12 */ +# .macro sqr12 +# # .word 0x4AA80428 +# sqr 1 +# .endm + +/* sqr0 - 0x4AA00428 square of ir 1,31, 0 */ +# .macro sqr0 +# # .word 0x4AA00428 +# sqr 0 +# .endm + +/* op12 - 0x4B78000C outer product 1,19,12 */ +.macro op12 + # .word 0x4B78000C + op 1 +.endm + +/* op0 - 0x4B70000C outer product 1,31, 0 */ +.macro op0 + # .word 0x4B70000C + op 0 +.endm + +/* gpf12 - 0x4B98003D general purpose interpolation 1,19,12 */ +.macro gpf12 + # .word 0x4B98003D + gpf 1 +.endm + +/* gpf0 - 0x4B90003D general purpose interpolation 1,31, 0 */ +.macro gpf0 + # .word 0x4B90003D + gpf 0 +.endm + +/* gpl12 - 0x4BA8003E general purpose interpolation 1,19,12 */ +.macro gpl12 + # .word 0x4BA8003E + gpl 1 +.endm + +/* gpl0 - 0x4BA0003E general purpose interpolation 1,31, 0 */ +.macro gpl0 + # .word 0x4BA0003E + gpl 0 +.endm + +.endif diff --git a/include/include_asm.h b/include/include_asm.h new file mode 100644 index 000000000..5409cb5ab --- /dev/null +++ b/include/include_asm.h @@ -0,0 +1,43 @@ +#ifndef INCLUDE_ASM_H +#define INCLUDE_ASM_H + +#if !defined(M2CTX) && !defined(PERMUTER) + +#ifndef INCLUDE_ASM +#define INCLUDE_ASM(FOLDER, NAME) \ + __asm__( \ + ".section .text\n" \ + " .set noat\n" \ + " .set noreorder\n" \ + " .include \"" FOLDER "/" #NAME ".s\"\n" \ + " .set reorder\n" \ + " .set at\n" \ + ) +#endif +#ifndef INCLUDE_RODATA +#define INCLUDE_RODATA(FOLDER, NAME) \ + __asm__( \ + ".section .rodata\n" \ + " .include \"" FOLDER "/" #NAME ".s\"\n" \ + ".section .text" \ + ) +#endif + +#if INCLUDE_ASM_USE_MACRO_INC +__asm__(".include \"include/macro.inc\"\n"); +#else +__asm__(".include \"include/labels.inc\"\n"); +#endif + +#else + +#ifndef INCLUDE_ASM +#define INCLUDE_ASM(FOLDER, NAME) +#endif +#ifndef INCLUDE_RODATA +#define INCLUDE_RODATA(FOLDER, NAME) +#endif + +#endif /* !defined(M2CTX) && !defined(PERMUTER) */ + +#endif /* INCLUDE_ASM_H */ diff --git a/include/labels.inc b/include/labels.inc new file mode 100644 index 000000000..03f1d4fa8 --- /dev/null +++ b/include/labels.inc @@ -0,0 +1,61 @@ +# This file is used by the original compiler/assembler. +# Defines the expected assembly macros. + +.include "gte_macros.inc" + +# A function symbol. +.macro glabel label, visibility=global + .\visibility \label + .type \label, @function + \label: + .ent \label +.endm + +# The end of a function symbol. +.macro endlabel label + .size \label, . - \label + .end \label +.endm + +# An alternative entry to a function. +.macro alabel label, visibility=global + .\visibility \label + .type \label, @function + \label: + .aent \label +.endm + +# A label referenced by an error handler table. +.macro ehlabel label, visibility=global + .\visibility \label + \label: +.endm + + +# A label referenced by a jumptable. +.macro jlabel label, visibility=local + .\visibility \label + \label: +.endm + + +# A data symbol. +.macro dlabel label, visibility=global + .\visibility \label + .type \label, @object + \label: +.endm + +# End of a data symbol. +.macro enddlabel label + .size \label, . - \label +.endm + + +# Label to signal the symbol haven't been matched yet. +.macro nonmatching label, size=1 + .global \label\().NON_MATCHING + .type \label\().NON_MATCHING, @object + .size \label\().NON_MATCHING, \size + \label\().NON_MATCHING: +.endm diff --git a/include/macro.inc b/include/macro.inc new file mode 100644 index 000000000..6cbd876f6 --- /dev/null +++ b/include/macro.inc @@ -0,0 +1,68 @@ +# This file is used by modern gas. +# Defines the expected assembly macros + +# Evaluate this file only once in case it's included more than once +.ifndef _MACRO_INC_GUARD +.internal _MACRO_INC_GUARD +.set _MACRO_INC_GUARD, 1 + +# A function symbol. +.macro glabel label, visibility=global + .\visibility \label + .type \label, @function + \label: + .ent \label +.endm + +# The end of a function symbol. +.macro endlabel label + .size \label, . - \label + .end \label +.endm + +# An alternative entry to a function. +.macro alabel label, visibility=global + .\visibility \label + .type \label, @function + \label: + .aent \label +.endm + +# A label referenced by an error handler table. +.macro ehlabel label, visibility=global + .\visibility \label + \label: +.endm + + +# A label referenced by a jumptable. +.macro jlabel label, visibility=global + .\visibility \label + \label: +.endm + + +# A data symbol. +.macro dlabel label, visibility=global + .\visibility \label + .type \label, @object + \label: +.endm + +# End of a data symbol. +.macro enddlabel label + .size \label, . - \label +.endm + + +# Label to signal the symbol haven't been matched yet. +.macro nonmatching label, size=1 + .global \label\().NON_MATCHING + .type \label\().NON_MATCHING, @object + .size \label\().NON_MATCHING, \size + \label\().NON_MATCHING: +.endm + +.include "gte_macros.inc" + +.endif diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index c4f088813..a33a16756 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -49,8 +49,8 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo `progress.py` loud-fail) — xHigh, R39 controls — see Log 2026-09-06 A2 - [x] **A3** `NO_SDK` knob + `make sdk-dual` (+ tools-health wiring, `[skip]` without SDK dirs) — Max — see Log 2026-09-06 A3 - [x] **A4** `tools/family_hseq.py` regen — Low (became an instrument fix: the map now carries its own coverage) — see Log -- [ ] **B1** `make disc-extract` (extract.py `--expect-manifest` / `--allow-missing-audio`; check-env; `.gitignore` - re-tightened; the 4 splat preset headers tracked; `clean` fixed) — Max +- [x] **B1** `make disc-extract` (extract.py `--expect-manifest` / `--allow-missing-audio`; check-env; `.gitignore` + re-tightened; the 4 splat preset headers tracked; `clean` fixed) — Max — see Log 2026-09-06 B1 - [ ] **B2** 34 absolute includes → `../shared/` + portable-include audit + 15-binary re-gate — xHigh - [ ] **B3** `tools/bootstrap.sh` / `make bootstrap` + check-env extensions + fresh-clone proof 218/218 — xHigh - [ ] **B4** `tools/fetch_psyq.sh` + CHECKSUMS rows (20 lib40 LIBs, psyq-obj-parser) BEFORE `tools/psyq/` leaves git — xHigh @@ -130,15 +130,30 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact denominator and warns "predates the coverage field" on an old-format map. Controls: current map → 0 `[warn]`; the same map with the keys removed → the predates warning; restored → 0. No other tool calls `family_hseq.load()` (grep; consumers read the json's `families` key, unchanged). `docs/family-hseq.md` regenerated (0 families). SETUP: P33 A4 section (R21). +- **2026-09-06 (S86) — B1 (`make disc-extract`).** `tools/bfm_extract/extract.py`: `--expect-manifest` (compare against the + committed oracle, never write it; mismatch → `.run/extract/` + the first 20 diffs, exit 1; refuses an internally + inconsistent oracle) and `--allow-missing-audio` (PARTIAL verdict for Track-1-only dumps, in both extract and `--verify`). + Makefile: `disc-extract` (probe → disc presence → `--verify-disc` → extract+compare → verify), `extract` runs it when + `$(EXE)` is absent, `extract-all` runs it once first, `check-env` step 6 WARN-on-absent + step 7 oracle self-consistency + + disc INFO, `help` rewritten, `clean` keeps the 4 preset headers (now tracked), `.PHONY`. `.gitignore` re-tightened to H1 + (see SETUP P33 B1). **Controls:** no disc → exit 2 + staging text; truncated 100 MB Track 1 → `--verify-disc` FAIL, nothing + written, `git diff --exit-code` on the oracle clean; the real run with `extracted/` moved aside and only the two manifest + files restored → `disc-extract: OK` in **15.7 s** (EXIT=0, `.run/P33/b1_disc_extract.log`), `diff -rq` against the previous + tree IDENTICAL for all 1,801 retail files; second run "up to date" in 0.7 s; `make check-env` OK with the new PASS lines. + Slip, recorded: the previous tree also held `extracted/proto/` (the two prototype EXEs from the prototype discs, not the + retail one) and I removed the moved-aside copy in the same command as the diff — regenerated from `disks/` with + `tools/bfm_extract/extract_proto_exe.py` (see the next line). -## 🛑 SESSION CHECKPOINT — A1 ✓ A2 ✓ A3 ✓ A4 ✓ (P6 rules check done after A4); NEXT = B1 `make disc-extract` (Max) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1 ✓; NEXT = B2 (the 34 absolute includes + the portable-include audit + 15-binary re-gate, xHigh) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **NEXT = B1** (`make disc-extract`, Max — a build-input step; wrong = a green build on the -wrong bytes). If `git log -1 --format=%s` does not start with `fix(phase-33): A4`, A4's commit did not land: re-run -`.venv/bin/python tools/family_hseq.py` and `tools/audit_binaries.py` (expect 0 `[warn]`) before committing. +(R1–R73), then WAIT for Drew. **NEXT = B2** (xHigh: the 34 absolute `#include "/home/musashi/bfm-decomp/src/shared/…"` +lines in 19 `src/ov_*/…_jr_*.c` files across 15 binaries → `../shared/`; extend `tools/audit_text_sources.py` with a +portability class; controls 34 → 0; re-gate the 15 binaries with `make extract BINARY= && make check BINARY=`). +If `git log -1 --format=%s` does not start with `feat(phase-33): B1`, B1's commit did not land: re-run `make disc-extract` +(expect "up to date" in ~1 s) and `make check-env` (OK) before committing. ### 1. Where we are **Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at @@ -207,14 +222,17 @@ items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2. ### 3. NEXT — in order 0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) · `.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not). -1. **B1** (Max) per the plan's Block B: `tools/bfm_extract/extract.py` gains `--expect-manifest` + `--allow-missing-audio`; - `make disc-extract`; `extract`/`extract-all`/`check-env`/`help`/`clean` changes; `.gitignore` re-tightened; the 4 splat - preset headers tracked. Gotchas: `extract.py:379-381` overwrites the manifest on every run (compare, never write); the - committed manifest has 1,801 rows incl. 3 `.DA` audio files from Tracks 2–4; `disks/` here holds all 4 tracks + cue; the - verification moves `extracted/` aside (`mv extracted .run/extracted.off`) — put it back if anything fails. -2. Then B2 → B3 → B4 → B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box, +1. **B2** (xHigh): `grep -rl '#include "/home/musashi/bfm-decomp/src/shared/' src | xargs sed -i 's|#include + "/home/musashi/bfm-decomp/src/shared/|#include "../shared/|'`; `grep -rn '/home/musashi' src include` must print nothing; + `tools/audit_text_sources.py` gains the portability class (absolute, `<…>`, outside-repo includes are offenders; run it + BEFORE the sed → exactly 34 offenders in 19 files, AFTER → 0); re-gate the 15 binaries: `for b in ov_SC02_000 ov_SC02_011 + ov_SC02_016 ov_SC02_017 ov_SC02_026 ov_SC02_039 ov_SC03_001 ov_SC03_006 ov_SC03_091 ov_SC03_105 ov_SC04_018 ov_SC05_003 + ov_SC05_010 ov_SC06_000 ov_SC06_029; do make extract BINARY=$b && make -j8 check BINARY=$b || exit 1; done` (derive the + list from the grep, do not trust this one). Commit "feat(phase-33): B2 …". +2. Then B3 → B4 → B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box, add a Log line, refresh this checkpoint block (the 🛑 block is the ONLY in-phase context the next session inherits), - commit. Next P6 rules check after B4 (8 tasks done). + commit. Next P6 rules check after B4 (8 tasks done). `extracted/proto/` (sep8 + aug31 EXEs) is regenerated from the + prototype discs by `tools/bfm_extract/extract_proto_exe.py` (docstring examples) — B5 needs both files. ### 4. Files this session touched A1: `phase-ends/DIGEST.md`, `phase-ends/CURRENT_PHASE.md`. A2: `tools/main_seed_ends.py` (new), `Makefile` (`sig-main` diff --git a/tools/bfm_extract/extract.py b/tools/bfm_extract/extract.py index 01bba8313..08c5d1d7a 100644 --- a/tools/bfm_extract/extract.py +++ b/tools/bfm_extract/extract.py @@ -29,6 +29,7 @@ from __future__ import annotations import argparse import hashlib +import json import sys from dataclasses import dataclass, field from pathlib import Path @@ -348,13 +349,47 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Re-hash the output tree against its manifest and exit (no extraction).", ) + p.add_argument( + "--expect-manifest", + type=Path, + default=None, + help=( + "P33 B1 (the public `make disc-extract`): compare the extraction against this committed " + "manifest.jsonl (the oracle) INSTEAD of writing one. Identical -> nothing written, exit 0; " + "different -> the actual manifest goes to .run/extract/, the first differences are listed, " + "exit 1. The committed oracle is never overwritten by a build step." + ), + ) + p.add_argument( + "--allow-missing-audio", + action="store_true", + help=( + "With no sibling `(Track 2).bin` staged, compare/verify WITHOUT the 3 .DA audio rows and " + "report PARTIAL instead of failing (a Track-1-only dump). Without this flag a partial " + "extraction against the 4-track oracle FAILS." + ), + ) return p.parse_args(argv) -def cmd_verify(out_root: Path) -> int: +def _is_audio_row(path: str) -> bool: + return path.upper().endswith(".DA") + + +def cmd_verify(out_root: Path, allow_missing_audio: bool = False) -> int: ok, problems = manifest.verify(out_root) + partial = [] + if allow_missing_audio: + partial = [p for p in problems if p.startswith("missing file: ") and _is_audio_row(p)] + problems = [p for p in problems if p not in partial] + ok = not problems if ok: - print(f"RESULT: PASS - every artifact under {out_root}/ matches {manifest.MANIFEST_NAME}.") + if partial: + print(f"RESULT: PASS (PARTIAL) - every artifact under {out_root}/ matches " + f"{manifest.MANIFEST_NAME} except {len(partial)} unstaged .DA audio file(s): " + + ", ".join(p.split(': ', 1)[1] for p in partial)) + else: + print(f"RESULT: PASS - every artifact under {out_root}/ matches {manifest.MANIFEST_NAME}.") return 0 print(f"RESULT: FAIL - {len(problems)} problem(s):", file=sys.stderr) for p in problems[:20]: @@ -362,10 +397,66 @@ def cmd_verify(out_root: Path) -> int: return 1 +ACTUAL_DIR = Path(".run") / "extract" # where a NON-matching extraction's manifest is written (R12) + + +def compare_with_oracle(records: list[dict], oracle: Path, allow_missing_audio: bool) -> int: + """Compare the freshly built records against the committed oracle manifest.jsonl. + + Never writes into the oracle's directory. On a match prints the oracle's own SHA1 (the single + value a contributor quotes to prove their extraction). On a mismatch writes the ACTUAL manifest + to .run/extract/ and prints the first 20 differences (missing / extra / mismatched paths).""" + if not oracle.is_file(): + print(f"ERROR: oracle manifest not found: {oracle}", file=sys.stderr) + return 2 + text = oracle.read_text(encoding="ascii") + oracle_sha1 = hashlib.sha1(text.encode("ascii")).hexdigest() + sha_file = oracle.with_name(manifest.MANIFEST_SHA1_NAME) + if sha_file.is_file(): + recorded = sha_file.read_text(encoding="ascii").strip() + if recorded != oracle_sha1: + print(f"ERROR: {sha_file} records {recorded} but {oracle.name} hashes to {oracle_sha1} — " + f"the committed oracle is internally inconsistent; refusing to compare", file=sys.stderr) + return 2 + expected = {} + for line in text.splitlines(): + if line.strip(): + r = json.loads(line) + expected[r["path"]] = (r["size"], r["sha1"]) + actual = {r["path"]: (r["size"], r["sha1"]) for r in records} + partial = [] + if allow_missing_audio: + partial = sorted(p for p in expected if _is_audio_row(p) and p not in actual) + for p in partial: + del expected[p] + missing = sorted(set(expected) - set(actual)) + extra = sorted(set(actual) - set(expected)) + mismatch = sorted(p for p in expected if p in actual and expected[p] != actual[p]) + if not (missing or extra or mismatch): + note = (f" PARTIAL: {len(partial)} .DA audio row(s) unverified ({', '.join(partial)})" + if partial else "") + print(f"\nManifest: {len(actual)} artifacts == the committed oracle {oracle} " + f"(sha1 {oracle_sha1}); nothing written.{note}") + return 0 + ACTUAL_DIR.mkdir(parents=True, exist_ok=True) + digest = manifest.write(ACTUAL_DIR, records) + print(f"\nManifest MISMATCH against the oracle {oracle} (sha1 {oracle_sha1}): " + f"{len(missing)} missing, {len(extra)} extra, {len(mismatch)} mismatched — " + f"actual manifest written to {ACTUAL_DIR}/ (sha1 {digest})", file=sys.stderr) + shown = 0 + for label, paths in (("missing", missing), ("extra", extra), ("mismatch", mismatch)): + for p in paths: + if shown >= 20: + break + print(f" - {label}: {p}", file=sys.stderr) + shown += 1 + return 1 + + def main(argv: list[str] | None = None) -> int: args = parse_args(argv) if args.verify: - return cmd_verify(args.out) + return cmd_verify(args.out, args.allow_missing_audio) try: with Iso9660Image(args.bin) as img: if args.list: @@ -377,11 +468,17 @@ def main(argv: list[str] | None = None) -> int: return 2 records = manifest.build(args.out) - digest = manifest.write(args.out, records) - print( - f"\nManifest: {len(records)} artifacts -> " - f"{args.out}/{manifest.MANIFEST_NAME} (sha1 {digest})" - ) + if args.expect_manifest: + rc = compare_with_oracle(records, args.expect_manifest, + allow_missing_audio=args.allow_missing_audio and not audio_tracks) + if rc: + return rc + else: + digest = manifest.write(args.out, records) + print( + f"\nManifest: {len(records)} artifacts -> " + f"{args.out}/{manifest.MANIFEST_NAME} (sha1 {digest})" + ) if check_exe_roundtrip(exe_data): print("\nRESULT: PASS - full disc extracted; EXE reproduces the known-good binary.")