feat(phase-33): B1 make disc-extract — the rom->decoder step in the build: extract.py --expect-manifest (compare against the committed oracle, never write it; mismatch -> .run/extract/ + diffs, exit 1) + --allow-missing-audio (explicit PARTIAL for Track-1-only dumps); disc-extract = probe (0.7 s no-op) -> disc presence -> redump SHA1/CRC32 -> extract+compare -> verify (15.7 s full); extract/extract-all call it; check-env WARN-on-absent EXE + oracle self-consistency; help rewritten; the 4 splat preset headers TRACKED (clean keeps them); .gitignore re-tightened to H1 (dumps/*.bin, ghidra/, tools/psyq/, session archive/, ghidra-ext zips, brave.exe; EXE re-include dropped); controls: no-disc exit 2, truncated disc FAILs with the oracle untouched, regenerated tree byte-identical to the previous (1,801 files)

This commit is contained in:
Drew T
2026-09-06 19:02:34 -06:00
parent 62854caca0
commit 0265712916
9 changed files with 869 additions and 81 deletions
+42 -46
View File
@@ -1,22 +1,32 @@
# ============================================================
# ROM-content policy — see PROJECT_CONTEXT.md rule H1
# ROM-content policy — PROJECT_CONTEXT.md rule H1, IN FORCE (Phase 33, the public flip).
#
# While the repo is PRIVATE, ROM-derived content MAY be committed
# (asm/, assets/, extracted/, decompressed .CD output). Compliance is
# handled by a pre-public history scrub + a rom->decoder regeneration
# tool. Two things stay ignored regardless:
# 1. the raw multi-GB disc dump (GitHub rejects >100 MB files; it is
# trivially reproducible from the disc), and
# 2. pure regenerated build churn / large binary caches.
# Re-add asm/ assets/ extracted/ to this list before going public.
# The repository ships NO ROM-derived bytes and NO proprietary SDK: the game disc, the
# extracted payloads (`extracted/`, regenerated from YOUR dump by `make disc-extract` and
# verified against the committed manifest), the splat disassembly (`asm/`, `assets/`), the
# RAM-dump corpus (`dumps/*.bin`), the Ghidra project (`ghidra/`, whose database embeds the
# program bytes — regenerable from text via tools/ghidra_rebuild.sh), Sony's PsyQ SDK
# (`tools/psyq/`, optional and user-supplied), the session-transcript archive, and
# redistributed third-party binaries. Symbol names, addresses, hashes, configs and the
# decompiled C are what a decomp publishes (sotn precedent). The private history that
# carried these while the repo was private (rule R1, 2026-06-10 .. 2026-09-06) lives in the
# archive repo only.
# ============================================================
# Raw disc dump — never committed (size + reproducible)
# Raw disc dump — never committed (size + reproducible; your own dump goes here)
/Brave Fencer Musashi (USA)/
/disks/
# RAM-dump corpus (dumps/*.bin) is COMMITTED while private (H1 relaxed; static, not regenerable;
# off-machine backup). Joins the pre-public ROM-content scrub. INDEX.md documents each.
# RAM-dump corpus — 28 × 2 MB images of the running game (local-only; dumps/INDEX.md +
# dumps/CHECKSUMS.sha1 stay tracked and describe them)
/dumps/*.bin
# The Claude Code session-transcript archive (private; the archive repo holds it)
/session archive/
# Redistributed third-party binaries: download per docs/SETUP.md (sha256 recorded there)
/tools/ghidra-ext/*.zip
/tools/brave-CUE/brave.exe
# Project-local runtime scratch (MCP server log + stop sentinel) — replaces /tmp use.
# Contents-exclude form (the /tools/bin/*.sha256 precedent) so the IRREPLACEABLE recon
@@ -115,27 +125,20 @@
/expected/
# Regenerated splat outputs — recreated by `make extract` from the committed
# config/splat.us.exe.yaml + config/symbols.us.txt + the EXE. The durable RE record
# is the symbol file, not these (build scaffolding, not committed work).
# config/splat.*.yaml + config/symbols*.txt + the extracted payloads. The durable RE record
# is the symbol file, not these (build scaffolding, not committed work). The four splat preset
# headers (include/include_asm.h, macro.inc, labels.inc, gte_macros.inc) are TRACKED since P33 B1:
# generic presets, identical for every binary, ROM-free — the compile-only CI needs them.
/asm/
/assets/
/include/include_asm.h
/include/macro.inc
/include/labels.inc
/include/gte_macros.inc
/undefined_syms_auto.txt
/undefined_funcs_auto.txt
# Ghidra project — COMMITTED as the irreplaceable RE-work backup (private repo; rule R20,
# Drew 2026-06-15). NOT regenerable by hand (annotations/types/structs beyond symbols.us.txt).
# Track the project; exclude ONLY live-session transients so a running/restarted MCP server
# doesn't dirty the tree. Re-commit after significant RE + a Ghidra save. (Ghidra pinned at
# 12.1 -> the committed .rep restores directly.) Joins the pre-public ROM-content scrub list.
/ghidra/*.lock
/ghidra/**/*.lock
/ghidra/*.lock~
/ghidra/**/*.lock~
/ghidra/**/tmp*.ps
# Ghidra project — its database embeds the program bytes (ROM-derived), so it is not tracked
# (P33 B1; it was committed while private under rule R20). The RE work it holds is exported to
# text (config/ghidra/*.jsonl) and the project is rebuilt from that + the disc by
# tools/ghidra_rebuild.sh. NEVER `git clean -x` here — that deletes the working database.
/ghidra/
# Toolchain downloads & caches (not source)
/.venv/
@@ -145,14 +148,12 @@ __pycache__/
# pattern). Contents-exclude form (not /tools/bin/) so the !re-include can apply.
/tools/bin/*
!/tools/bin/*.sha256
# PsyQ SDK — Sony-copyrighted (stays PRIVATE; excluded from the public mirror, like ROM content).
# Back up the SMALL gathered working artifacts (lib40/ *.LIB, lib40_elf/ *.a, the PsyQ *.EXE tools,
# psyq-obj-parser, conv47/) — hard to re-source if the upstream archives go down (rule R20). Ignore
# ONLY the >100MB raw source archives (GitHub rejects them; the working libs are already extracted
# from them — the disc-dump-equivalent, re-sourceable from redump if ever truly needed).
/tools/psyq/psyq40usa.zip
/tools/psyq/*.bin
/tools/psyq/*.cue
# PsyQ SDK — Sony-copyrighted; OPTIONAL and user-supplied (tools/fetch_psyq.sh), never distributed.
# The whole directory is ignored (P33 B1; the small working artifacts were committed while private
# under rule R20 and live in the archive repo). The build is byte-identical without it (the
# INCLUDE_ASM fallback tiles); with it, main links Sony's real objects (`make sdk-dual` proves both).
# The checksums of the archives a user must supply are tracked at tools/psyq_CHECKSUMS.sha256.
/tools/psyq/
/tools/pcsx-redux/
*.tar.gz
# ...EXCEPT the old-gcc cc1 compiler tarballs — back them up (gathered compilers; small; verified by
@@ -177,19 +178,17 @@ Thumbs.db
# incl. FMV) and NOT the prototype EXEs. Per-ROM subfolders under extracted/:
# extracted/retail/ — retail SLUS-00726 disc extraction (3 files committed)
# extracted/proto/ — prototype main EXEs (ROM-derived; fully ignored)
# Nested allowlist: ignore everything under extracted/, re-include only the 3
# retail committed files (EXE + manifest + manifest hash — always safe; sotn
# precedent). This keeps the "zero ROM-derived bulk staged" guarantee.
# Nested allowlist: ignore everything under extracted/, re-include only the 2 manifest
# files (hashes, never bytes — the oracle `make disc-extract` verifies YOUR extraction
# against). The EXE itself left the tree at P33 B1 (H1): it is regenerated from your disc.
/extracted/*
!/extracted/retail/
/extracted/retail/*
!/extracted/retail/SLUS_007.26
!/extracted/retail/manifest.jsonl
!/extracted/retail/manifest.sha1
# NOTE (H1 relaxed, private phase): assets/ and other ROM-derived content stay
# committable while the repo is private. asm/ + the splat-generated include macros +
# undefined_*_auto.txt are IGNORED above: `make extract` regenerates them deterministically
# NOTE: asm/ + assets/ + undefined_*_auto.txt are IGNORED above: `make extract` regenerates
# them deterministically from the extracted payloads
# from the committed yaml + symbols, so they are build scaffolding, not durable RE work.
# (Phase 5 refined the earlier "commit asm/" stance once the regeneration flow existed;
# the durable record is config/symbols.us.txt.)
@@ -234,9 +233,6 @@ unsloth_compiled_cache/
!/.run/s45/*.jsonl
!/.run/s45/*.tsv
!/.run/s45/onboard*.log
/tools/psyq/lib46/
/tools/psyq/lib42/
/tools/psyq/lib421/
# P32 T3 (2026-09-05): the one-agent-per-function wave's ledger + drafts (R20 — the drafts are the only copy of
# byte-verified MATCH/NEAR bodies outside the harness transcripts; scratch dirs stay ignored)
+79 -14
View File
@@ -144,19 +144,24 @@ CC1_SMOKE_FLAGS := -quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-lin
BINUTILS_WARN_MAJOR := 2
BINUTILS_WARN_MINOR := 38
.PHONY: help check-env extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health
.PHONY: help check-env disc-extract extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health
# -----------------------------------------------------------------------------
help:
@echo "BFM-decomp — make targets:"
echo " make check-env Phase-4 toolchain preflight (the only live target)"
echo " make extract [Phase 5] splat split -> asm/ + linker scripts"
echo " make build [Phase 5] full pipeline -> build/us/SLUS_007.26 (+ SHA1 check)"
echo " make check [Phase 5] standalone SHA1 verification"
echo " make expected [Phase 5] snapshot build/us -> expected/ (asm-differ baseline)"
echo " make clean [Phase 5] remove build output"
echo " make report [Phase 7] regenerate docs/ progress+difficulty+duplicate digests"
echo " make sig-refresh [Phase 7] regenerate .run/sig.*.jsonl from Ghidra (MCP must be stopped)"
@echo "BFM-decomp — make targets (218 binaries: main + resident + 141 overlays + 75 modules; BINARY=<alias> scopes a target)"
echo " make check-env toolchain preflight (python/venv/cc1/maspsx/binutils; the extracted EXE if present)"
echo " make disc-extract regenerate extracted/ from YOUR redump dump in disks/ and verify it against the committed manifest (H1: no ROM in the repo)"
echo " make extract splat split one binary -> asm/, the linker script (runs disc-extract if its payload is absent)"
echo " make extract-all disc-extract + splat split every binary (main first, then parallel)"
echo " make check build one binary and compare its SHA1 to config/check.<alias>.sha (build is an alias)"
echo " make check-all build + SHA1-check every binary in parallel (build-all is an alias)"
echo " make sdk-dual main byte-identical WITH and WITHOUT the (optional, user-supplied) PsyQ objects"
echo " make report regenerate docs/ progress + difficulty + duplicate digests (BINARY=main also the fleet roll-up)"
echo " make tools-health the pre-work ritual: fresh sigs, both boundary oracles, every audit, report, digest assertion"
echo " make sig-main | sig-main-oracle | sig-overlays | sig-resident | sig-modules the byte-derived function sigs (.run/sig.*.jsonl)"
echo " make audit-corpus | audit-binaries | audit-text-sources | audit-digest | audit-disc | audit-frontier the oracles"
echo " make clean remove build/, expected/, asm/, assets/ (fleet-wide; then: make extract-all && make check-all)"
echo "The R22 contract proof: make clean && make extract-all && make check-all (expects: check-all: 218 passed, 0 failed of 218)"
# -----------------------------------------------------------------------------
# Phase 7 reports: deterministic, committable docs/ digests. progress/difficulty/dup_report
@@ -577,7 +582,9 @@ check-env:
echo "[PASS] mipsel binutils $$asver (< 2.38)"
fi
fi
# 6) committed EXE hash == EXPECTED_EXE_SHA1 (reused constant; fresh-clone-safe)
# 6) the extracted EXE hash == EXPECTED_EXE_SHA1 (reused constant). P33 B1: the EXE is no longer
# committed — it is regenerated from the user's disc by `make disc-extract`, so its absence is a
# WARN with the instruction, not a FAIL; a PRESENT but wrong EXE is still a FAIL.
if [ -f "$(EXE)" ]; then
want=$$($(PYTHON) -c 'from tools.bfm_extract.extract_exe import EXPECTED_EXE_SHA1 as h; print(h)' 2>/dev/null)
got=$$(sha1sum "$(EXE)" | cut -d' ' -f1)
@@ -587,8 +594,21 @@ check-env:
echo "[FAIL] $(EXE) sha1 $${got:-none} != expected $${want:-unknown}"; fail=1
fi
else
echo "[FAIL] $(EXE) missing (committed retail EXE)"; fail=1
echo "[WARN] $(EXE) absent — run 'make disc-extract' with your redump dump in $(DISC_DIR)/ (docs/SETUP.md §4.4)"
fi
# 7) the committed manifest oracle is self-consistent (manifest.sha1 == sha1(manifest.jsonl)); ms.
if [ -f "$(EXTRACT_ROOT)/manifest.jsonl" ] && [ -f "$(EXTRACT_ROOT)/manifest.sha1" ]; then
mgot=$$(sha1sum "$(EXTRACT_ROOT)/manifest.jsonl" | cut -d' ' -f1)
mwant=$$(cat "$(EXTRACT_ROOT)/manifest.sha1")
if [ "$$mgot" = "$$mwant" ]; then
echo "[PASS] $(EXTRACT_ROOT)/manifest.sha1 == sha1(manifest.jsonl) ($$mgot)"
else
echo "[FAIL] $(EXTRACT_ROOT)/manifest.sha1 ($$mwant) != sha1(manifest.jsonl) ($$mgot) — the committed oracle is inconsistent"; fail=1
fi
else
echo "[FAIL] $(EXTRACT_ROOT)/manifest.jsonl + manifest.sha1 missing (the committed extraction oracle)"; fail=1
fi
if [ -f "$(DISC_TRACK1)" ]; then echo "[INFO] disc dump present: '$(DISC_TRACK1)'"; else echo "[INFO] no disc dump under $(DISC_DIR)/ (needed only to (re)generate extracted/)"; fi
echo
if [ "$$fail" -ne 0 ]; then
echo "check-env: FAIL — see the [FAIL] lines above."
@@ -764,9 +784,48 @@ C_DEPS := $(C_SRCS:%.c=build/%.d)
ASSET_BINS := $(shell find assets/$(BINARY) -name '*.bin' 2>/dev/null)
ASSET_OBJS := $(ASSET_BINS:assets/%.bin=build/assets/%.o)
# -----------------------------------------------------------------------------
# disc-extract (P33 B1): the rom->decoder step. The repository ships NO ROM bytes (H1): every
# payload under extracted/ — the EXE, the .CD archives, the PAC entries, the LZSS-decoded 0.4.dec
# overlays and module blobs — is regenerated from the USER'S OWN redump dump by
# tools/bfm_extract/extract.py, and the result is compared against the COMMITTED oracle
# extracted/retail/manifest.jsonl (1,801 rows; manifest.sha1 is its hash). The oracle is never
# written by a build step (--expect-manifest compares; a plain `extract.py` run is how the oracle
# was made). Idempotent: a tree that already matches the oracle is a ~5 s hash probe and a no-op.
# The oracle lists the 3 .DA audio files from Tracks 2-4, so the canonical input is the 4-track
# BIN/CUE; a Track-1-only dump is accepted with an explicit PARTIAL verdict (never a silent pass).
DISC_DIR ?= disks
DISC_TRACK1 := $(DISC_DIR)/Brave Fencer Musashi (USA) (Track 1).bin
DISC_TRACK2 := $(DISC_DIR)/Brave Fencer Musashi (USA) (Track 2).bin
EXTRACT_ROOT := extracted/retail
disc-extract:
@set -e
audio=""
if [ ! -f "$(DISC_TRACK2)" ]; then audio="--allow-missing-audio"; fi
if $(PYTHON) tools/bfm_extract/extract.py --verify --out "$(EXTRACT_ROOT)" $$audio >/dev/null 2>&1; then
echo "disc-extract: up to date — $(EXTRACT_ROOT)/ matches the committed manifest (sha1 $$(cat $(EXTRACT_ROOT)/manifest.sha1))"
exit 0
fi
if [ ! -f "$(DISC_TRACK1)" ]; then
echo "[FAIL] disc-extract: no disc dump at '$(DISC_TRACK1)'"
echo " Stage your own redump dump of Brave Fencer Musashi (USA) — the 4-track BIN/CUE (Track 1 = data;"
echo " Tracks 2-4 = the .DA audio) — under $(DISC_DIR)/ (docs/SETUP.md §4.4). The repository ships no ROM bytes (H1)."
exit 2
fi
# 1) the dump is the canonical one: full-track SHA1 + CRC32 vs redump (refuses a non-canonical dump)
$(PYTHON) tools/bfm_extract/extract_exe.py --bin "$(DISC_TRACK1)" --verify-disc
if [ -n "$$audio" ]; then echo "disc-extract: NOTE — '$(DISC_TRACK2)' absent: the 3 .DA audio files are not extracted; the result is PARTIAL"; fi
# 2) extract everything and COMPARE against the committed oracle (writes nothing on a match)
$(PYTHON) tools/bfm_extract/extract.py --bin "$(DISC_TRACK1)" --out "$(EXTRACT_ROOT)" --expect-manifest "$(EXTRACT_ROOT)/manifest.jsonl" $$audio
# 3) the idempotency probe of the tree that was just written
$(PYTHON) tools/bfm_extract/extract.py --verify --out "$(EXTRACT_ROOT)" $$audio
echo "disc-extract: OK — $(EXTRACT_ROOT)/ regenerated from '$(DISC_TRACK1)' and verified against the committed manifest (sha1 $$(cat $(EXTRACT_ROOT)/manifest.sha1))"
# extract: splat split -> asm/, the linker script, include/ macros, undefined_*_auto.txt.
extract:
@mkdir -p $(OUT_DIR)
# P33 B1: the payload is the user's — regenerate extracted/ from the disc when it is absent (H1).
if [ ! -f "$(EXE)" ]; then $(MAKE) --no-print-directory disc-extract; fi
# A re-extract REWRITES every .s — and an object's assembly arrives through INCLUDE_ASM, which
# expands to a `.include` consumed by maspsx/as AFTER cpp. So `.o <- .s` is NOT a dependency make
# can see (-MMD tracks headers only), and an incremental build after an extract silently links
@@ -1034,6 +1093,9 @@ JOBS ?= 16 # parallel binary builds/extracts (override: `make check-all JOBS=
# (+ build/psyq) exist before the parallel fan-out; then extract the rest in parallel.
extract-all:
@mkdir -p .run; : > .run/extract-all.txt
# P33 B1: the payloads come from the user's disc — regenerate/verify extracted/ ONCE, serially,
# before anything reads it (a ~5 s no-op when the tree already matches the committed manifest).
$(MAKE) --no-print-directory disc-extract
# Under the global `-e` a failing main extract now aborts here. It previously did NOT: its status
# was swallowed by .ONESHELL, and the closing `! grep -q` then passed regardless — a seed failure
# could sail through as green.
@@ -1114,7 +1176,10 @@ clean:
echo "clean: this removes asm/ for ALL binaries. Recover with 'make extract-all'."; \
fi
@rm -rf build expected asm assets undefined_syms_auto.txt undefined_funcs_auto.txt
@rm -f include/include_asm.h include/macro.inc include/labels.inc include/gte_macros.inc
@echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, assets/, include macros, undefined_*_auto.txt)."
@# P33 B1: the four splat preset headers (include/include_asm.h, macro.inc, labels.inc, gte_macros.inc)
@# are TRACKED now — generic splat presets, identical for every binary and ROM-free, so a compile-only
@# CI can assemble without a disc. `make extract` rewrites them only if splat's presets change (a
@# visible diff), so clean no longer deletes them.
@echo "clean: removed build/, expected/, and the regenerated splat tree (asm/, assets/, undefined_*_auto.txt)."
+25
View File
@@ -1069,6 +1069,31 @@ fills fast). Nothing is leaking — but the host does not get the memory back on
* `gate_main` **REFUSES a draft containing its own `INCLUDE_ASM`** (substituting it restores the stub,
so the build passes for free and the function counts as banked), and counts banks from the SOURCE.
### P33 B1 (S86, 2026-09-06) — `make disc-extract`: the rom→decoder step, promoted into the build
- **`make disc-extract`** (`DISC_DIR ?= disks`): the repository ships no ROM bytes (H1 in force). The target (1) probes
`extracted/retail/` against the committed oracle (`extract.py --verify`, **0.7 s** when up to date → no-op), else (2)
requires `disks/Brave Fencer Musashi (USA) (Track 1).bin` (exit 2 with the staging instruction), (3) verifies the dump
is the canonical redump one (`extract_exe.py --verify-disc`: Track-1 SHA1 `b44f0f0a…` + CRC32 `c238191b`; a truncated
or foreign dump FAILS and nothing is written), (4) extracts everything and COMPARES against the committed
`extracted/retail/manifest.jsonl` (`extract.py --expect-manifest`: identical → nothing written; different → the actual
manifest goes to `.run/extract/` with the first 20 differences listed, exit 1), (5) re-verifies the tree. Measured:
**15.7 s wall** for the full 4-track extraction + manifest compare + verify (1,801 files, 759 MB). `extract`/`extract-all`
call it when a payload is absent / once up front, so the R22 chain on a fresh clone is `make disc-extract && make
extract-all && make check-all`. A Track-1-only dump is accepted with an explicit **PARTIAL** verdict (`--allow-missing-
audio`: the 3 `.DA` audio rows from Tracks 2–4 are excluded from the compare and the verify; without the flag a partial
extraction FAILS, R43). The committed oracle is never overwritten by a build step; regenerating it is a deliberate plain
`extract.py` run. `extracted/proto/` (the two prototype EXEs) is NOT produced here — it comes from the prototype discs via
`tools/bfm_extract/extract_proto_exe.py`.
- `make check-env`: the EXE's absence is now a `[WARN]` with the instruction (present-but-wrong stays `[FAIL]`); new step 7
asserts the oracle is self-consistent (`manifest.sha1 == sha1(manifest.jsonl)`); `[INFO]` disc presence. `make help`
rewritten for the live targets. `make clean` no longer deletes the four splat preset headers — `include/include_asm.h`,
`macro.inc`, `labels.inc`, `gte_macros.inc` are TRACKED (generic splat presets, identical for every binary, ROM-free; the
compile-only CI needs them; a splat preset change shows as a diff).
- `.gitignore` re-tightened to H1: `/dumps/*.bin`, `/session archive/`, `/tools/ghidra-ext/*.zip`, `/tools/brave-CUE/brave.exe`,
`/ghidra/` (whole), `/tools/psyq/` (whole; checksums move to `tools/psyq_CHECKSUMS.sha256` in C3), the EXE re-include
dropped (only the two manifest files stay under `extracted/`), the header rewritten. Tracked files under those paths stay
tracked until the C3 `git rm --cached` commit (ignore rules bind untracked paths only).
### P33 A4 (S86, 2026-09-06) — the family map carries its own coverage
- `tools/family_hseq.py` → `.run/family_hseq.json` now records `"binaries"` (the binaries it SCANNED — 217: 141 overlays +
75 modules + the resident) and `"open_instances"`; `load()` returns `(instances, scanned)`. Reason: at 100% `families`
+415
View File
@@ -0,0 +1,415 @@
.ifndef .L_GTE_MACRO_INC
.L_GTE_MACRO_INC:
## GTE instruction macros
## These are meant for use with GAS and replace DMPSX
.macro cop2op fake_op, op, gbg = 0, sf = 1, mx = 0, v = 0, cv = 0, lm = 0
cop2 \fake_op << 20 | \gbg << 20 | \sf << 19 | \mx << 17 | \v << 15 | \cv << 13 | \lm << 10 | \op
.endm
/* RTPS 15 0x4A180001 Perspective transform */
.macro rtps
cop2op 0x01, 0x01
.endm
/* RTPT 23 0x4A280030 Perspective transform on 3 points */
.macro rtpt
cop2op 0x02, 0x30
.endm
/* DPCL 8 0x4A680029 Depth Cue Color light */
.macro dpcl
cop2op 0x06, 0x29
.endm
/* DPCS 8 0x4A780010 Depth Cueing */
.macro dpcs
cop2op 0x07, 0x10
.endm
/* DPCT 17 0x4AF8002A Depth cue color RGB0,RGB1,RGB2 */
.macro dpct
cop2op 0x0F, 0x2A
.endm
/* INTPL 8 0x4A980011 Interpolation of vector and far color */
.macro intpl
cop2op 0x09, 0x11
.endm
/* NCS 14 0x4AC8041E Normal color v0 */
.macro ncs
cop2op 0x0C, 0x1E, lm = 1
.endm
/* NCT 30 0x4AD80420 Normal color v0, v1, v2 */
.macro nct
cop2op 0x0D, 0x20, lm = 1
.endm
/* NCDS 19 0x4AE80413 Normal color depth cuev0 */
.macro ncds
cop2op 0x0E, 0x13, lm = 1
.endm
/* NCDT 44 0x4AF80416 Normal color depth cue v0, v1, v2 */
.macro ncdt
cop2op 0x0F, 0x16, lm = 1
.endm
/* NCCS 17 0x4B08041B Normal color col. v0 */
.macro nccs
cop2op 0x10, 0x1B, lm = 1
.endm
/* NCCT 39 0x4B18043F Normal color col.v0, v1, v2 */
.macro ncct
cop2op 0x11, 0x3F, lm = 1
.endm
/* CDP 13 0x4B280414 Color Depth Queue */
.macro cdp
cop2op 0x12, 0x14, lm = 1
.endm
/* CC 11 0x4B38041C Color Col. */
.macro cc
cop2op 0x13, 0x1C, lm = 1
.endm
/* NCLIP 8 0x4B400006 Normal clipping */
.macro nclip
cop2op 0x14, 0x06, sf = 0
.endm
/* AVSZ3 5 0x4B58002D Average of three Z values */
.macro avsz3
cop2op 0x15, 0x2D
.endm
/* AVSZ4 6 0x4B68002E Average of four Z values */
.macro avsz4
cop2op 0x16, 0x2E
.endm
## Instructions which take an argument
# gbg: arg is 5 bit wide
# sf : arg is 1 bit wide
# mx : arg is 2 bit wide
# v : arg is 2 bit wide
# cv : arg is 2 bit wide
# lm : arg is 1 bit wide
/* mvmva 8 0x4A4nnn12 Multiply vector by matrix and vector addition. */
.macro mvmva sf, mx, v, cv, lm
cop2op 0x04, 0x12, sf = \sf, mx = \mx, v = \v, cv = \cv, lm = \lm
.endm
/* SQR 5 0x4AAn0428 Square of vector */
.macro sqr sf
cop2op 0x0A, 0x28, sf = \sf, lm = 1
.endm
/* OP 6 0x4B7n000C Outer Product */
.macro op sf
cop2op 0x17, 0x0C, sf = \sf
.endm
/* GPF 6 0x4B9n003D General purpose interpolation */
.macro gpf sf
cop2op 0x19, 0x3D, sf = \sf
.endm
/* GPL 5 0x4BAn003E general purpose interpolation */
.macro gpl sf
cop2op 0x1A, 0x3E, sf = \sf
.endm
## Convenience macros
/* rtv0 - 0x4A486012 v0 * rotmatrix */
.macro rtv0
# .word 0x4A486012
mvmva 1, 0, 0, 3, 0
.endm
/* rtv1 - 0x4A48E012 v1 * rotmatrix */
.macro rtv1
# .word 0x4A48E012
mvmva 1, 0, 1, 3, 0
.endm
/* rtv2 - 0x4A496012 v2 * rotmatrix */
.macro rtv2
# .word 0x4A496012
mvmva 1, 0, 2, 3, 0
.endm
/* rtir12 - 0x4A49E012 ir * rotmatrix */
.macro rtir12
# .word 0x4A49E012
mvmva 1, 0, 3, 3, 0
.endm
/* rtir0 - 0x4A41E012 ir * rotmatrix */
.macro rtir0
# .word 0x4A41E012
mvmva 0, 0, 3, 3, 0
.endm
/* rtv0tr - 0x4A480012 v0 * rotmatrix + tr vector */
.macro rtv0tr
# .word 0x4A480012
mvmva 1, 0, 0, 0, 0
.endm
/* rtv1tr - 0x4A488012 v1 * rotmatrix + tr vector */
.macro rtv1tr
# .word 0x4A488012
mvmva 1, 0, 1, 0, 0
.endm
/* rtv2tr - 0x4A490012 v2 * rotmatrix + tr vector */
.macro rtv2tr
# .word 0x4A490012
mvmva 1, 0, 2, 0, 0
.endm
/* rtirtr - 0x4A498012 ir * rotmatrix + tr vector */
.macro rtirtr
# .word 0x4A498012
mvmva 1, 0, 3, 0, 0
.endm
/* rtv0bk - 0x4A482012 v0 * rotmatrix + bk vector */
.macro rtv0bk
# .word 0x4A482012
mvmva 1, 0, 0, 1, 0
.endm
/* rtv1bk - 0x4A48A012 v1 * rotmatrix + bk vector */
.macro rtv1bk
# .word 0x4A48A012
mvmva 1, 0, 1, 1, 0
.endm
/* rtv2bk - 0x4A492012 v2 * rotmatrix + bk vector */
.macro rtv2bk
# .word 0x4A492012
mvmva 1, 0, 2, 1, 0
.endm
/* rtirbk - 0x4A49A012 ir * rotmatrix + bk vector */
.macro rtirbk
# .word 0x4A49A012
mvmva 1, 0, 3, 1, 0
.endm
/* ll - 0x4A4A6412 v0 * light matrix. Lower limit result to 0 */
.macro ll
# .word 0x4A4A6412
mvmva 1, 1, 0, 3, 1
.endm
/* llv0 - 0x4A4A6012 v0 * light matrix */
.macro llv0
# .word 0x4A4A6012
mvmva 1, 1, 0, 3, 0
.endm
/* llv1 - 0x4A4AE012 v1 * light matrix */
.macro llv1
# .word 0x4A4AE012
mvmva 1, 1, 1, 3, 0
.endm
/* llv2 - 0x4A4B6012 v2 * light matrix */
.macro llv2
# .word 0x4A4B6012
mvmva 1, 1, 2, 3, 0
.endm
/* llvir - 0x4A4BE012 ir * light matrix */
.macro llvir
# .word 0x4A4BE012
mvmva 1, 1, 3, 3, 0
.endm
/* llv0tr - 0x4A4A0012 v0 * light matrix + tr vector */
.macro llv0tr
# .word 0x4A4A0012
mvmva 1, 1, 0, 0, 0
.endm
/* llv1tr - 0x4A4A8012 v1 * light matrix + tr vector */
.macro llv1tr
# .word 0x4A4A8012
mvmva 1, 1, 1, 0, 0
.endm
/* llv2tr - 0x4A4B0012 v2 * light matrix + tr vector */
.macro llv2tr
# .word 0x4A4B0012
mvmva 1, 1, 2, 0, 0
.endm
/* llirtr - 0x4A4B8012 ir * light matrix + tr vector */
.macro llirtr
# .word 0x4A4B8012
mvmva 1, 1, 3, 0, 0
.endm
/* llv0bk - 0x4A4A2012 v0 * light matrix + bk vector */
.macro llv0bk
# .word 0x4A4A2012
mvmva 1, 1, 0, 1, 0
.endm
/* llv1bk - 0x4A4AA012 v1 * light matrix + bk vector */
.macro llv1bk
# .word 0x4A4AA012
mvmva 1, 1, 1, 1, 0
.endm
/* llv2bk - 0x4A4B2012 v2 * light matrix + bk vector */
.macro llv2bk
# .word 0x4A4B2012
mvmva 1, 1, 2, 1, 0
.endm
/* llirbk - 0x4A4BA012 ir * light matrix + bk vector */
.macro llirbk
# .word 0x4A4BA012
mvmva 1, 1, 3, 1, 0
.endm
/* lc - 0x4A4DA412 v0 * color matrix, Lower limit clamped to 0 */
.macro lc
# .word 0x4A4DA412
mvmva 1, 2, 3, 1, 1
.endm
/* lcv0 - 0x4A4C6012 v0 * color matrix */
.macro lcv0
# .word 0x4A4C6012
mvmva 1, 2, 0, 3, 0
.endm
/* lcv1 - 0x4A4CE012 v1 * color matrix */
.macro lcv1
# .word 0x4A4CE012
mvmva 1, 2, 1, 3, 0
.endm
/* lcv2 - 0x4A4D6012 v2 * color matrix */
.macro lcv2
# .word 0x4A4D6012
mvmva 1, 2, 2, 3, 0
.endm
/* lcvir - 0x4A4DE012 ir * color matrix */
.macro lcvir
# .word 0x4A4DE012
mvmva 1, 2, 3, 3, 0
.endm
/* lcv0tr - 0x4A4C0012 v0 * color matrix + tr vector */
.macro lcv0tr
# .word 0x4A4C0012
mvmva 1, 2, 0, 0, 0
.endm
/* lcv1tr - 0x4A4C8012 v1 * color matrix + tr vector */
.macro lcv1tr
# .word 0x4A4C8012
mvmva 1, 2, 1, 0, 0
.endm
/* lcv2tr - 0x4A4D0012 v2 * color matrix + tr vector */
.macro lcv2tr
# .word 0x4A4D0012
mvmva 1, 2, 2, 0, 0
.endm
/* lcirtr - 0x4A4D8012 ir * color matrix + tr vector */
.macro lcirtr
# .word 0x4A4D8012
mvmva 1, 2, 3, 0, 0
.endm
/* lev0bk - 0x4A4C2012 v0 * color matrix + bk vector */
.macro lev0bk
# .word 0x4A4C2012
mvmva 1, 2, 0, 1, 0
.endm
/* lev1bk - 0x4A4CA012 v1 * color matrix + bk vector */
.macro lev1bk
# .word 0x4A4CA012
mvmva 1, 2, 1, 1, 0
.endm
/* lev2bk - 0x4A4D2012 v2 * color matrix + bk vector */
.macro lev2bk
# .word 0x4A4D2012
mvmva 1, 2, 2, 1, 0
.endm
/* leirbk - 0x4A4DA012 ir * color matrix + bk vector */
.macro leirbk
# .word 0x4A4DA012
mvmva 1, 2, 3, 1, 0
.endm
/* sqr12 - 0x4AA80428 square of ir 1,19,12 */
# .macro sqr12
# # .word 0x4AA80428
# sqr 1
# .endm
/* sqr0 - 0x4AA00428 square of ir 1,31, 0 */
# .macro sqr0
# # .word 0x4AA00428
# sqr 0
# .endm
/* op12 - 0x4B78000C outer product 1,19,12 */
.macro op12
# .word 0x4B78000C
op 1
.endm
/* op0 - 0x4B70000C outer product 1,31, 0 */
.macro op0
# .word 0x4B70000C
op 0
.endm
/* gpf12 - 0x4B98003D general purpose interpolation 1,19,12 */
.macro gpf12
# .word 0x4B98003D
gpf 1
.endm
/* gpf0 - 0x4B90003D general purpose interpolation 1,31, 0 */
.macro gpf0
# .word 0x4B90003D
gpf 0
.endm
/* gpl12 - 0x4BA8003E general purpose interpolation 1,19,12 */
.macro gpl12
# .word 0x4BA8003E
gpl 1
.endm
/* gpl0 - 0x4BA0003E general purpose interpolation 1,31, 0 */
.macro gpl0
# .word 0x4BA0003E
gpl 0
.endm
.endif
+43
View File
@@ -0,0 +1,43 @@
#ifndef INCLUDE_ASM_H
#define INCLUDE_ASM_H
#if !defined(M2CTX) && !defined(PERMUTER)
#ifndef INCLUDE_ASM
#define INCLUDE_ASM(FOLDER, NAME) \
__asm__( \
".section .text\n" \
" .set noat\n" \
" .set noreorder\n" \
" .include \"" FOLDER "/" #NAME ".s\"\n" \
" .set reorder\n" \
" .set at\n" \
)
#endif
#ifndef INCLUDE_RODATA
#define INCLUDE_RODATA(FOLDER, NAME) \
__asm__( \
".section .rodata\n" \
" .include \"" FOLDER "/" #NAME ".s\"\n" \
".section .text" \
)
#endif
#if INCLUDE_ASM_USE_MACRO_INC
__asm__(".include \"include/macro.inc\"\n");
#else
__asm__(".include \"include/labels.inc\"\n");
#endif
#else
#ifndef INCLUDE_ASM
#define INCLUDE_ASM(FOLDER, NAME)
#endif
#ifndef INCLUDE_RODATA
#define INCLUDE_RODATA(FOLDER, NAME)
#endif
#endif /* !defined(M2CTX) && !defined(PERMUTER) */
#endif /* INCLUDE_ASM_H */
+61
View File
@@ -0,0 +1,61 @@
# This file is used by the original compiler/assembler.
# Defines the expected assembly macros.
.include "gte_macros.inc"
# A function symbol.
.macro glabel label, visibility=global
.\visibility \label
.type \label, @function
\label:
.ent \label
.endm
# The end of a function symbol.
.macro endlabel label
.size \label, . - \label
.end \label
.endm
# An alternative entry to a function.
.macro alabel label, visibility=global
.\visibility \label
.type \label, @function
\label:
.aent \label
.endm
# A label referenced by an error handler table.
.macro ehlabel label, visibility=global
.\visibility \label
\label:
.endm
# A label referenced by a jumptable.
.macro jlabel label, visibility=local
.\visibility \label
\label:
.endm
# A data symbol.
.macro dlabel label, visibility=global
.\visibility \label
.type \label, @object
\label:
.endm
# End of a data symbol.
.macro enddlabel label
.size \label, . - \label
.endm
# Label to signal the symbol haven't been matched yet.
.macro nonmatching label, size=1
.global \label\().NON_MATCHING
.type \label\().NON_MATCHING, @object
.size \label\().NON_MATCHING, \size
\label\().NON_MATCHING:
.endm
+68
View File
@@ -0,0 +1,68 @@
# This file is used by modern gas.
# Defines the expected assembly macros
# Evaluate this file only once in case it's included more than once
.ifndef _MACRO_INC_GUARD
.internal _MACRO_INC_GUARD
.set _MACRO_INC_GUARD, 1
# A function symbol.
.macro glabel label, visibility=global
.\visibility \label
.type \label, @function
\label:
.ent \label
.endm
# The end of a function symbol.
.macro endlabel label
.size \label, . - \label
.end \label
.endm
# An alternative entry to a function.
.macro alabel label, visibility=global
.\visibility \label
.type \label, @function
\label:
.aent \label
.endm
# A label referenced by an error handler table.
.macro ehlabel label, visibility=global
.\visibility \label
\label:
.endm
# A label referenced by a jumptable.
.macro jlabel label, visibility=global
.\visibility \label
\label:
.endm
# A data symbol.
.macro dlabel label, visibility=global
.\visibility \label
.type \label, @object
\label:
.endm
# End of a data symbol.
.macro enddlabel label
.size \label, . - \label
.endm
# Label to signal the symbol haven't been matched yet.
.macro nonmatching label, size=1
.global \label\().NON_MATCHING
.type \label\().NON_MATCHING, @object
.size \label\().NON_MATCHING, \size
\label\().NON_MATCHING:
.endm
.include "gte_macros.inc"
.endif
+31 -13
View File
@@ -49,8 +49,8 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo
`progress.py` loud-fail) — xHigh, R39 controls — see Log 2026-09-06 A2
- [x] **A3** `NO_SDK` knob + `make sdk-dual` (+ tools-health wiring, `[skip]` without SDK dirs) — Max — see Log 2026-09-06 A3
- [x] **A4** `tools/family_hseq.py` regen — Low (became an instrument fix: the map now carries its own coverage) — see Log
- [ ] **B1** `make disc-extract` (extract.py `--expect-manifest` / `--allow-missing-audio`; check-env; `.gitignore`
re-tightened; the 4 splat preset headers tracked; `clean` fixed) — Max
- [x] **B1** `make disc-extract` (extract.py `--expect-manifest` / `--allow-missing-audio`; check-env; `.gitignore`
re-tightened; the 4 splat preset headers tracked; `clean` fixed) — Max — see Log 2026-09-06 B1
- [ ] **B2** 34 absolute includes → `../shared/` + portable-include audit + 15-binary re-gate — xHigh
- [ ] **B3** `tools/bootstrap.sh` / `make bootstrap` + check-env extensions + fresh-clone proof 218/218 — xHigh
- [ ] **B4** `tools/fetch_psyq.sh` + CHECKSUMS rows (20 lib40 LIBs, psyq-obj-parser) BEFORE `tools/psyq/` leaves git — xHigh
@@ -130,15 +130,30 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact
denominator and warns "predates the coverage field" on an old-format map. Controls: current map → 0 `[warn]`; the same map
with the keys removed → the predates warning; restored → 0. No other tool calls `family_hseq.load()` (grep; consumers read
the json's `families` key, unchanged). `docs/family-hseq.md` regenerated (0 families). SETUP: P33 A4 section (R21).
- **2026-09-06 (S86) — B1 (`make disc-extract`).** `tools/bfm_extract/extract.py`: `--expect-manifest` (compare against the
committed oracle, never write it; mismatch → `.run/extract/` + the first 20 diffs, exit 1; refuses an internally
inconsistent oracle) and `--allow-missing-audio` (PARTIAL verdict for Track-1-only dumps, in both extract and `--verify`).
Makefile: `disc-extract` (probe → disc presence → `--verify-disc` → extract+compare → verify), `extract` runs it when
`$(EXE)` is absent, `extract-all` runs it once first, `check-env` step 6 WARN-on-absent + step 7 oracle self-consistency +
disc INFO, `help` rewritten, `clean` keeps the 4 preset headers (now tracked), `.PHONY`. `.gitignore` re-tightened to H1
(see SETUP P33 B1). **Controls:** no disc → exit 2 + staging text; truncated 100 MB Track 1 → `--verify-disc` FAIL, nothing
written, `git diff --exit-code` on the oracle clean; the real run with `extracted/` moved aside and only the two manifest
files restored → `disc-extract: OK` in **15.7 s** (EXIT=0, `.run/P33/b1_disc_extract.log`), `diff -rq` against the previous
tree IDENTICAL for all 1,801 retail files; second run "up to date" in 0.7 s; `make check-env` OK with the new PASS lines.
Slip, recorded: the previous tree also held `extracted/proto/` (the two prototype EXEs from the prototype discs, not the
retail one) and I removed the moved-aside copy in the same command as the diff — regenerated from `disks/` with
`tools/bfm_extract/extract_proto_exe.py` (see the next line).
## 🛑 SESSION CHECKPOINT — A1 ✓ A2 ✓ A3 ✓ A4 ✓ (P6 rules check done after A4); NEXT = B1 `make disc-extract` (Max) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks)
## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1 ✓; NEXT = B2 (the 34 absolute includes + the portable-include audit + 15-binary re-gate, xHigh) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks)
### 0. How to use this block
You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file,
and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest
(R1–R73), then WAIT for Drew. **NEXT = B1** (`make disc-extract`, Max — a build-input step; wrong = a green build on the
wrong bytes). If `git log -1 --format=%s` does not start with `fix(phase-33): A4`, A4's commit did not land: re-run
`.venv/bin/python tools/family_hseq.py` and `tools/audit_binaries.py` (expect 0 `[warn]`) before committing.
(R1–R73), then WAIT for Drew. **NEXT = B2** (xHigh: the 34 absolute `#include "/home/musashi/bfm-decomp/src/shared/…"`
lines in 19 `src/ov_*/…_jr_*.c` files across 15 binaries → `../shared/`; extend `tools/audit_text_sources.py` with a
portability class; controls 34 → 0; re-gate the 15 binaries with `make extract BINARY=<b> && make check BINARY=<b>`).
If `git log -1 --format=%s` does not start with `feat(phase-33): B1`, B1's commit did not land: re-run `make disc-extract`
(expect "up to date" in ~1 s) and `make check-env` (OK) before committing.
### 1. Where we are
**Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at
@@ -207,14 +222,17 @@ items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2.
### 3. NEXT — in order
0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) ·
`.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not).
1. **B1** (Max) per the plan's Block B: `tools/bfm_extract/extract.py` gains `--expect-manifest` + `--allow-missing-audio`;
`make disc-extract`; `extract`/`extract-all`/`check-env`/`help`/`clean` changes; `.gitignore` re-tightened; the 4 splat
preset headers tracked. Gotchas: `extract.py:379-381` overwrites the manifest on every run (compare, never write); the
committed manifest has 1,801 rows incl. 3 `.DA` audio files from Tracks 2–4; `disks/` here holds all 4 tracks + cue; the
verification moves `extracted/` aside (`mv extracted .run/extracted.off`) — put it back if anything fails.
2. Then B2 → B3 → B4 → B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box,
1. **B2** (xHigh): `grep -rl '#include "/home/musashi/bfm-decomp/src/shared/' src | xargs sed -i 's|#include
"/home/musashi/bfm-decomp/src/shared/|#include "../shared/|'`; `grep -rn '/home/musashi' src include` must print nothing;
`tools/audit_text_sources.py` gains the portability class (absolute, `<…>`, outside-repo includes are offenders; run it
BEFORE the sed → exactly 34 offenders in 19 files, AFTER → 0); re-gate the 15 binaries: `for b in ov_SC02_000 ov_SC02_011
ov_SC02_016 ov_SC02_017 ov_SC02_026 ov_SC02_039 ov_SC03_001 ov_SC03_006 ov_SC03_091 ov_SC03_105 ov_SC04_018 ov_SC05_003
ov_SC05_010 ov_SC06_000 ov_SC06_029; do make extract BINARY=$b && make -j8 check BINARY=$b || exit 1; done` (derive the
list from the grep, do not trust this one). Commit "feat(phase-33): B2 …".
2. Then B3 → B4 → B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box,
add a Log line, refresh this checkpoint block (the 🛑 block is the ONLY in-phase context the next session inherits),
commit. Next P6 rules check after B4 (8 tasks done).
commit. Next P6 rules check after B4 (8 tasks done). `extracted/proto/` (sep8 + aug31 EXEs) is regenerated from the
prototype discs by `tools/bfm_extract/extract_proto_exe.py` (docstring examples) — B5 needs both files.
### 4. Files this session touched
A1: `phase-ends/DIGEST.md`, `phase-ends/CURRENT_PHASE.md`. A2: `tools/main_seed_ends.py` (new), `Makefile` (`sig-main`
+105 -8
View File
@@ -29,6 +29,7 @@ from __future__ import annotations
import argparse
import hashlib
import json
import sys
from dataclasses import dataclass, field
from pathlib import Path
@@ -348,13 +349,47 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
action="store_true",
help="Re-hash the output tree against its manifest and exit (no extraction).",
)
p.add_argument(
"--expect-manifest",
type=Path,
default=None,
help=(
"P33 B1 (the public `make disc-extract`): compare the extraction against this committed "
"manifest.jsonl (the oracle) INSTEAD of writing one. Identical -> nothing written, exit 0; "
"different -> the actual manifest goes to .run/extract/, the first differences are listed, "
"exit 1. The committed oracle is never overwritten by a build step."
),
)
p.add_argument(
"--allow-missing-audio",
action="store_true",
help=(
"With no sibling `(Track 2).bin` staged, compare/verify WITHOUT the 3 .DA audio rows and "
"report PARTIAL instead of failing (a Track-1-only dump). Without this flag a partial "
"extraction against the 4-track oracle FAILS."
),
)
return p.parse_args(argv)
def cmd_verify(out_root: Path) -> int:
def _is_audio_row(path: str) -> bool:
return path.upper().endswith(".DA")
def cmd_verify(out_root: Path, allow_missing_audio: bool = False) -> int:
ok, problems = manifest.verify(out_root)
partial = []
if allow_missing_audio:
partial = [p for p in problems if p.startswith("missing file: ") and _is_audio_row(p)]
problems = [p for p in problems if p not in partial]
ok = not problems
if ok:
print(f"RESULT: PASS - every artifact under {out_root}/ matches {manifest.MANIFEST_NAME}.")
if partial:
print(f"RESULT: PASS (PARTIAL) - every artifact under {out_root}/ matches "
f"{manifest.MANIFEST_NAME} except {len(partial)} unstaged .DA audio file(s): "
+ ", ".join(p.split(': ', 1)[1] for p in partial))
else:
print(f"RESULT: PASS - every artifact under {out_root}/ matches {manifest.MANIFEST_NAME}.")
return 0
print(f"RESULT: FAIL - {len(problems)} problem(s):", file=sys.stderr)
for p in problems[:20]:
@@ -362,10 +397,66 @@ def cmd_verify(out_root: Path) -> int:
return 1
ACTUAL_DIR = Path(".run") / "extract" # where a NON-matching extraction's manifest is written (R12)
def compare_with_oracle(records: list[dict], oracle: Path, allow_missing_audio: bool) -> int:
"""Compare the freshly built records against the committed oracle manifest.jsonl.
Never writes into the oracle's directory. On a match prints the oracle's own SHA1 (the single
value a contributor quotes to prove their extraction). On a mismatch writes the ACTUAL manifest
to .run/extract/ and prints the first 20 differences (missing / extra / mismatched paths)."""
if not oracle.is_file():
print(f"ERROR: oracle manifest not found: {oracle}", file=sys.stderr)
return 2
text = oracle.read_text(encoding="ascii")
oracle_sha1 = hashlib.sha1(text.encode("ascii")).hexdigest()
sha_file = oracle.with_name(manifest.MANIFEST_SHA1_NAME)
if sha_file.is_file():
recorded = sha_file.read_text(encoding="ascii").strip()
if recorded != oracle_sha1:
print(f"ERROR: {sha_file} records {recorded} but {oracle.name} hashes to {oracle_sha1} — "
f"the committed oracle is internally inconsistent; refusing to compare", file=sys.stderr)
return 2
expected = {}
for line in text.splitlines():
if line.strip():
r = json.loads(line)
expected[r["path"]] = (r["size"], r["sha1"])
actual = {r["path"]: (r["size"], r["sha1"]) for r in records}
partial = []
if allow_missing_audio:
partial = sorted(p for p in expected if _is_audio_row(p) and p not in actual)
for p in partial:
del expected[p]
missing = sorted(set(expected) - set(actual))
extra = sorted(set(actual) - set(expected))
mismatch = sorted(p for p in expected if p in actual and expected[p] != actual[p])
if not (missing or extra or mismatch):
note = (f" PARTIAL: {len(partial)} .DA audio row(s) unverified ({', '.join(partial)})"
if partial else "")
print(f"\nManifest: {len(actual)} artifacts == the committed oracle {oracle} "
f"(sha1 {oracle_sha1}); nothing written.{note}")
return 0
ACTUAL_DIR.mkdir(parents=True, exist_ok=True)
digest = manifest.write(ACTUAL_DIR, records)
print(f"\nManifest MISMATCH against the oracle {oracle} (sha1 {oracle_sha1}): "
f"{len(missing)} missing, {len(extra)} extra, {len(mismatch)} mismatched — "
f"actual manifest written to {ACTUAL_DIR}/ (sha1 {digest})", file=sys.stderr)
shown = 0
for label, paths in (("missing", missing), ("extra", extra), ("mismatch", mismatch)):
for p in paths:
if shown >= 20:
break
print(f" - {label}: {p}", file=sys.stderr)
shown += 1
return 1
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
if args.verify:
return cmd_verify(args.out)
return cmd_verify(args.out, args.allow_missing_audio)
try:
with Iso9660Image(args.bin) as img:
if args.list:
@@ -377,11 +468,17 @@ def main(argv: list[str] | None = None) -> int:
return 2
records = manifest.build(args.out)
digest = manifest.write(args.out, records)
print(
f"\nManifest: {len(records)} artifacts -> "
f"{args.out}/{manifest.MANIFEST_NAME} (sha1 {digest})"
)
if args.expect_manifest:
rc = compare_with_oracle(records, args.expect_manifest,
allow_missing_audio=args.allow_missing_audio and not audio_tracks)
if rc:
return rc
else:
digest = manifest.write(args.out, records)
print(
f"\nManifest: {len(records)} artifacts -> "
f"{args.out}/{manifest.MANIFEST_NAME} (sha1 {digest})"
)
if check_exe_roundtrip(exe_data):
print("\nRESULT: PASS - full disc extracted; EXE reproduces the known-good binary.")