From 95c142d2ae9f81f9e0ff78d298813aed90b7c0e4 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:12:08 -0600 Subject: [PATCH] =?UTF-8?q?docs+tools(phase-33):=20E1=20=E2=80=94=20docs/d?= =?UTF-8?q?ecompme-preset.md=20+=20tools/decompme=5Freplica.sh:=20the=20de?= =?UTF-8?q?comp.me=20preset=20proven=20through=20decomp.me's=20OWN=20toolc?= =?UTF-8?q?hain=20locally=20(old-gcc=200.13=20+=20maspsx=2086ccd7d8=20rebu?= =?UTF-8?q?ilt=20from=20decompme/compilers'=20Dockerfile;=20func=5F80018F2?= =?UTF-8?q?0=2026/26=20words=20BYTE-IDENTICAL,=20control=20identical,=20cc?= =?UTF-8?q?1/maspsx=20deltas=20text-identical;=20-O1=20and=20empty-TU=20ne?= =?UTF-8?q?gative=20controls;=20--upstream=20drift=20check);=20SETUP's=20s?= =?UTF-8?q?tale=20'decomp.me=20uses=20our=20maspsx=20pin'=20corrected=20(3?= =?UTF-8?q?=20rows),=20=C2=A76.5=20rewritten,=20E1=20section=20+=20rows,?= =?UTF-8?q?=20ledger=2014;=20presets=20are=20created=20by=20decomp.me's=20?= =?UTF-8?q?maintainers=20from=20an=20issue=20template=20(no=20UI=20create,?= =?UTF-8?q?=20no=20owner=20delete)=20=E2=80=94=20Drew's=20post-flip=20step?= =?UTF-8?q?=20=3D=20prove=20the=20scratch,=20file=20the=20request,=20the?= =?UTF-8?q?=20manual=20search;=20doc=5Flinks=20default;=20.run/P33/e1=20lo?= =?UTF-8?q?g=20allowlisted;=20log=20+=20checkpoint=20(35=20of=2041)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 + .run/P33/e1/decompme_replica.log | 9 ++ docs/SETUP.md | 49 ++++++++-- docs/decompme-preset.md | 151 +++++++++++++++++++++++++++++ phase-ends/CURRENT_PHASE.md | 42 ++++++--- tools/decompme_replica.sh | 157 +++++++++++++++++++++++++++++++ tools/doc_links.py | 2 +- 7 files changed, 394 insertions(+), 21 deletions(-) create mode 100644 .run/P33/e1/decompme_replica.log create mode 100644 docs/decompme-preset.md create mode 100644 tools/decompme_replica.sh diff --git a/.gitignore b/.gitignore index 2a4a0c401..c272a1141 100644 --- a/.gitignore +++ b/.gitignore @@ -335,6 +335,11 @@ unsloth_compiled_cache/ !/.run/P33/verify/ /.run/P33/verify/* !/.run/P33/verify/*.log +# P33 E1 (S89, 2026-09-07): the decomp.me toolchain-replica verdict log (tools/decompme_replica.sh) — the log only; the +# regenerated target .s (game disassembly, H1), the probe objects and the old-gcc tarball under .run/P33/e1/ stay ignored +!/.run/P33/e1/ +/.run/P33/e1/* +!/.run/P33/e1/decompme_replica.log !/.run/P33/verify/*.map !/.run/P33/verify/*.md !/.run/P33/verify/*.txt diff --git a/.run/P33/e1/decompme_replica.log b/.run/P33/e1/decompme_replica.log new file mode 100644 index 000000000..fe803797f --- /dev/null +++ b/.run/P33/e1/decompme_replica.log @@ -0,0 +1,9 @@ +target: main:func_80018F20 — 26 words at 0x80018F20 (tools/verbatim_target_s.py, from the image) +A. decomp.me replica — old-gcc 0.13 driver + maspsx 86ccd7d8 as-wrapper; flags: -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div + decomp.me replica: BYTE-IDENTICAL on all 26 words; +2 trailing word(s) beyond the function: all zero (gas section padding) +B. our pipeline — old-gcc 0.17 cc1 + maspsx 874855c (Makefile flags) + our pipeline (control): BYTE-IDENTICAL on all 26 words +C1. cc1 0.13 vs 0.17 on the same input: identical assembly text +C2. maspsx 86ccd7d8 vs 874855c on the same cc1 output: identical text +scratch: /home/musashi/bfm-decomp/.run/decompme/probe_func_80018F20 +decompme_replica: PASS — the preset reproduces func_80018F20 byte-identically through decomp.me's toolchain diff --git a/docs/SETUP.md b/docs/SETUP.md index 6c60d61ee..127b4273d 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -20,7 +20,7 @@ All components run inside WSL2 Ubuntu 24.04 (single ext4 clone). The "Side" colu | WSL2 distro | Ubuntu-24.04 (Python 3.12 ships with it) | WSL2 (Linux) | 4 | | splat | pip `splat64[mips]` `>=0.41.0,<1.0.0` — **as-built 0.41.0** (Phase 4); freeze exact once Phase 5 green | WSL2 (Linux) | 4 | | Vintage compiler | decompals/old-gcc **release 0.17**: `gcc-2.7.2-psx` + `gcc-2.7.2-cdk` | WSL2 (Linux) | 4 | -| maspsx | git submodule, `mkst/maspsx` (decomp.me pins commit `874855c53f65f8fa57447e1da6bde6236dbef9d5` — reasonable default pin) | WSL2 (Linux) | 4 | +| maspsx | git submodule, `mkst/maspsx` at `874855c53f65f8fa57447e1da6bde6236dbef9d5` (decomp.me's pin when we adopted it, June 2026; decomp.me moved to `86ccd7d8` on 2026-08-29 — 4 commits later, behaviour identical for `--aspsx-version` ≥ 2.30; see `docs/decompme-preset.md` §3) | WSL2 (Linux) | 4 | | asm-differ / m2c / decomp-permuter | git submodules (URLs in §4.6) | WSL2 (Linux) | 4 | | binutils (mipsel) | apt `binutils-mipsel-linux-gnu` — **as-built 2.42** (Phase 4; ≥2.38 → check-env WARN, verdict deferred to Phase 5), 2.35 known-good | WSL2 (Linux) | 4 | @@ -296,7 +296,7 @@ Submodules (add under `tools/`): | Submodule | URL | Pin | |---|---|---| -| `tools/maspsx` | `https://github.com/mkst/maspsx.git` | commit `874855c53f65f8fa57447e1da6bde6236dbef9d5` (decomp.me's pin — keeps local results comparable to decomp.me scratches) | +| `tools/maspsx` | `https://github.com/mkst/maspsx.git` | commit `874855c53f65f8fa57447e1da6bde6236dbef9d5` (decomp.me's pin in June 2026; decomp.me now runs `86ccd7d8` — measured byte-equivalent for our aspsx version, `docs/decompme-preset.md` §3; `tools/decompme_replica.sh --upstream` reports drift) | | `tools/asm-differ` | `https://github.com/simonlindholm/asm-differ.git` | pin current HEAD at adoption | | `tools/m2c` | `https://github.com/matt-kempster/m2c.git` | pin current HEAD at adoption | | `tools/decomp-permuter` | `https://github.com/simonlindholm/decomp-permuter` | sotn pins `b44b0622269fb4bff29e79fbbad26b9f47beda79` — sane default | @@ -514,13 +514,19 @@ Modern cpp preprocesses → **vintage cc1** compiles to asm → **maspsx** emula ### §6.5 decomp.me settings for BFM -- Platform: **PlayStation**; Compiler: **`gcc2.7.2-psx`** (or `gcc2.7.2-cdk`) **+ maspsx** family — these images bundle old-gcc cc1 + maspsx at the same pinned commit we use. -- Starting flags: `-O2 -G0` (adjust per §5). +- Platform: **PlayStation** (`ps1`); Compiler: **`gcc2.7.2-psx`**. **The project's preset — `docs/decompme-preset.md` (P33 E1):** + flags `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div`; name + `Brave Fencer Musashi (SLUS-00726)`; **requested from decomp.me's maintainers via their GitHub issue template** (there is no + create button in the UI and no owner delete) by Drew after the flip, with a proving scratch attached — proven before it is + requested (`docs/decompme-preset.md` §5 carries the ready-to-paste issue). +- **decomp.me does NOT run our binaries** (measured 2026-09-07 from `decompme/compilers`): its image is old-gcc **0.13** + + maspsx **`86ccd7d8`** with `as` = a maspsx `--run-assembler` wrapper (so `-Wa,` args reach maspsx); we run old-gcc 0.17 + + maspsx `874855c5`. Both deltas measured text-identical on the probe; the maspsx delta is gated on aspsx < 2.30 anyway. + **`tools/decompme_replica.sh`** rebuilds decomp.me's toolchain under `.run/decompme/` and proves a function through it + locally (PASS on `func_80018F20`, 26/26 words); `--upstream` reports when decomp.me's pins drift. - **Do NOT use the SOTN preset** (`Castlevania: Symphony of the Night` / `gcc 2.6.3-psx` / `psyq_263_221`) — wrong era, guaranteed near-miss diffs. -- decomp.me's API is Cloudflare-challenged (403 to scripts) — scratch searches/uploads needing the API must be done manually in a browser. -- **The project's preset (P33 E1, `docs/decompme-preset.md`):** platform `ps1`, compiler `gcc2.7.2-psx`, flags - `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div` (decomp.me's image wraps - `as` with maspsx and forwards `-Wa,` args). Presets are created in the browser by a logged-in user (Drew, after the flip). +- decomp.me's API is Cloudflare-challenged (403 to scripts) — scratch searches/uploads needing the API must be done manually in a browser + (the one-time manual BFM search is ledger row 14, closed by the E1 browser session). ### §6.6 Matching a function (INCLUDE_ASM → C; the NON_MATCHING guard) — As-built Phase 6 @@ -769,6 +775,8 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/permuter/upstream/0001-reloc-masked-scorer.patch` | **(P33 E5)** The upstream PR as a `git format-patch` (one commit against `simonlindholm/decomp-permuter` main `41bd0bfc`, 2026-09-05): `src/reloc_scorer.py` (`RelocMaskedScorer`, a `Scorer` subclass), `--score-mode {mnemonic,reloc-masked}` + the `score_mode` settings key, docs, `test/test_reloc_scorer.py` (10, no cross toolchain). Regenerate the branch: clone upstream, `git am` the patch (proven clean). Two commits (the scorer; a USAGE note on restarting from the best candidate), plain contributor style. Submitted 2026-09-07: PR simonlindholm/decomp-permuter#213 (from the fork's `reloc-masked-scorer`), issue #214 (the symbol-regex proposal; `docs/permuter-ils.md` §3). | | | `docs/matching-drafter-pipeline.md` | **(P33 E6)** The write-up of the fine-tuned local matching drafter (Phases 22–25): the pipeline (`export_pairs` → `format_finetune` → `train_lora` → `eval_lora` → `serve_local` → `api_draft` / `lora_grind` / `bulk_harvest` → the byte gate; `ab_score` + `workflows/ab_match.js`; `grinder`), every measurement in order (stock floor ~0 → v2 85% on 6–15 ins → v3 57.5% held-out and ~352 production banks → v4 negative → the GLM hard-band A/B and the def-side wall), the portable lessons, the hardware, a five-step recipe, and what is NOT published (the ROM-derived pair dataset `datasets/`, the adapter weights `models/` — both gitignored). | | | `docs/gen3-handoff.md` | **(P33 G1)** The Gen3 seed: where Gen2 ends, the owner's next intent (casts → structs, pins off, names), the starter census with the commands that re-derive it (143 raw address casts · 61,898 `D_` · 16,335 `func_` · 44,243 register pins · 1,083 symbol-file entries · 1,232 struct defs · 2,220 dedup groups · 5 verbatim bodies), the one invariant (every edit byte-gated; shared bodies change every member; types are comprehension not bytes), the inherited levers, shiftability honestly scoped (position-locked slots, LZSS not byte-stable, the Sony regions), the parked ideas, and the governance for a new generation. | +| | `tools/decompme_replica.sh` | **(P33 E1)** Runs a function through decomp.me's EXACT PS1 `gcc2.7.2-psx` toolchain locally — old-gcc 0.13 (tarball sha256-checked into `.run/decompme/`) + maspsx `86ccd7d8` (from the submodule's object store) behind the image's two-line `as` wrapper, driven by the backend's own two commands — and compares the code words to the ROM-derived target (`verbatim_target_s.py`); our Makefile pipeline on the same TU is the control (exit 3 if IT fails), then cc1 and maspsx are swapped one at a time so a difference names its producer. Default probe `func_80018F20`; `--src TU --fn NAME [--binary B] [--flags "…"]`; `--upstream` compares decomp.me's current Dockerfile pins with the script's constants. Trailing gas padding is reported, never counted. Exit 0 = the preset reproduces the function byte-identically on decomp.me's toolchain. | +| | `docs/decompme-preset.md` | **(P33 E1)** The decomp.me preset (fields, name, flags), why each flag, the measured decomp.me-vs-ours toolchain table (0.13 vs 0.17 cc1; maspsx `86ccd7d8` vs `874855c5` — 4 commits, both behavioural ones gated on aspsx < 2.30), the recorded `decompme_replica.sh` PASS + its two negative controls, and Drew's post-flip browser session (scratch → 100% → the preset-request issue on decompme/decomp.me → the manual search closing ledger row 14). | | | `tools/objdiff_report.py [--in docs/progress.json] [--out report.json]` | **(P33 D3)** progress.json → objdiff's report format (report.proto v2, snake_case — validated with `objdiff-cli` 3.8.1 `report changes`): one unit per binary (code = instructions × 4, functions byte-identical / matchable, metadata complete), categories `game-code` and `linked-sony-objects` (functions only). `.github/workflows/progress.yml` runs it on every push (no rebuild — the committed JSON) and uploads the artifact **`SLUS_007.26_report`** for decomp.dev (Drew registers at decomp.dev/manage/new after the flip). | | | `tools/frogress_upload.py [--push --project bfm --version us]` | **(P33 D3)** stdlib; `--dry-run` is the default (prints the payload); `--push` POSTs `{"api_key","entries":[{git_hash,timestamp,categories:{default:{measures…}}}]}` to `progress.deco.mp/data///` with `FROGRESS_API_SECRET` from the environment (never a file). frogress projects are admin-created — Drew requests the slug + key after the flip. | | | `tools/public_rewrite/` (P33 C1) | **The history-rewrite package** (`docs/public-flip-runbook.md` §3 is the operating table). `common.py` (shared: the purge rules, the DERIVED content-hash sets, identities from the log, the one hash regex, a persistent `cat-file --batch`) · `hash_dict.py [--write-mailmap]` (every commit OBJECT → `commit:NNNN` / twin / orphan; prefix index 7..40; asserts 0 ambiguous; records content-hash collisions as excluded; writes the scratch mailmap) · `scrub.py --test \| --sample \| --file` (THE scrub: hash tokens, addresses → noreply, trailer lines in messages; 12 known-true cases; the HEAD sample with git's own object lookup as the independent oracle) · `gate_scan.py --all\|--refs … [--worktree] [--expect-fail FIXTURE]` (paths ever touched × purge rules; every reachable blob's content sha1 × the ROM set; 5 byte signatures; 50 MiB; emits `rom_blob_ids.txt` = hits ∪ every blob ever under a purge path; the fixture `expected_offenders.txt` is the R39 negative control) · `run_filter.py [--sample]` (the git-filter-repo 2.47.0 module-API run inside the scratch bare clone; refuses elsewhere) · `verify_rewrite.py --old --new` (the pairwise proof) · `build_commit_map.py [--out]` (`docs/commit-map.tsv`, asserted free of old hashes) · `resolve_tokens.py [--check] [--map]` (tokens → shortest unique ≥9-char new abbreviations at the tip) · `absent_scan.py [--repo] [--tree]` (nothing old anywhere) · `probe_github.sh [--after-flip]` (Drew's purge probe). Scratch (`.run/public_rewrite/`, never committed): `dict.json`, `mailmap`, `rom_blob_ids.txt`, `old-to-new.tsv`, `repo.git`, the bundle. · `probe_github.sh [--after-flip]` (Drew's daily post-purge probe, C10: 33 sampled old shas via `gh api` + a fetch; **S88, R57:** the fetch runs in a throwaway bare repo under `.run/public_rewrite/` with `--filter=blob:none --depth=1`, never in the working repo — a successful fetch of an old sha imports its purged closure, which the S87/S88 runs did (5.97 GiB unreachable) — and it ends with a self-check naming any sampled old commit the working repo still holds + the gc recipe) | @@ -986,7 +994,7 @@ A Track-1 match proves the dump is the canonical redump dump, which transitively | 11 | Canonical git remote URL (off-box push/pull backup) | **CLOSED (P33 C9, 2026-09-07):** `https://github.com/Druthulu/BFM-decomp.git`, public from C10; the pre-rewrite history in the private archive `Druthulu/BFM-decomp-archive` (§4.3) | | 12 | Per-libnum stamp detail (raw-track scan reported 16 hits vs 12 genuine in extracted EXE — extracted-EXE scan is ground truth, see §5.1) | **RESOLVED 2026-06-13** — DetectPsyQ at headless import recorded `PsyQ Version = 4.0.0` (§2.5 step 3) | | 13 | Overlay load addresses (resident 0x800CDF58 / location 0x80128508, EXE ptr table ~0x62620) | **JP-only — re-derive for US** (owned by docs/memory-map.md) | -| 14 | Greenfield claim: decomp.me scratch search is script-blocked (Cloudflare) | **TBD → closes at P33 E1:** the preset-creation session (Drew, in the browser, after the flip) includes the one-time manual search for BFM scratches; record the result here | +| 14 | Greenfield claim: decomp.me scratch search is script-blocked (Cloudflare) | **Open until the E1 browser session (Drew, after the flip; `docs/decompme-preset.md` §5 step 4):** the one-time manual search for BFM scratches — record the result here. The preset itself is already PROVEN locally (2026-09-07, `tools/decompme_replica.sh` PASS through decomp.me's 0.13 + `86ccd7d8` toolchain). | ### `tools/gap_triage.py` — harvest pre-filter (added P31 S55) @@ -1148,6 +1156,29 @@ fills fast). Nothing is leaking — but the host does not get the memory back on `tools/ghidra_*.sh` are repo-relative (`BFM_GHIDRA_PROJ` overrides the project dir; `ghidra_mcp_verify.sh [PROG]`); Makefile `GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra)`. +### P33 E1 (S89, 2026-09-07) — the decomp.me preset, proven through decomp.me's own toolchain before it exists +- `docs/decompme-preset.md` (NEW; in `doc_links`' default set). The preset: platform `ps1`, `gcc2.7.2-psx`, + `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div`, name + `Brave Fencer Musashi (SLUS-00726)`; **no create-preset UI exists** (the frontend only reads presets) — decomp.me's + maintainers create them from the `create-or-update-a-compiler-preset.md` issue template; `name`/`platform` immutable and no + owner delete (`views/preset.py`: 405) → proven before it is requested. +- **Measured (R14) — the stale claim "decomp.me bundles maspsx at the same pinned commit we use" (§6.5, rows for maspsx) + was wrong:** decomp.me's image (`decompme/compilers` `platforms/ps1/gcc2.7.2-psx/Dockerfile` + `values.yaml`) is old-gcc + **0.13** (sha256 `aca64479…ffbc`) + maspsx **`86ccd7d8`** (2026-08-29) with `as` = `python3 maspsx.py --run-assembler + -I${COMPILER_DIR} "$@"`; the backend (`coreapp/compilers.py` `PS1_GCC`) runs `/usr/bin/cpp -nostdinc | sed` then + `gcc ${FLAGS} -c -pipe -B${COMPILER_DIR}/`. Our pin `874855c5` is 4 commits behind; the two behavioural commits (lw,lw and + lwl/lwr nop handling) are gated on aspsx < 2.30. old-gcc 0.13→0.17 changed build scripts and the cdk patches only + (`compare/0.13...0.17`). The three §6.5/maspsx rows were corrected in place. +- **`tools/decompme_replica.sh`** (NEW): the replica + control + attribution matrix described in its inventory row. Recorded run + (`.run/P33/e1/decompme_replica.log`): **decomp.me replica BYTE-IDENTICAL on all 26 words of `func_80018F20`** (+2 trailing + zero words = gas padding, reported not counted) · our pipeline BYTE-IDENTICAL (control) · C1 cc1 0.13 vs 0.17 identical + text · C2 maspsx 86ccd7d8 vs 874855c5 identical text · `PASS`. Controls (R39): `-O1` flags → FAIL listing the words (rc 1); + an empty-body TU → the CONTROL fails first (rc 3, R56). `--upstream` 2026-09-07: `pins UNCHANGED`. Instrument lesson (R40): + the first comparison read the padded `.text` and called the identical object DIFFERS. +- `.gitignore`: `.run/P33/e1/` allowlisted for the verdict log only (the regenerated target `.s` is game disassembly — H1). +- Pending (Drew, after the flip — `docs/decompme-preset.md` §5): create the scratch, see 100%, file the preset-request issue + (text in §5), run the manual search, record the issue/preset URL + search result here and in ledger row 14. + ### P33 G1 (S88, 2026-09-07) — `docs/gen3-handoff.md` - The census was DERIVED from the tree at writing (R33/R41; the commands are in the doc): 143 raw address casts, 61,898 distinct `D_80xxxxxx`, 16,335 distinct `func_80xxxxxx` — the plan's three numbers reproduced exactly — plus 44,243 diff --git a/docs/decompme-preset.md b/docs/decompme-preset.md new file mode 100644 index 000000000..8baaa0548 --- /dev/null +++ b/docs/decompme-preset.md @@ -0,0 +1,151 @@ +# The decomp.me preset for Brave Fencer Musashi (SLUS-00726) + +> **P33 E1 (2026-09-07).** This page carries the project's decomp.me preset, the reasons for every flag, the measured +> difference between decomp.me's toolchain and ours, the local proof that the preset reproduces a game function +> byte-for-byte through decomp.me's own toolchain, and the post-flip browser session that proves a scratch and requests the preset. +> Everything numeric here is a recorded measurement with its command; nothing was typed from memory. + +## 1. The preset + +| Field (decomp.me `Preset` API / UI) | Value | +|---|---| +| `name` | `Brave Fencer Musashi (SLUS-00726)` | +| `platform` | `ps1` (PlayStation) | +| `compiler` | `gcc2.7.2-psx` | +| `compiler_flags` | `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div` | +| `assembler_flags` | (empty — decomp.me assembles the pasted target with `mips-linux-gnu-as -EL -march=r3000 -mabi=32` plus its PS1 prelude; the words compare regardless) | +| `diff_flags`, `decompiler_flags`, `libraries` | defaults (none) | + +**How a preset comes to exist on decomp.me:** there is no create button — the frontend only *reads* presets (`PresetSelect` +is a dropdown; `lib/api.ts` has `usePresets`/`usePreset` and no create call). The FAQ's answer to "Can you add a preset for a +game I'm working on?" is the GitHub issue template `.github/ISSUE_TEMPLATE/create-or-update-a-compiler-preset.md` on +`decompme/decomp.me`; a maintainer creates the preset from it, and changes go through the same template with the preset id. +`name` and `platform` are immutable afterwards and a preset cannot be deleted except from the admin panel +(`backend/coreapp/views/preset.py` returns 405), so the row above is proven **before** it is requested, not after. (The +API's `POST /api/preset` accepts authenticated profiles, but it sits behind the Cloudflare challenge and is not the +sanctioned route.) + +**Do not use the SOTN preset** (`gcc 2.6.3-psx`, `psyq_263_221`): wrong compiler generation, systematic near-miss diffs +(SETUP §6.5). Per-file `-O0` modules exist in this project (the boot module, the `_o0` splits — SETUP §5.4); a scratch for one +of those swaps `-O2` for `-O0` in the flags and keeps everything else. + +## 2. Why these flags + +The string is the pinned triple of SETUP §5.4 / the Makefile (`CC1FLAGS`, `ASPSX_VERSION`, `MASPSX_FLAGS`), spelled for +decomp.me's `gcc` driver: + +| Flag | Role | Evidence | +|---|---|---| +| `-O2 -G0` | optimisation level; no `$gp`-relative small data (the EXE header's `gp_value` is 0, zero `%gp_rel` relocs) | SETUP §5.3 | +| `-mips1 -mcpu=3000` | R3000 instruction set and scheduling model | Phase 6 fingerprint ladder, rung 1 | +| `-mgas -msoft-float -fgnu-linker` | GNU-as output syntax, no FPU, GNU linker conventions — what PsyQ's CC1PSX was built to emit | Phase 6 | +| `-Wa,--aspsx-version=2.56,--expand-div` | the assembler side of the triple, forwarded to maspsx | see below | + +The `-Wa,` part works on decomp.me for a structural reason, not by luck: its `gcc2.7.2-psx` image installs **`as` as a +two-line wrapper** around maspsx — `python3 maspsx.py --run-assembler -I${COMPILER_DIR} "$@"` (decompme/compilers +`platforms/ps1/gcc2.7.2-psx/Dockerfile`, generated from `templates/ps1/maspsx.j2`). The gcc 2.7.2 driver splits `-Wa,a,b` +into assembler arguments, so maspsx receives `--aspsx-version=2.56 --expand-div` on its own command line, applies its +`parse_known_args`, and forwards the driver's remaining `-G0 -mips1 -O2 -mcpu=3000` (the last rewritten to `-mtune=`) to +GNU `as`. Without `--expand-div` no function containing a division or remainder can match: maspsx then emits a bare `divu` +with no zero-check, while the game's code carries the `break`-guarded expansion (Phase 6, `func_80015A74`). `2.56` is the +ASPSX behaviour tier the game code fingerprints to; `2.56 ≈ 2.67` on non-discriminating functions (SETUP §5.4). + +## 3. decomp.me's toolchain versus ours — measured, not assumed + +decomp.me does **not** run our exact binaries. Read from its build recipes on 2026-09-07: + +| Component | decomp.me (`decompme/compilers` main) | This repo | Delta | +|---|---|---|---| +| cc1 (`gcc-2.7.2-psx`) | decompals/old-gcc **release 0.13** (tarball sha256 `aca64479…ffbc`; i386 static) | release **0.17** (`tools/bin/`, sha256 in `tools/bin/CHECKSUMS.sha256`) | the two `cc1` binaries differ byte-wise; old-gcc's 0.13→0.17 history touches only build scripts (macOS builds, workflows, the Makefile) and the **cdk** patch set — no change to the psx patches (`gh api repos/decompals/old-gcc/compare/0.13...0.17`) | +| maspsx | commit **`86ccd7d8`** (2026-08-29; `values.yaml` `maspsx_hash`) | commit `874855c5` (the pin decomp.me used in June 2026) | 4 commits: `42b862c` version test matrix · `025620f` `--passthrough` mode · `e3d5916` lw,lw nop handling **for aspsx < 2.30** · `86ccd7d8` lwl/lwr nop fix **for aspsx < 2.30** — nothing reaches `--aspsx-version=2.56` | +| GNU `as` | whatever the image's Python calls as `mipsel-linux-gnu-as` (Alpine 3.18); no `-no-pad-sections` | binutils 2.42 `-march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0` | `.text` is padded to 16 bytes on decomp.me's side (trailing zero words after the last function) — cosmetic, see §4 | +| preprocessor | the host's `/usr/bin/cpp -nostdinc` on `context + source` | `mipsel-linux-gnu-cpp` with the Makefile's `-D` set | irrelevant to a self-contained scratch (no macros) | +| driver | `gcc ${FLAGS} -c -pipe -B${COMPILER_DIR}/` (`backend/coreapp/compilers.py`, `PS1_GCC`) | `cpp \| cc1 \| maspsx \| as` by hand (Makefile) | the driver adds `-mfp32 -mgp32` to cc1 — the mips1 defaults | + +The two differences that could matter (a different cc1 build, a newer maspsx) were **measured to produce identical text** +on the probe (§4, lines C1/C2). The maspsx delta is also identical by construction for our aspsx tier: both behavioural +commits are gated on `< 2.30`. + +## 4. The local proof: `tools/decompme_replica.sh` + +decomp.me's API is Cloudflare-blocked to scripts (SETUP §6.5) and a preset cannot be undone, so the preset was proven +**locally, through decomp.me's own toolchain**, before anyone opened a browser. `tools/decompme_replica.sh` rebuilds the +image's compiler directory under `.run/decompme/` (the 0.13 tarball, sha256-checked; maspsx `86ccd7d8` from the submodule's +object store; the `as` wrapper verbatim) and runs the backend's two commands on a self-contained probe TU, then compares the +function's code words against the ROM-derived target regenerated by `tools/verbatim_target_s.py` (from the image, never from +our source — R34). Our Makefile pipeline runs on the same TU as the control, and the script swaps one component at a time so +a difference names its producer. + +The recorded run (`.run/P33/e1/decompme_replica.log`, 2026-09-07), probe = `func_80018F20`, the Phase-6 fingerprint function +(26 instructions, the `sltiu` range-check idiom, no division): + +``` +target: main:func_80018F20 — 26 words at 0x80018F20 (tools/verbatim_target_s.py, from the image) +A. decomp.me replica — old-gcc 0.13 driver + maspsx 86ccd7d8 as-wrapper; flags: -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div + decomp.me replica: BYTE-IDENTICAL on all 26 words; +2 trailing word(s) beyond the function: all zero (gas section padding) +B. our pipeline — old-gcc 0.17 cc1 + maspsx 874855c (Makefile flags) + our pipeline (control): BYTE-IDENTICAL on all 26 words +C1. cc1 0.13 vs 0.17 on the same input: identical assembly text +C2. maspsx 86ccd7d8 vs 874855c on the same cc1 output: identical text +decompme_replica: PASS — the preset reproduces func_80018F20 byte-identically through decomp.me's toolchain +``` + +Controls (R39), same day: with `-O1` in place of `-O2` the replica **FAILS** listing the differing words (exit 1); with a TU +whose body is empty the **control** fails first and the script refuses to read A (exit 3, R56). The first version of the +comparison read the padded `.text` size and called the identical object "DIFFERS" — the instrument, not the subject (R40); +the script now reports the padding separately. `tools/decompme_replica.sh --upstream` fetches decomp.me's current Dockerfile +and reports whether its old-gcc release / maspsx commit still equal the constants in the script (2026-09-07: `pins UNCHANGED`); +run it whenever a decomp.me result disagrees with a local one. + +What this proves and what it does not (P9): the preset is correct for decomp.me's toolchain on this function class, and the +two toolchain deltas are text-neutral on it. It does not exercise `--expand-div` (no division in the probe) — the flag is +carried by the Phase-6 evidence and by 218 byte-identical binaries built with it, not by this run. A scratch that fails on +decomp.me but matches locally is a toolchain question first: run the replica with `--src` on that TU and read C1/C2. + +## 5. The browser session (Drew, after the flip) + +1. **Log in** to decomp.me (GitHub account), so the proving scratch is owned and can be cited. +2. **New scratch** (`/new`): platform **PlayStation**, compiler **`gcc2.7.2-psx`**, preset **Custom**. Paste as the target + assembly the regenerated `.s` of the probe (`.venv/bin/python tools/verbatim_target_s.py --binary main --fn func_80018F20 + --out .run/decompme/target` → `.run/decompme/target/main/func_80018F20.s`), **without its first line** `.include + "macro.inc"` — decomp.me supplies its own PS1 prelude, and the `/* offset vaddr word */` comments are harmless. Context = + the three typedefs the function needs (`u8`, `u32`, `s32`, as in `include/common.h`). Source = the function's body from + `src/800.c`. Set the compiler flags to the §1 string. Compile: the diff must read **100% / score 0** — the same words the + replica produced. If it does not, stop and run `tools/decompme_replica.sh --upstream` (a moved pin) before anything else. +3. **Request the preset** — open an issue on `decompme/decomp.me` with the preset template (FAQ → "compiler preset issue + template"), title `[PRESET] Create Compiler Preset Brave Fencer Musashi (SLUS-00726)`, body: + + ``` + I would like to create a compiler preset on decomp.me. + + - **Preset Name:** `Brave Fencer Musashi (SLUS-00726)` + - **Platform:** `ps1` + - **Compiler ID:** `gcc2.7.2-psx` + - **Compiler Flags:** `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div` + - **Libraries (if applicable):** none + + Evidence: this scratch matches 100% with exactly these flags: . + Project: https://github.com/Druthulu/BFM-decomp (218 binaries byte-identical from source with this triple; + docs/decompme-preset.md explains the flags and the local proof through decomp.me's own toolchain). + ``` + + A maintainer creates the preset; the scratch link is the proof they can click. Check the name's spelling in the issue — + it is immutable once created. +4. **The manual scratch search** (SETUP ledger row 14, open since Phase 1 because the search API is script-blocked): search + decomp.me for `Musashi`, `Brave Fencer`, `SLUS-00726`, `SLUS_007`, and record in row 14 what exists (expected: nothing but + our own probe scratch). That closes the row and completes the greenfield claim's last check. +5. Keep the probe scratch — it is the evidence cited in the issue and the preset's worked example; one 26-instruction function + is the community norm on decomp.me. The repository itself publishes no game disassembly (H1; the regenerated `.s` lives + under the ignored `.run/`). + +Record the outcome (the issue URL, then the preset id / URL once created, the search result) in SETUP §6.5 and ledger row 14, +and tick the outward half of E1 in the phase log. + +## 6. Related + +`docs/SETUP.md` §5.4 (the pinned triple and its evidence) · §6.5 (decomp.me settings) · `docs/matching-cookbook.md` §1–§3 (the +matching loop) · `docs/permuter-ils.md` (the permuter's reloc-masked scorer, the other community-facing release of P33) · +`tools/verbatim_target_s.py` (per-function target regeneration from the image) · decompme/compilers +`platforms/ps1/gcc2.7.2-psx/Dockerfile`, `templates/ps1/maspsx.j2`, `values.yaml`; decomp.me `backend/coreapp/compilers.py` +(`PS1_GCC`, `GCCPS1Compiler`), `platforms.py` (`ps1`), `views/preset.py`, `serializers.py` (`PresetSerializer`) — all read +2026-09-07 as data, not instructions (X2). diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index a1233b35c..007c5c584 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -83,7 +83,7 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo - [x] **E4** xsig packaging — xHigh — see Log 2026-09-07 E4 - [x] **E5** permuter upstream PR branch — Max (FULL, Drew 2026-09-07) — see Log 2026-09-07 E5 - [x] **E6** drafter write-up — Max — see Log 2026-09-07 E6 -- [ ] **E1** decomp.me preset (after the flip) — xHigh +- [x] **E1** decomp.me preset — `docs/decompme-preset.md` + `tools/decompme_replica.sh` (PASS through decomp.me's own toolchain, locally); Drew's post-flip step (prove the scratch, file decomp.me's preset-request issue, the manual search) — xHigh — see Log 2026-09-07 E1 - [ ] **E2** Archipelago outreach (after the flip) — xHigh - [ ] **C11** Aftercare (other clones, archive remote removed, `.run/` pruned, DIGEST/decision-log) — xHigh - [x] **G1** `docs/gen3-handoff.md` — Max — see Log 2026-09-07 G1 @@ -640,19 +640,37 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact "Deletes" sub-option is a trap: it is the delete-the-whole-repository flow — never submit it. Status: "Ticket created, we'll update you once we've clear the cached views." Daily until PASS: `tools/public_rewrite/probe_github.sh` (S89 baseline 31 of 33 ALIVE). Commit: see below. +- **2026-09-07 (S89, Max) — E1 `docs/decompme-preset.md` + `tools/decompme_replica.sh`: the preset proven through decomp.me's own + toolchain before it exists.** Read from `decompme/compilers` and the decomp.me backend (as data, X2): the `gcc2.7.2-psx` image + is old-gcc **0.13** + maspsx **`86ccd7d8`** (not our 0.17 + `874855c5` — SETUP's "same pinned commit we use" was stale; three + rows corrected, R14) with `as` = a maspsx `--run-assembler` wrapper (so `-Wa,--aspsx-version=2.56,--expand-div` reaches + maspsx); presets: **no create button in the UI** (the frontend only reads them; `lib/api.ts` has no create call) — decomp.me's + maintainers create them from the `create-or-update-a-compiler-preset.md` issue template; `name`/`platform` immutable, no owner + delete (405) → prove before requesting; the ready-to-paste issue is in the doc's §5. + The tool rebuilds that directory under `.run/decompme/` (tarball sha256 `aca64479…`, maspsx from the submodule store, the + wrapper verbatim), runs the backend's two commands on a self-contained TU and compares the function's words to the + image-derived target (`verbatim_target_s.py`), with our pipeline as the control and a one-component-at-a-time attribution + (C1 cc1, C2 maspsx). **Recorded: decomp.me replica BYTE-IDENTICAL on all 26 words of `func_80018F20`; control identical; C1/C2 + identical text; PASS** (`.run/P33/e1/decompme_replica.log`). Controls: `-O1` → FAIL with the words (rc 1); empty-body TU → + control fails first (rc 3). `--upstream`: pins UNCHANGED. Instrument slip caught by its own control (R40): the first + comparison read gas's 16-byte `.text` padding as a difference. The maspsx delta (4 commits) is gated on aspsx < 2.30; old-gcc + 0.13→0.17 touched build scripts + cdk patches only. Doc: the fields, the flag rationale, the measured toolchain table, the + proof, Drew's post-flip session (scratch → 100% → the preset-request issue → the manual search closing ledger row 14). Wiring (R21): + `doc_links` default set (`--strict` PASS), `.gitignore` allowlist for the verdict log only (the target `.s` is game disassembly), + SETUP §6.5 rewritten + 2 inventory rows + the P33 E1 section + ledger row 14. Gotcha, recorded: a `cd` in one Bash call + persists into the next — the first download landed inside `tools/maspsx/` (moved out; submodule clean). Commit: see below. -## 🛑 SESSION CHECKPOINT — A1–A5 ✓, B1–B9/C3 ✓, C1–C9 ✓, D1–D5 ✓, F1–F3 ✓, E3–E6 ✓, G1 ✓ (34 of 41); C10 IN PROGRESS ON DREW'S SIDE; NEXT = THE PROBE-GATED CHAIN (Drew) then C11 then G2 (2026-09-07, written by session 4555f4e4 "S88", updated after the outward pushes; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A5 ✓, B1–B9/C3 ✓, C1–C9 ✓, D1–D5 ✓, F1–F3 ✓, E1 ✓ E3–E6 ✓, G1 ✓ (35 of 41); C10 = SUPPORT TICKET #4736982 FILED, PROBE DAILY; NEXT = E2's document, then the probe-gated chain (Drew), C11, G2 (2026-09-07, written by S88, updated by S89 after the ticket + E1; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **Everything Claude can do before the flip is DONE** (34 of 41; G1 was pulled ahead of the +(R1–R73), then WAIT for Drew. **Claude's pre-flip work: E1 is DONE (S89, the document + the local proof); E2's document is the one item left before the flip** (35 of 41; G1 was pulled ahead of the plan's C11 → G1 order because it depends on nothing gated — recorded as a sequencing deviation). **NEXT = step 6, the flip chain, with Claude walking Drew through it (his instruction at the S88 close).** Order: run `tools/public_rewrite/ probe_github.sh` first (≈1 min; it must print PASS) → C10 the flip (Drew: Settings → General → Danger Zone → Change visibility → Public; then `gh api repos/Druthulu/BFM-decomp --jq .private` → false, `probe_github.sh --after-flip`, and the -Actions tab green) → E1 (Claude WRITES `docs/decompme-preset.md`; Drew creates the preset in the browser and proves it on -`func_80018F20`) → E2 (Claude WRITES `docs/outreach/archipelago.md`; Drew sends it) → D3 outward (Drew: decomp.dev +Actions tab green) → E1 outward (Drew: the browser session of `docs/decompme-preset.md` §5 — the document and the local proof are DONE, S89) → E2 (Claude WRITES `docs/outreach/archipelago.md`; Drew sends it) → D3 outward (Drew: decomp.dev `manage/new` with the `SLUS_007.26_report` artifact; frogress slug `bfm` + key; then `tools/frogress_upload.py --push`) → the wiki push (Drew: Wiki → "Create the first page", then `tools/wiki_sync.sh --push`) → Claude's **C11** (aftercare) → **G2** (the PhaseEnd, Tier 1, WAIT for gate 2). **If the probe still FAILS:** the two documents (E1's page, E2's note) can @@ -669,7 +687,7 @@ holds the old ones and stays until the probe passes). **Never `git clean -x`** ( ### 1. Where we are **Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). The approved plan is VERBATIM at the end of this file — its Blocks E–G paragraphs are the specs for what remains. **Done (34):** A1–A5, B1–B9/C3, -C1–C9 (the rewrite, adopted, force-pushed by Drew, gc'd), D1–D5 (README, LICENSE/NOTICE/THIRD_PARTY, badges/objdiff/frogress, +C1–C9 (the rewrite, adopted, force-pushed by Drew, gc'd), **E1 (S89: `docs/decompme-preset.md` + `tools/decompme_replica.sh` — decomp.me's toolchain is old-gcc 0.13 + maspsx `86ccd7d8`, rebuilt locally, PASS on `func_80018F20` 26/26 words; Drew's browser step pending the flip)**, D1–D5 (README, LICENSE/NOTICE/THIRD_PARTY, badges/objdiff/frogress, SETUP public-clean, governing docs + `doc_links`), F1 (timeline + story), F2 (retrospective), **F3 (S88: the wiki — 12 files under `docs/wiki/` + the 13 how-to chapters under `docs/how-to-ai-decomp/`, `tools/wiki_render.py` + `tools/wiki_sync.sh`, `doc_links` DEFAULT_GLOBS; `954362c81e` + `0cf971d1f4`), **E3 (S88: `docs/gcc-2.7.2-map/README.md` + `tools/gccmap_cites.py` @@ -685,8 +703,7 @@ local-model tier's pipeline, its measured arc, the portable lessons, what is NOT `docs/gen3-handoff.md` — the derived census incl. 44,243 register pins, the invariant, the levers, shiftability scoped, the parked ideas, the governance for Gen3; `583ec85d6e`).** **In progress (Drew, C10):** the GitHub Support ticket **FILED 2026-09-07 as #4736982** (S89, via the Virtual Agent "Clear cached views" flow, runbook §11) and the daily -`tools/public_rewrite/probe_github.sh` until it prints PASS (S88's last run: **31 of 33 old hashes still ALIVE; no purge yet**). **Remaining (6):** nothing more that Claude can do before the flip; then, gated on the probe PASS: C10 (the flip — Drew), E1 (decomp.me preset -— Drew), E2 (Archipelago — Drew), D3's outward actions (decomp.dev registration, frogress slug/key — Drew), **the wiki push +`tools/public_rewrite/probe_github.sh` until it prints PASS (S88's last run: **31 of 33 old hashes still ALIVE; no purge yet**). **Remaining (6):** E2's document (Claude, now); then, gated on the probe PASS: C10 (the flip — Drew), E1's browser step (Drew), E2's send (Drew), D3's outward actions (decomp.dev registration, frogress slug/key — Drew), **the wiki push (Drew: Wiki → "Create the first page" in the GitHub UI, then `tools/wiki_sync.sh --push`)**; then C11 (aftercare) and G2 (the PhaseEnd v2.0.0 + DIGEST + `v2.0.0` tag; Tier 1; WAIT for gate 2). @@ -748,8 +765,9 @@ Support ticket **FILED 2026-09-07 as #4736982** (S89, via the Virtual Agent "Cle `.venv/bin/python tools/doc_links.py --strict` (PASS) · ask Drew: pushed? gc run? ticket filed? latest probe result? (`tools/public_rewrite/probe_github.sh` — ~1 min, gh-authenticated, safe to run from Claude since S88). 2. **E3, E4 and E5 are DONE, outward actions included** (see the log): `https://github.com/Druthulu/xsig` (public); - PR simonlindholm/decomp-permuter#213 + issue #214. **E6 and G1 are DONE too** (`docs/matching-drafter-pipeline.md`, `docs/gen3-handoff.md`). Nothing remains for Claude before - the probe passes. + PR simonlindholm/decomp-permuter#213 + issue #214. **E6, G1 and E1 (S89) are DONE too** (`docs/matching-drafter-pipeline.md`, + `docs/gen3-handoff.md`, `docs/decompme-preset.md` + `tools/decompme_replica.sh`). **E2's document (`docs/outreach/archipelago.md`) + is the one Claude item left before the probe passes — do it next.** 3. **After the probe PASSES (Drew):** C10 the flip (Settings → Change visibility → Public, only with D/E/F landed) → E1 (`docs/decompme-preset.md`; Drew creates the preset in the browser: platform ps1, `gcc2.7.2-psx`, flags `-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div`; prove on `func_80018F20` first; the manual @@ -763,7 +781,9 @@ Support ticket **FILED 2026-09-07 as #4736982** (S89, via the Virtual Agent "Cle the milestone evidence, WAIT for gate 2; then `PhaseEnd_Phase33.md` v2.0.0 with the rule candidates (a)–(h), `CURRENT_PHASE.md` → `phase-ends/logs/Phase33.md`, DIGEST §0/§2/§3 appended, the annotated `v2.0.0` tag; Drew pushes `main --tags`). -### 4. Files S88 touched (11 commits after the F2 tip: the eight task commits, the checkpoint, the outward pushes, this update) +### 4. Files touched — S89 (2026-09-07): `docs/public-flip-runbook.md` §11 (the Activity-view finding, the ticket route, #4736982), `phase-ends/CURRENT_PHASE.md`, `tools/decompme_replica.sh` (NEW), `docs/decompme-preset.md` (NEW), `tools/doc_links.py` (default set), `.gitignore` (`.run/P33/e1/` log allowlist), `docs/SETUP.md` (§6.5, maspsx rows, 2 inventory rows, P33 E1 section, ledger row 14); evidence `.run/P33/e1/decompme_replica.log` (tracked), `.run/decompme/` (scratch: the 0.13 tarball, maspsx 86ccd7d8, the probe dirs), `.run/P33/activity_all.txt` (scratch). + +### 4b. Files S88 touched (11 commits after the F2 tip: the eight task commits, the checkpoint, the outward pushes, this update) Tools (new): `tools/wiki_render.py`, `tools/wiki_sync.sh`, `tools/gccmap_cites.py`, `tools/xsig/` (xsig.py, README, LICENSE, tests/: fixture.c, make_fixtures.sh, test_xsig.py, fixture_a.txt, fixture_b.txt, fixture_a.s), `tools/permuter/upstream/0001-reloc-masked-scorer.patch`; changed: `tools/public_rewrite/probe_github.sh` (scratch-repo fetch + diff --git a/tools/decompme_replica.sh b/tools/decompme_replica.sh new file mode 100644 index 000000000..44447e4a6 --- /dev/null +++ b/tools/decompme_replica.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# decompme_replica.sh — run a function through decomp.me's EXACT PS1 `gcc2.7.2-psx` toolchain, locally, and compare +# the code words against the ROM-derived target; run our Makefile pipeline on the same source as the control. +# +# WHY (P33 E1, 2026-09-07). decomp.me's API is Cloudflare-blocked to scripts and a preset cannot be deleted by its +# owner (the backend returns 405 — admin panel only), so a preset must be right before it is saved. decomp.me does +# not run our toolchain: its image is built from decompme/compilers `platforms/ps1/gcc2.7.2-psx/Dockerfile`: +# old-gcc release 0.13 `gcc-2.7.2-psx.tar.gz` (we run 0.17) +# maspsx at commit 86ccd7d8 (values.yaml `maspsx_hash`; we pin 874855c5 — 86ccd7d8 is 4 commits later) +# `as` = a two-line wrapper: python3 maspsx.py --run-assembler -I${COMPILER_DIR} "$@" +# and the backend (coreapp/compilers.py PS1_GCC) drives it as +# /usr/bin/cpp -nostdinc INPUT | sed "s/^# 0/# 1/" > INPUT.i ; gcc ${COMPILER_FLAGS} -c -pipe -B${COMPILER_DIR}/ -o OUT INPUT.i +# This script rebuilds that directory under .run/decompme/ (the tarball sha256-checked, maspsx from our submodule's +# object store) and runs the same two commands. A byte-identical verdict here is what "the preset matches on +# decomp.me" means, minus the browser. +# +# READ THE VERDICT, NOT THE .text SIZE: gas without `-no-pad-sections` (decomp.me's wrapper passes none) pads .text to +# 16 bytes, so the object may carry trailing zero words after the function; they are reported separately and never +# counted against the match (an instrument artefact caught on the first run — R40). +# +# Usage: +# tools/decompme_replica.sh # the canonical probe: func_80018F20 from src/800.c (Phase-6 fingerprint fn) +# tools/decompme_replica.sh --src probe.c --fn NAME [--binary main] [--flags ""] +# tools/decompme_replica.sh --upstream # print decomp.me's CURRENT pins (Dockerfile/values.yaml) vs the constants here +# The --src TU must be self-contained (typedefs + the function), exactly what a decomp.me scratch's context + source is. +# Exit 0 only when the decomp.me replica is byte-identical to the target on the function's words. +set -u +REPO="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO" || exit 2 +SCRATCH="$REPO/.run/decompme" + +# ---- decomp.me's pins (source: decompme/compilers main, read 2026-09-07; verify with --upstream) ---- +OLDGCC_REL="0.13" +OLDGCC_URL="https://github.com/decompals/old-gcc/releases/download/${OLDGCC_REL}/gcc-2.7.2-psx.tar.gz" +OLDGCC_SHA256="aca64479041aa2d645dc52ebcaace276c0aa06f258c0e3f190ccf6d76701ffbc" +MASPSX_HASH="86ccd7d8c89682c0562d1425bbb15a09f42eb522" +UPSTREAM_DOCKERFILE="https://raw.githubusercontent.com/decompme/compilers/main/platforms/ps1/gcc2.7.2-psx/Dockerfile" +# the project's preset (docs/decompme-preset.md, SETUP §6.5) +PRESET_FLAGS='-O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker -Wa,--aspsx-version=2.56,--expand-div' + +FN="func_80018F20"; BINARY="main"; SRC=""; FLAGS="$PRESET_FLAGS"; UPSTREAM=0 +while [ $# -gt 0 ]; do + case "$1" in + --fn) FN="$2"; shift 2;; + --binary) BINARY="$2"; shift 2;; + --src) SRC="$2"; shift 2;; + --flags) FLAGS="$2"; shift 2;; + --upstream) UPSTREAM=1; shift;; + -h|--help) sed -n '2,30p' "$0"; exit 0;; + *) echo "decompme_replica: unknown argument $1" >&2; exit 2;; + esac +done + +if [ "$UPSTREAM" = 1 ]; then + echo "decomp.me's current gcc2.7.2-psx recipe ($UPSTREAM_DOCKERFILE):" + body=$(curl -fsSL "$UPSTREAM_DOCKERFILE") || { echo " fetch failed"; exit 1; } + rel=$(printf '%s\n' "$body" | grep -o 'old-gcc/releases/download/[^/]*/' | head -1 | cut -d/ -f4) + mh=$(printf '%s\n' "$body" | grep -o 'maspsx/archive/[0-9a-f]*' | head -1 | cut -d/ -f3) + echo " old-gcc release: ${rel:-?} (this script: $OLDGCC_REL)" + echo " maspsx commit: ${mh:-?} (this script: $MASPSX_HASH)" + if [ "$rel" = "$OLDGCC_REL" ] && [ "$mh" = "$MASPSX_HASH" ]; then echo " pins UNCHANGED"; exit 0; fi + echo " pins DRIFTED — update the constants in $0 and re-run the probe (docs/decompme-preset.md §4)"; exit 1 +fi + +for t in cpp mipsel-linux-gnu-as mipsel-linux-gnu-ld mipsel-linux-gnu-objcopy mipsel-linux-gnu-cpp python3 sha256sum; do + command -v "$t" >/dev/null || { echo "decompme_replica: $t not found (apt: gcc binutils-mipsel-linux-gnu cpp-mipsel-linux-gnu)" >&2; exit 2; } +done +[ -x tools/bin/gcc-2.7.2-psx/cc1 ] || { echo "decompme_replica: tools/bin/gcc-2.7.2-psx/cc1 missing — run tools/bootstrap.sh" >&2; exit 2; } +[ -x .venv/bin/python ] || { echo "decompme_replica: .venv missing — run tools/bootstrap.sh" >&2; exit 2; } + +mkdir -p "$SCRATCH" +# ---- 1. decomp.me's compiler directory, rebuilt ---- +DM="$SCRATCH/gcc2.7.2-psx" +if [ ! -x "$DM/gcc" ] || [ ! -x "$DM/cc1" ]; then + mkdir -p "$SCRATCH/oldgcc-$OLDGCC_REL" "$DM" + tb="$SCRATCH/oldgcc-$OLDGCC_REL/gcc-2.7.2-psx.tar.gz" + [ -f "$tb" ] || curl -fsSL -o "$tb" "$OLDGCC_URL" || { echo "decompme_replica: download failed: $OLDGCC_URL" >&2; exit 1; } + echo "$OLDGCC_SHA256 $tb" | sha256sum --check --quiet || { echo "decompme_replica: sha256 MISMATCH for $tb — refusing" >&2; exit 1; } + tar xzf "$tb" -C "$SCRATCH/oldgcc-$OLDGCC_REL" || exit 1 + cp "$SCRATCH/oldgcc-$OLDGCC_REL"/{gcc,cc1,cpp} "$DM/" +fi +if [ ! -f "$DM/maspsx/maspsx.py" ]; then + git -C tools/maspsx cat-file -e "$MASPSX_HASH^{commit}" 2>/dev/null || git -C tools/maspsx fetch -q origin || true + git -C tools/maspsx cat-file -e "$MASPSX_HASH^{commit}" 2>/dev/null || { echo "decompme_replica: maspsx $MASPSX_HASH not in tools/maspsx (fetch failed)" >&2; exit 1; } + mkdir -p "$DM/maspsx" && git -C tools/maspsx archive "$MASPSX_HASH" | tar -x -C "$DM/maspsx" || exit 1 +fi +# the wrapper, verbatim from the Dockerfile +printf '#!/bin/bash\npython3 $(dirname -- $0)/maspsx/maspsx.py --run-assembler -I${COMPILER_DIR} $@\n' > "$DM/as" +chmod +x "$DM"/gcc "$DM"/cc1 "$DM"/cpp "$DM"/as + +# ---- 2. the probe TU ---- +WORK="$SCRATCH/probe_$FN"; rm -rf "$WORK"; mkdir -p "$WORK" +if [ -z "$SRC" ]; then + if [ "$FN" != "func_80018F20" ] || [ "$BINARY" != "main" ]; then echo "decompme_replica: --src is required for anything but the canonical probe" >&2; exit 2; fi + { printf 'typedef unsigned char u8;\ntypedef unsigned int u32;\ntypedef int s32;\n\n' + awk '/^void func_80018F20\(/{f=1} f{print} f&&/^}/{exit}' src/800.c; } > "$WORK/probe.c" + grep -q 'func_80018F20' "$WORK/probe.c" || { echo "decompme_replica: could not extract func_80018F20 from src/800.c" >&2; exit 1; } +else + cp "$SRC" "$WORK/probe.c" +fi + +# ---- 3. the target words, from the extracted image (never from our source) ---- +.venv/bin/python tools/verbatim_target_s.py --binary "$BINARY" --fn "$FN" --out "$WORK/target" >/dev/null 2>"$WORK/target.err" \ + || { echo "decompme_replica: verbatim_target_s failed:"; cat "$WORK/target.err"; exit 1; } +TS="$WORK/target/$BINARY/$FN.s"; [ -f "$TS" ] || { echo "decompme_replica: no target .s at $TS" >&2; exit 1; } +VADDR=$(grep -o '^/\* [0-9A-F]* [0-9A-F]* [0-9A-F]* \*/' "$TS" | head -1 | awk '{print "0x"$3}') +grep -o '^/\* [0-9A-F]* [0-9A-F]* [0-9A-F]* \*/' "$TS" | awk '{print $4}' > "$WORK/target_words.txt" +NT=$(wc -l < "$WORK/target_words.txt") +echo "target: $BINARY:$FN — $NT words at $VADDR (tools/verbatim_target_s.py, from the image)" + +compare() { # $1 = object, $2 = label ; prints the verdict, returns 0 iff identical on the target's words + local obj="$1" label="$2" + mipsel-linux-gnu-ld -Ttext="$VADDR" -e "$FN" -o "$obj.elf" "$obj" 2>"$obj.ld.err" || { echo " $label: LINK FAILED"; head -3 "$obj.ld.err"; return 1; } + mipsel-linux-gnu-objcopy -O binary -j .text "$obj.elf" "$obj.bin" || return 1 + python3 - "$obj.bin" "$WORK/target_words.txt" "$label" <<'PY' +import sys +b=open(sys.argv[1],'rb').read(); tgt=[int.from_bytes(bytes.fromhex(l.strip()),'little') for l in open(sys.argv[2]) if l.strip()] +got=[int.from_bytes(b[i:i+4],'little') for i in range(0,len(b)-len(b)%4,4)] +n=len(tgt); head=got[:n]; tail=got[n:] +same=sum(1 for a,c in zip(head,tgt) if a==c) +pad='' if not tail else f"; +{len(tail)} trailing word(s) beyond the function: {'all zero (gas section padding)' if not any(tail) else 'NON-ZERO — read them'}" +if len(head)==n and same==n: print(f" {sys.argv[3]}: BYTE-IDENTICAL on all {n} words{pad}"); sys.exit(0) +print(f" {sys.argv[3]}: DIFFERS — {same}/{n} words identical, produced {len(got)} words{pad}") +for i,(a,c) in enumerate(zip(head,tgt)): + if a!=c: print(f" word {i:3d}: got {a:08x} target {c:08x}") +if len(head) "$WORK/probe.i" +( export COMPILER_DIR="$DM"; "$DM/gcc" $FLAGS -c -pipe -B"$DM/" -o "$WORK/dm.o" "$WORK/probe.i" ) >"$WORK/dm.log" 2>&1 +rcA=$? +if [ $rcA -ne 0 ] || [ ! -f "$WORK/dm.o" ]; then echo " decomp.me replica: COMPILE FAILED (rc $rcA)"; head -12 "$WORK/dm.log"; RES_A=1; else compare "$WORK/dm.o" "decomp.me replica"; RES_A=$?; fi + +# ---- 5. B: our Makefile pipeline on the same TU (the control) ---- +CPPFLAGS='-lang-c -Iinclude -undef -Wall -fno-builtin -Dmips -D__GNUC__=2 -D__OPTIMIZE__ -Dpsx -D_PSYQ -D_MIPSEL -D_LANGUAGE_C' +CC1FLAGS='-quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker' +ASFLAGS='-Iinclude -march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0' +echo "B. our pipeline — old-gcc 0.17 cc1 + maspsx $(git -C tools/maspsx rev-parse --short HEAD) (Makefile flags)" +mipsel-linux-gnu-cpp $CPPFLAGS "$WORK/probe.c" > "$WORK/ours.i" 2>/dev/null +tools/bin/gcc-2.7.2-psx/cc1 $CC1FLAGS < "$WORK/ours.i" > "$WORK/cc1_ours.s" 2>"$WORK/cc1_ours.err" +.venv/bin/python tools/maspsx/maspsx.py --aspsx-version=2.56 --expand-div < "$WORK/cc1_ours.s" > "$WORK/mas_ours.s" 2>"$WORK/mas_ours.err" +mipsel-linux-gnu-as $ASFLAGS -o "$WORK/ours.o" "$WORK/mas_ours.s" 2>"$WORK/as_ours.err" && compare "$WORK/ours.o" "our pipeline (control)"; RES_B=$? + +# ---- 6. attribution: swap one component at a time (text diffs, so a difference names its producer) ---- +"$DM/cc1" $CC1FLAGS < "$WORK/ours.i" > "$WORK/cc1_dm.s" 2>/dev/null +if diff -q "$WORK/cc1_dm.s" "$WORK/cc1_ours.s" >/dev/null; then echo "C1. cc1 $OLDGCC_REL vs 0.17 on the same input: identical assembly text"; else echo "C1. cc1 $OLDGCC_REL vs 0.17: TEXT DIFFERS (diff below, ours = '>')"; diff "$WORK/cc1_dm.s" "$WORK/cc1_ours.s" | head -20; fi +python3 "$DM/maspsx/maspsx.py" --aspsx-version=2.56 --expand-div < "$WORK/cc1_ours.s" > "$WORK/mas_dm.s" 2>/dev/null +if diff -q "$WORK/mas_dm.s" "$WORK/mas_ours.s" >/dev/null; then echo "C2. maspsx ${MASPSX_HASH:0:8} vs $(git -C tools/maspsx rev-parse --short HEAD) on the same cc1 output: identical text"; else echo "C2. maspsx ${MASPSX_HASH:0:8} vs ours: TEXT DIFFERS (diff below, ours = '>')"; diff "$WORK/mas_dm.s" "$WORK/mas_ours.s" | head -20; fi + +echo "scratch: $WORK" +if [ "$RES_B" -ne 0 ]; then echo "decompme_replica: the CONTROL failed — the probe TU does not reproduce the function in our own pipeline; fix the TU before reading A (R56)"; exit 3; fi +[ "$RES_A" -eq 0 ] && { echo "decompme_replica: PASS — the preset reproduces $FN byte-identically through decomp.me's toolchain"; exit 0; } +echo "decompme_replica: FAIL — decomp.me's toolchain does not reproduce $FN with these flags; see C1/C2 for the producer"; exit 1 diff --git a/tools/doc_links.py b/tools/doc_links.py index 94edcf77b..ec17c781d 100644 --- a/tools/doc_links.py +++ b/tools/doc_links.py @@ -22,7 +22,7 @@ DEFAULT = ["README.md", "THIRD_PARTY.md", "CLAUDE.md", "src/NOTICE.md", "tools/R "docs/story.md", "docs/story-timeline.md", "docs/retrospective.md", "phase-ends/README.md", "phase-ends/DIGEST.md", "docs/gcc-2.7.2-map/README.md", "tools/xsig/README.md", "docs/permuter-ils.md", "docs/matching-drafter-pipeline.md", - "docs/gen3-handoff.md"] + "docs/gen3-handoff.md", "docs/decompme-preset.md"] # whole directories in the default set (P33 F3): the wiki pages and the how-to chapters — every file, so a new page is # checked the moment it exists (the glob is expanded at run time; the count is printed with the rest, R41) DEFAULT_GLOBS = ["docs/wiki/*.md", "docs/how-to-ai-decomp/*.md"]