From 9c4d32d651bebdfd40bdfc3eeeda6b9c5048e5b7 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:51:02 -0600 Subject: [PATCH] =?UTF-8?q?tools(phase-33):=20E4=20=E2=80=94=20xsig=20pack?= =?UTF-8?q?aged:=20tools/xsig/=20(xsig.py=20library=20+=20CLI=20sign-s=20|?= =?UTF-8?q?=20sign-objdump=20|=20cross=20|=20verify=20|=20selftest,=20from?= =?UTF-8?q?=20the=20Phase-21=20.run/xdedup=20probe;=20README=20with=20the?= =?UTF-8?q?=20recorded=20worked=20example=20=E2=80=94=20BFM=20=C3=97=20Xen?= =?UTF-8?q?ogears=20+=20Vagrant=20Story=20103=20hits=20all=20PsyQ=20librar?= =?UTF-8?q?y/BIOS,=20BFM=20=C3=97=20Tomba=20126=20hits=20124=20library:=20?= =?UTF-8?q?the=20decision=20log's=20clean=20negative;=20MIT=20LICENSE;=20t?= =?UTF-8?q?ests/=20from=20a=20game-free=20fixture=20compiled=20with=20the?= =?UTF-8?q?=20pinned=20triple=20and=20linked=20at=20two=20addresses=20with?= =?UTF-8?q?=20--emit-relocs,=20relocation=20records=20merged=20into=20the?= =?UTF-8?q?=20listings,=208/8)=20in=20tools-health=20+=20CI=20+=20doc=5Fli?= =?UTF-8?q?nks;=20SETUP=20row=20+=20P33=20E4=20section;=20the=20standalone?= =?UTF-8?q?=20copy=20at=20.run/P33/xsig-repo/=20(one=20commit,=20noreply?= =?UTF-8?q?=20identity,=20system-python=20tests=20OK)=20for=20Drew=20to=20?= =?UTF-8?q?push=20as=20Druthulu/xsig;=20log=20+=20checkpoint=20(NEXT=20=3D?= =?UTF-8?q?=20E5,=20a=20cut=20candidate=20for=20Drew)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/no-rom.yml | 2 + Makefile | 1 + docs/SETUP.md | 23 +++ phase-ends/CURRENT_PHASE.md | 68 ++++-- tools/doc_links.py | 2 +- tools/xsig/LICENSE | 21 ++ tools/xsig/README.md | 106 ++++++++++ tools/xsig/tests/fixture.c | 20 ++ tools/xsig/tests/fixture_a.s | 36 ++++ tools/xsig/tests/fixture_a.txt | 52 +++++ tools/xsig/tests/fixture_b.txt | 52 +++++ tools/xsig/tests/make_fixtures.sh | 84 ++++++++ tools/xsig/tests/test_xsig.py | 90 ++++++++ tools/xsig/xsig.py | 329 ++++++++++++++++++++++++++++++ 14 files changed, 867 insertions(+), 19 deletions(-) create mode 100644 tools/xsig/LICENSE create mode 100644 tools/xsig/README.md create mode 100644 tools/xsig/tests/fixture.c create mode 100644 tools/xsig/tests/fixture_a.s create mode 100644 tools/xsig/tests/fixture_a.txt create mode 100644 tools/xsig/tests/fixture_b.txt create mode 100644 tools/xsig/tests/make_fixtures.sh create mode 100644 tools/xsig/tests/test_xsig.py create mode 100644 tools/xsig/xsig.py diff --git a/.github/workflows/no-rom.yml b/.github/workflows/no-rom.yml index ab2b4565d..36db3cefc 100644 --- a/.github/workflows/no-rom.yml +++ b/.github/workflows/no-rom.yml @@ -48,6 +48,8 @@ jobs: run: python tools/cookbook_index.py --check - name: gccmap_cites — every codegen-map citation carries its source-tree tag run: python tools/gccmap_cites.py --check + - name: xsig — the cross-project signature tool's tests (fixtures carry no game bytes) + run: python tools/xsig/tests/test_xsig.py - name: ghidra_roster — derived roster is fresh run: python tools/ghidra_roster.py --check - name: work_evidence self-test diff --git a/Makefile b/Makefile index 51bc6baeb..4ab9dded6 100644 --- a/Makefile +++ b/Makefile @@ -298,6 +298,7 @@ tools-health: # P33 D5: every relative link in the public-facing docs resolves (pending pages are listed, and must be gone by gate 2) $(VENV_PY) tools/doc_links.py $(VENV_PY) tools/wiki_render.py --selftest + $(VENV_PY) tools/xsig/tests/test_xsig.py 2>&1 | tail -1 | grep -q '^OK' && echo 'xsig tests: OK (8)' || { echo 'xsig tests: FAIL'; exit 1; } # Behavioural guards (P31 S70): tools-health audits DATA integrity; these assert that a tool # ACTUALLY DID the work it reports. A guard that is not running is not a guard (R54). $(VENV_PY) tools/work_evidence.py --selftest diff --git a/docs/SETUP.md b/docs/SETUP.md index 9cc6923fc..4db43fed2 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -765,6 +765,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/wiki_render.py OUT_DIR \| --list \| --selftest` | **(P33 F3)** Render `docs/wiki/*.md` + `docs/how-to-ai-decomp/*.md` into GitHub-wiki page names with every relative link rewritten deterministically (wiki page → its name; a chapter → `How-to-AI-decomp-NN-name`; any other repo path → a `blob/main` / `tree/main` / raw URL; URLs, mailto and anchors untouched; **a dead link is an error**, R43). `--selftest` = the 12-case fixture incl. the dead-link negative control, in `make tools-health`. | | | `tools/wiki_sync.sh [--push] [--wiki-url URL]` | **(P33 F3)** Render into `.run/wiki/render/` → clone or fast-forward `BFM-decomp.wiki.git` under `.run/wiki/` → REPLACE its pages with the rendered set (the repo is the source of truth; a page edited on GitHub is overwritten) → `git status --short`; **`--push` is Drew's (R6)**: commit + push. Before the wiki repo exists (after the flip AND the first page created in the GitHub UI) the dry run renders and lists (exit 0) and `--push` refuses (exit 2). | | | `tools/gccmap_cites.py [--dry-run \| --check \| --verify \| --controls \| --explain] [--retag]` | **(P33 E3)** Tag every `file.c:NNN` cite in `docs/gcc-2.7.2-map/*.md` with the source tree its line number belongs to — `[2.7.2]` (the vanilla subset), `[2.8.1 pm]` (gcc-papermario), `[repo]` — derived from the trees (quoted snippets, identifiers with function extents and nearest distance, the author's cues; a contradiction fails loudly; ties → `cite_overrides.tsv` → cue → 2.7.2); writes in place, idempotent, never silently changes an existing tag. `--check` is textual (every cite tagged, no stale override) and runs in `make tools-health` + CI; the others need both reference trees and refuse without them. | +| | `tools/xsig/xsig.py sign-s \| sign-objdump \| cross \| verify \| selftest` | **(P33 E4; Phase-21 origin)** Relocation-masked per-function signatures for CROSS-PROJECT code identification: mask `j`/`jal` targets and HI16/LO16 immediates (from `%hi`/`%lo` operands or `objdump -dr` records), keep opcodes/registers/constants/branches; `cross` joins two JSONL sets on `sig` with a coverage line; `verify` prints the instruction-by-instruction diff (opcode/register/immediate/length). Self-contained (stdlib, MIT, own README + LICENSE + `tests/` from a game-free fixture compiled at two link addresses with `--emit-relocs`); `tests/test_xsig.py` (8) in `make tools-health` + CI. The standalone repo copy is prepared under `.run/P33/xsig-repo/` (Drew creates + pushes `Druthulu/xsig`). | | | `tools/objdiff_report.py [--in docs/progress.json] [--out report.json]` | **(P33 D3)** progress.json → objdiff's report format (report.proto v2, snake_case — validated with `objdiff-cli` 3.8.1 `report changes`): one unit per binary (code = instructions × 4, functions byte-identical / matchable, metadata complete), categories `game-code` and `linked-sony-objects` (functions only). `.github/workflows/progress.yml` runs it on every push (no rebuild — the committed JSON) and uploads the artifact **`SLUS_007.26_report`** for decomp.dev (Drew registers at decomp.dev/manage/new after the flip). | | | `tools/frogress_upload.py [--push --project bfm --version us]` | **(P33 D3)** stdlib; `--dry-run` is the default (prints the payload); `--push` POSTs `{"api_key","entries":[{git_hash,timestamp,categories:{default:{measures…}}}]}` to `progress.deco.mp/data///` with `FROGRESS_API_SECRET` from the environment (never a file). frogress projects are admin-created — Drew requests the slug + key after the flip. | | | `tools/public_rewrite/` (P33 C1) | **The history-rewrite package** (`docs/public-flip-runbook.md` §3 is the operating table). `common.py` (shared: the purge rules, the DERIVED content-hash sets, identities from the log, the one hash regex, a persistent `cat-file --batch`) · `hash_dict.py [--write-mailmap]` (every commit OBJECT → `commit:NNNN` / twin / orphan; prefix index 7..40; asserts 0 ambiguous; records content-hash collisions as excluded; writes the scratch mailmap) · `scrub.py --test \| --sample \| --file` (THE scrub: hash tokens, addresses → noreply, trailer lines in messages; 12 known-true cases; the HEAD sample with git's own object lookup as the independent oracle) · `gate_scan.py --all\|--refs … [--worktree] [--expect-fail FIXTURE]` (paths ever touched × purge rules; every reachable blob's content sha1 × the ROM set; 5 byte signatures; 50 MiB; emits `rom_blob_ids.txt` = hits ∪ every blob ever under a purge path; the fixture `expected_offenders.txt` is the R39 negative control) · `run_filter.py [--sample]` (the git-filter-repo 2.47.0 module-API run inside the scratch bare clone; refuses elsewhere) · `verify_rewrite.py --old --new` (the pairwise proof) · `build_commit_map.py [--out]` (`docs/commit-map.tsv`, asserted free of old hashes) · `resolve_tokens.py [--check] [--map]` (tokens → shortest unique ≥9-char new abbreviations at the tip) · `absent_scan.py [--repo] [--tree]` (nothing old anywhere) · `probe_github.sh [--after-flip]` (Drew's purge probe). Scratch (`.run/public_rewrite/`, never committed): `dict.json`, `mailmap`, `rom_blob_ids.txt`, `old-to-new.tsv`, `repo.git`, the bundle. · `probe_github.sh [--after-flip]` (Drew's daily post-purge probe, C10: 33 sampled old shas via `gh api` + a fetch; **S88, R57:** the fetch runs in a throwaway bare repo under `.run/public_rewrite/` with `--filter=blob:none --depth=1`, never in the working repo — a successful fetch of an old sha imports its purged closure, which the S87/S88 runs did (5.97 GiB unreachable) — and it ends with a self-check naming any sampled old commit the working repo still holds + the gc recipe) | @@ -1144,6 +1145,28 @@ fills fast). Nothing is leaking — but the host does not get the memory back on `tools/ghidra_*.sh` are repo-relative (`BFM_GHIDRA_PROJ` overrides the project dir; `ghidra_mcp_verify.sh [PROG]`); Makefile `GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra)`. +### P33 E4 (S88, 2026-09-07) — xsig packaged: `tools/xsig/` + the standalone repo, from the Phase-21 `.run/xdedup/xsig.py` +- **What it is.** The Phase-21 cross-project dedup probe's signature library (`.run/xdedup/xsig.py`, 2026-06-25) plus the + driver scripts around it (`sign_bfm.py`, `sign_xeno.py`, `cross.py`, `analyze.py`) folded into ONE stdlib file with a CLI + (`sign-s | sign-objdump | cross | verify | selftest`), a README that carries the recorded worked example (BFM × Xenogears + + Vagrant Story: 103 hits, all PsyQ library/BIOS — libapi 34, libcd 14, libetc 14, libspu 11, libcard 7 …, the largest + `_spu_setReverbAttr` 307 ins; BFM × Tomba: 126 hits, 124 library, one 19-ins non-library HIGH — the decision log's "clean + negative: zero engine code"), an MIT `LICENSE`, and `tests/`. +- **The fixture (no game bytes, R-candidate (a)):** `tests/fixture.c` (two functions of our own) compiled with the pinned + triple by `tests/make_fixtures.sh` and linked TWICE (`ld -q` = `--emit-relocs`, `-Ttext` 0x80010000 vs 0x80200000, + `--defsym` helper/table at different addresses). Gotcha: `objdump -dr` interleaves relocation records only for OBJECT + files; for a linked ELF they are listed separately (`-r`, section-relative offsets) — the generator merges them into the + listing in the object-listing format (`\t\t\t: R_MIPS_HI16\ttable`), which is what the front-end reads. A second gotcha: + a linked listing puts the address at column 0 (an object listing indents it) — the instruction regex is `^\s*` now. + `tests/test_xsig.py` (8 checks): same function at two link addresses signs identically while the raw words differ; the + relocated fields are present and the only masked ones (`verify`: IDENTICAL up to relocation, 3 fields); a different + function differs; a register flip changes the sig and is classified `register`; the `.s` and objdump front-ends agree; + `sign-objdump` + `cross` find exactly the two true pairs; `verify` accepts the pair and rejects the mismatch; `selftest`. +- **Wiring (R21):** the tests in `make tools-health` and the CI `audits` job; `tools/xsig/README.md` in `doc_links`' default + set; this row/section. The standalone repo: `.run/P33/xsig-repo/` = the same files, `git init`, the repo-local NOREPLY + identity (never the global one), one initial commit; Drew creates `Druthulu/xsig` on GitHub (empty) and pushes. + + ### P33 E3 (S88, 2026-09-07) — the codegen map's README + every citation tagged with its source tree - **Why.** `docs/gcc-2.7.2-map/` was written (Phase 23) against `tools/reference/gcc-papermario`, which is gcc **2.8.1**, and audited (P29 S23) against the vanilla 2.7.2 subset — the mechanisms held (119 confirmed · 40 line-drift · 7 refuted of 21 diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 0632162f3..1cc4ce46a 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -80,7 +80,7 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo - [x] **F2** retrospective — Max — see Log 2026-09-07 F2 - [x] **F3** wiki + how-to-ai-decomp + `wiki_sync.sh` — Max (FULL, Drew 2026-09-07) — see Log 2026-09-07 F3 - [x] **E3** gcc-2.7.2 map README + `gccmap_cites.py` — xHigh — see Log 2026-09-07 E3 -- [ ] **E4** xsig packaging — xHigh +- [x] **E4** xsig packaging — xHigh — see Log 2026-09-07 E4 - [ ] **E5** permuter upstream PR branch — Max - [ ] **E6** drafter write-up — Max - [ ] **E1** decomp.me preset (after the flip) — xHigh @@ -509,37 +509,66 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact harness's low-memory guard during the report step (a transient spike; 29 GB available afterwards) — and the process table held **8 orphaned `tools/permuter/run_masked.py` workers from a closed phase, 49 h old (parent PID 18)**, stopped by PID (never `pkill -f` with a literal the calling shell carries); the foreground re-run passed. Commit: see below. +- **2026-09-07 (S88, Max) — E4 xsig packaged: `tools/xsig/` + the standalone repo.** The Phase-21 cross-project dedup + probe's library (`.run/xdedup/xsig.py`, 2026-06-25) and its four driver scripts folded into ONE stdlib file, + `tools/xsig/xsig.py` (library + CLI `sign-s | sign-objdump | cross | verify | selftest`; `compare()` = the + instruction-by-instruction diff a score cannot replace — opcode/register/immediate/length, R63), with `README.md` (the + recorded worked example: BFM × Xenogears + Vagrant Story = 103 hits, all PsyQ library/BIOS — libapi 34, libcd 14, libetc + 14, libspu 11, libcard 7, largest `_spu_setReverbAttr` 307 ins, 37 already named; BFM × Tomba = 126 hits, 124 library, one + 19-ins non-library HIGH; the decision log's "clean negative — zero engine code"), an MIT `LICENSE`, and `tests/`: a + game-free fixture — `tests/fixture.c` (two functions of our own) compiled with the pinned triple by + `tests/make_fixtures.sh` and linked TWICE (`ld -q` = `--emit-relocs`, `-Ttext` 0x80010000 vs 0x80200000, `--defsym` + helper/table at different addresses) → `fixture_a.txt` / `fixture_b.txt` (objdump listings) + `fixture_a.s` (the splat + form, `%hi`/`%lo` from the reloc records); `tests/test_xsig.py` 8 checks (same function at two link addresses signs + identically while the raw words differ; the 3 relocated fields are present and the only masked ones — `verify`: IDENTICAL + up to relocation; a different function differs; a register flip changes the sig and is classified `register`; the two + front-ends agree; `sign-objdump` + `cross` find exactly the two true pairs; `verify` rejects the mismatch; `selftest`) — + **8/8 OK**. Two gotchas, recorded: `objdump -dr` interleaves relocation records only for OBJECT files — a linked ELF lists + them separately (`-r`, section-relative offsets), so the generator merges them into the listing in the object-listing + format; and a linked listing puts the address at column 0 (an object listing indents it) — the instruction regex is + `^\s*`. Sibling-repo URLs in the README taken from the reference clones' `git remote get-url origin`, not from memory. + Wiring (R21): the tests in `make tools-health` (`xsig tests: OK (8)`) + the CI `audits` job; `tools/xsig/README.md` in + `doc_links`' default set (`--strict` PASS); SETUP row + the P33 E4 section. **The standalone repo:** `.run/P33/xsig-repo/` + = `xsig.py`, `README.md`, `LICENSE`, `tests/` (+ `.gitignore`), `git init`, the repo-local NOREPLY identity (the global + identity DIFFERS and was never used), the tests run there with the SYSTEM `python3` (self-contained, no venv), one + initial commit — **Drew creates `Druthulu/xsig` on GitHub (EMPTY) and pushes** (`git -C .run/P33/xsig-repo remote add + origin … && git push -u origin main`). `make tools-health` re-run with the new line: **`tools-health: OK — sigs fresh; corpus(+resident) + cdecl + binaries + report(lint+dedup) + cookbook-index all green.` (EXIT=0)** + (`.run/P33/e4_tools_health.log`). Commit: see below. -## 🛑 SESSION CHECKPOINT — A1–A5 ✓, B1–B9/C3 ✓, C1–C9 ✓, D1–D5 ✓, F1–F3 ✓, E3 ✓ (30 of 41); C10 IN PROGRESS ON DREW'S SIDE; NEXT = E4 (2026-09-07, written by session 4555f4e4 "S88" at the E3 close; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A5 ✓, B1–B9/C3 ✓, C1–C9 ✓, D1–D5 ✓, F1–F3 ✓, E3 ✓, E4 ✓ (31 of 41); C10 IN PROGRESS ON DREW'S SIDE; NEXT = E5 (2026-09-07, written by session 4555f4e4 "S88" at the E4 close; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **NEXT = E4** (xHigh, ≈0.5 session — the xsig packaging; the plan's remaining cut candidate is -E5 (the permuter PR, Max, 1 session) — Drew chose full scope for F3 and E3 ran in full, so ask only if he raises it). Rebuild -the harness task list (40 items, R28) marking A1–A5, B1–B9/C3, C1–C9, D1–D5, F1–F3, E3 completed and C10 in progress. **Every commit cites NEW +(R1–R73), then WAIT for Drew. **NEXT = E5** (Max, ≈1 session — the permuter upstream PR branch; it is the plan's last +named CUT CANDIDATE: ask Drew whether E5 runs in full, trimmed to the issue + `docs/permuter-ils.md`, or is cut — F3 and E3 +ran in full). Rebuild the harness task list (40 items, R28) marking A1–A5, B1–B9/C3, C1–C9, D1–D5, F1–F3, E3, E4 completed and +C10 in progress. **Every commit cites NEW hashes only** (the history was rewritten; `docs/commit-map.tsv` maps ordinals → new hashes; the scratch `.run/public_rewrite/` holds the old ones and stays until the probe passes). **Never `git clean -x`** (CLAUDE.md fail-safe). ### 1. Where we are **Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). The approved plan -is VERBATIM at the end of this file — its Blocks E–G paragraphs are the specs for what remains. **Done (30):** A1–A5, B1–B9/C3, +is VERBATIM at the end of this file — its Blocks E–G paragraphs are the specs for what remains. **Done (31):** A1–A5, B1–B9/C3, C1–C9 (the rewrite, adopted, force-pushed by Drew, gc'd), D1–D5 (README, LICENSE/NOTICE/THIRD_PARTY, badges/objdiff/frogress, SETUP public-clean, governing docs + `doc_links`), F1 (timeline + story), F2 (retrospective), **F3 (S88: the wiki — 12 files under `docs/wiki/` + the 13 how-to chapters under `docs/how-to-ai-decomp/`, `tools/wiki_render.py` + `tools/wiki_sync.sh`, `doc_links` DEFAULT_GLOBS; `954362c81e` + `0cf971d1f4`), **E3 (S88: `docs/gcc-2.7.2-map/README.md` + `tools/gccmap_cites.py` + `cite_overrides.tsv`; all 135 map citations tagged `[2.7.2]`/`[2.8.1 pm]`/`[repo]` — 79/55/1 — verified, controls 6/6; -the commit that carries this block).** **In progress (Drew, C10):** the GitHub +`50c1b69e4d`), **E4 (S88: `tools/xsig/` — `xsig.py` library+CLI, README with the Phase-21 worked example, MIT LICENSE, +`tests/` from a game-free fixture at two link addresses, 8/8; in tools-health + CI; the standalone copy at `.run/P33/xsig-repo/` +with one commit under the noreply identity — Drew creates `Druthulu/xsig` EMPTY and pushes; the commit that carries this +block).** **In progress (Drew, C10):** the GitHub Support ticket (text: `docs/public-flip-runbook.md` §11 — its filing was never confirmed to S88; ask) and the daily `tools/public_rewrite/probe_github.sh` until it prints PASS (S88's run: **31 of 33 old hashes still ALIVE = the S87 baseline; -no purge yet**). **Remaining (10):** E4 (xsig) · E5 (permuter PR branch) · E6 (drafter write-up) — all on the still-private repo; then, gated on the probe PASS: C10 (the flip — Drew), E1 (decomp.me preset +no purge yet**). **Remaining (9):** E5 (permuter PR branch) · E6 (drafter write-up) — on the still-private repo; then, gated on the probe PASS: C10 (the flip — Drew), E1 (decomp.me preset — Drew), E2 (Archipelago — Drew), D3's outward actions (decomp.dev registration, frogress slug/key — Drew), **the wiki push (Drew: Wiki → "Create the first page" in the GitHub UI, then `tools/wiki_sync.sh --push`)**; then C11 (aftercare), G1 (`docs/gen3-handoff.md`), G2 (the PhaseEnd v2.0.0 + DIGEST + `v2.0.0` tag; Tier 1; WAIT for gate 2). ### 2. Facts the remaining tasks depend on (measured S88; verify if in doubt, R14) - **Repository state:** `main` = the rewritten history (4,031 commits) + the S87 tip commits (C7 → F2) + the S88 commits - (`214d0dd15b` the probe fix, `954362c81e` F3 pages + tooling, `0cf971d1f4` the F3 wiring, the E3 commit = HEAD); `origin/main` == the F2 commit + (`214d0dd15b` the probe fix, `954362c81e` F3 pages + tooling, `0cf971d1f4` the F3 wiring, `50c1b69e4d` E3, the E4 commit = HEAD); `origin/main` == the F2 commit `5e57e88de2` — **Drew pushed the S87 tip on 2026-09-07 07:23Z; the S88 commits are NOT pushed** (a normal fast-forward push; R6). The first-ever GitHub runs of both workflows were GREEN on that push (`no-rom` 1 m 35 s, run 34095194524; `progress` 15 s, run 34095194479) — read the Actions tab again after the next push, fix red, never claim green unseen (P9). The repo is @@ -565,7 +594,8 @@ no purge yet**). **Remaining (10):** E4 (xsig) · E5 (permuter PR branch) · E6 any new forward link must be listed there with its creating task (gate 2 requires empty). `tools/wiki_render.py --selftest` 12/12; `tools/wiki_sync.sh` dry run: 25 pages, 264 links rewritten, "not clonable yet … dry run OK" (exit 0). `tools/gccmap_cites.py --check` OK / `--verify` 0 disagreements / `--controls` 6/6 (needs both `tools/reference/` trees — - gitignored; `--check` alone is textual and is what CI and tools-health run); `make tools-health` at the E3 close: + gitignored; `--check` alone is textual and is what CI and tools-health run); `tools/xsig/tests/test_xsig.py` 8/8 (in + tools-health + CI); `make tools-health` at the E4 close: `tools-health: OK — sigs fresh; corpus(+resident) + cdecl + binaries + report(lint+dedup) + cookbook-index all green.` (EXIT=0) (`.run/P33/e4_tools_health.log`); at the E3 close: `tools-health: OK — sigs fresh; corpus(+resident) + cdecl + binaries + report(lint+dedup) + cookbook-index all green.` (EXIT=0) (`.run/P33/e3_tools_health.log`). - **Generated, never typed (R51):** `docs/progress.json`, the README block, `docs/badges/*.json`, `docs/story-timeline.md/.svg`, `docs/commit-map.tsv`, `config/ghidra/ROSTER.md`, `docs/progress*.md`, `docs/cookbook-index.md` — regenerate with @@ -592,9 +622,9 @@ no purge yet**). **Remaining (10):** E4 (xsig) · E5 (permuter PR branch) · E6 Druthulu/BFM-decomp --limit 4`) · `git count-objects -v` (packs: 1 after Drew's gc; 30 = not yet run) · `df -h ~` · `.venv/bin/python tools/doc_links.py --strict` (PASS) · ask Drew: pushed? gc run? ticket filed? latest probe result? (`tools/public_rewrite/probe_github.sh` — ~1 min, gh-authenticated, safe to run from Claude since S88). -2. **E3 is DONE** (see the log). **E4** (xHigh, 0.5): `.run/xdedup/xsig.py` → `tools/xsig/` (library + CLI `sign-s | sign-objdump | cross | verify`, MIT - `LICENSE`, README with the Xenogears/Vagrant Story/Tomba hits, `tests/` from one 10-line C function compiled at two link - addresses — no game bytes); standalone repo prepared under `.run/P33/xsig-repo/` (Drew creates + pushes). **E5** (Max, 1): +2. **E3 and E4 are DONE** (see the log; E4's outward action — Drew creates `Druthulu/xsig` EMPTY on GitHub, then + `git -C .run/P33/xsig-repo remote add origin https://github.com/Druthulu/xsig.git && git -C .run/P33/xsig-repo push -u origin + main` — is pending until Drew does it; the `.run/P33/xsig-repo/` copy is regenerable from `tools/xsig/`). **E5** (Max, 1): the permuter upstream PR branch (`RelocMaskedScorer` behind `--score-mode reloc-masked`; upstream's `Scorer.__init__` gained `ign_branch_targets, objdump_command`; the MIPS symbol wildcard `"." in field` at `scorer.py:66-67`; fixture tests, mypy, black, `./run-tests.sh`; PR-2/issue: configurable `symbol_regex`; `docs/permuter-ils.md`); Drew opens the issue/PR. **E6** @@ -615,15 +645,17 @@ no purge yet**). **Remaining (10):** E4 (xsig) · E5 (permuter PR branch) · E6 the milestone evidence, WAIT for gate 2; then `PhaseEnd_Phase33.md` v2.0.0 with the rule candidates (a)–(h), `CURRENT_PHASE.md` → `phase-ends/logs/Phase33.md`, DIGEST §0/§2/§3 appended, the annotated `v2.0.0` tag; Drew pushes `main --tags`). -### 4. Files S88 touched (4 commits after the F2 tip) -Tools (new): `tools/wiki_render.py`, `tools/wiki_sync.sh`, `tools/gccmap_cites.py`; changed: `tools/public_rewrite/probe_github.sh` (scratch-repo fetch + -self-check), `tools/doc_links.py` (DEFAULT_GLOBS + the map README), `Makefile` (`wiki_render --selftest` + `gccmap_cites --check` in -tools-health), `.github/workflows/no-rom.yml` (the gccmap_cites step). Docs (new): `docs/wiki/*.md` (12), +### 4. Files S88 touched (5 commits after the F2 tip) +Tools (new): `tools/wiki_render.py`, `tools/wiki_sync.sh`, `tools/gccmap_cites.py`, `tools/xsig/` (xsig.py, README, LICENSE, +tests/: fixture.c, make_fixtures.sh, test_xsig.py, fixture_a.txt, fixture_b.txt, fixture_a.s); changed: `tools/public_rewrite/probe_github.sh` (scratch-repo fetch + +self-check), `tools/doc_links.py` (DEFAULT_GLOBS + the map README), `Makefile` (`wiki_render --selftest` + `gccmap_cites --check` + the xsig tests in +tools-health), `.github/workflows/no-rom.yml` (the gccmap_cites + xsig steps). Docs (new): `docs/wiki/*.md` (12), `docs/how-to-ai-decomp/*.md` (13), `docs/gcc-2.7.2-map/README.md` + `cite_overrides.tsv`; changed: the five map files (135 cites tagged in place), `docs/SETUP.md` (the probe clause in the public_rewrite row; 2 wiki rows; the P33 F3 section), `docs/public-flip-runbook.md` (§11: the R57 probe paragraph; the wiki push step), `docs/doc_links_pending.txt` (10 entries mid-task → EMPTY), `phase-ends/CURRENT_PHASE.md` (F3 ticked; the S88 preflight + F3 log -entries; this block). Evidence: `.run/P33/f3_tools_health.log`, `.run/P33/e3_tools_health.log`, `.run/wiki/render/` (regenerable). The plan file: +entries; this block). Evidence: `.run/P33/f3_tools_health.log`, `.run/P33/e3_tools_health.log`, `.run/P33/e4_tools_health.log`, `.run/wiki/render/` +(regenerable), `.run/P33/xsig-repo/` (the standalone copy, one commit). The plan file: `~/.claude/plans/max-effort-set-plan-twinkling-moonbeam.md` (copied below). --- diff --git a/tools/doc_links.py b/tools/doc_links.py index 5eb7505a8..0edb0872f 100644 --- a/tools/doc_links.py +++ b/tools/doc_links.py @@ -20,7 +20,7 @@ REPO = pathlib.Path(__file__).resolve().parent.parent DEFAULT = ["README.md", "THIRD_PARTY.md", "CLAUDE.md", "src/NOTICE.md", "tools/README.md", "docs/SETUP.md", "docs/verification.md", "docs/public-flip-runbook.md", "docs/decision-log.md", "docs/accelerators.md", "docs/story.md", "docs/story-timeline.md", "docs/retrospective.md", "phase-ends/README.md", "phase-ends/DIGEST.md", - "docs/gcc-2.7.2-map/README.md"] + "docs/gcc-2.7.2-map/README.md", "tools/xsig/README.md"] # whole directories in the default set (P33 F3): the wiki pages and the how-to chapters — every file, so a new page is # checked the moment it exists (the glob is expanded at run time; the count is printed with the rest, R41) DEFAULT_GLOBS = ["docs/wiki/*.md", "docs/how-to-ai-decomp/*.md"] diff --git a/tools/xsig/LICENSE b/tools/xsig/LICENSE new file mode 100644 index 000000000..90774273c --- /dev/null +++ b/tools/xsig/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Drew T (Druthulu) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/xsig/README.md b/tools/xsig/README.md new file mode 100644 index 000000000..f5abb5dd2 --- /dev/null +++ b/tools/xsig/README.md @@ -0,0 +1,106 @@ +# xsig — relocation-masked function signatures for cross-project code identification + +Two projects that compile the **same source with the same compiler** emit byte-identical machine code except in the +operand fields the linker fills in — `j`/`jal` targets and `%hi`/`%lo` address halves — which differ by link address. +`xsig` hashes a per-function signature that masks exactly those fields and keeps everything else verbatim (opcodes, +registers, real constants, PC-relative branch offsets), so a shared signature means *the same function, linked +somewhere else*. It was written for the [Brave Fencer Musashi decompilation](https://github.com/Druthulu/BFM-decomp) +to ask one question of its unmatched functions: *does any sibling decompilation already have this body?* + +MIPS (PlayStation-era) only. Single stdlib Python file, no dependencies. MIT. + +## What it answers, and what it found + +The question a byte-exact decompilation asks of its neighbours is not "is this code similar?" but "is this code the +same object?" — because if it is, the neighbour's C compiles to your bytes and the function costs nothing. In Phase 21 +of BFM-decomp the unmatched functions of every binary were signed and joined against three sibling decompilations built +with the same gcc 2.7.2 toolchain — [Xenogears](https://github.com/ladysilverberg/xenogears-decomp), +[Vagrant Story](https://github.com/ser-pounce/rood-reverse) and [Tomba!](https://github.com/hansbonini/psx_tomba) — +each compiled to objects and listed with `objdump -dr`: + +| Join | Hits | What they were | +|---|---|---| +| BFM × Xenogears + Vagrant Story | 103 (57 + 46) | every one a **PsyQ library or BIOS function** — `libapi` 34, `libcd` 14, `libetc` 14, `libspu` 11, `libcard` 7, the rest `libc2`/`libsn`/other SDK units; the largest `_spu_setReverbAttr` at 307 instructions; 37 of the 103 BFM functions were already named as library code | +| BFM × Tomba! | 126 | 124 library (77 high-confidence), **one** high-confidence non-library hit — a 19-instruction function — and one 2-instruction shape shared by 16 locations (a generic epilogue) | + +The recorded verdict (BFM's decision log, 2026-07): *a clean negative — shared byte-identical code exists only in the +PsyQ library objects, zero engine code.* That closed a question with byte evidence in an afternoon, and the library +hits fed the project's other lever: link Sony's real objects instead of decompiling them. The tool is published because +the *question* transfers to any decompilation with same-compiler neighbours, even when the answer is "no". + +## The signature + +``` +sig = sha1( little-endian masked word stream, trailing all-zero (nop) words stripped ) +``` + +- `j` / `jal` (opcode 2/3): keep the 6-bit opcode, mask the 26-bit target — by opcode, no relocation record needed. +- An instruction carrying a **HI16/LO16 relocation**: keep opcode + `rs` + `rt`, mask the low 16 bits. The relocation is + read from the input, never guessed: a splat-style `.s` line whose operands contain `%hi(` / `%lo(`, or an + `objdump -dr` listing whose next line names `R_MIPS_HI16` / `R_MIPS_LO16`. +- Everything else is kept. **Registers are never masked** (masking them is too loose), a constant `lui` stays a + constant, a branch offset is position-independent and stays. +- Trailing nops are inter-function alignment; stripping them on both sides removes a boundary-dependent length + ambiguity and cannot forge a match. + +Symmetric, explicit, and tighter than a heuristic that tracks `lui`/`addiu` pairs (which over-masks a genuine constant +`lui`). The price is the input: both sides need relocation information — a splitter's `.s` with `%hi`/`%lo`, or an object +listing — and a linked binary without emitted relocations cannot be signed faithfully. + +## Usage + +```bash +# sign a splitter's per-function .s files (one function per file), deduplicated by signature with a location list +python3 xsig.py sign-s asm/ --project bfm --dedup -o bfm.jsonl + +# sign every function in objdump -dr listings of a sibling's objects +mipsel-linux-gnu-objdump -dr build/libcd.o > libcd.txt +python3 xsig.py sign-objdump libcd.txt other.txt --project sibling -o sibling.jsonl + +# join: the same signature on both sides = the same function +python3 xsig.py cross bfm.jsonl sibling.jsonl --min-ins 8 + +# read the diff before believing any score: instruction-by-instruction comparison of two functions +python3 xsig.py verify asm/func_80044B14.s libcd.txt --name-b CD_flush + +# the built-in property checks (no files) +python3 xsig.py selftest +``` + +Records are JSON Lines: `{project, tu, name, kind, sig, nins, nins_raw, mnem_head, path}` (`--dedup` adds `n_loc` +and `locs`). `cross` prints a TSV of hits sorted by instruction count, preceded by a coverage line (records and unique +signatures on each side — a join without its denominators is a number about an unknown world). `verify` exits 0 only +when the two functions are identical up to relocation and classifies every differing instruction as `opcode`, +`register`, `immediate` or `length`. + +Library use: `import xsig` — `signature(pairs)`, `parse_splat_s(path)`, `parse_objdump_dr(text)`, +`compare(pairs_a, pairs_b)`, where `pairs` is a list of `(raw_word, is_address_immediate)`. + +## Tests + +`python3 tests/test_xsig.py` — eight checks on committed fixtures that contain **no game bytes**: one small function of +our own ([`tests/fixture.c`](tests/fixture.c)) compiled once with the gcc 2.7.2 toolchain and linked at two different +addresses with `--emit-relocs`, listed with `objdump` (the relocation records merged into the listing by +[`tests/make_fixtures.sh`](tests/make_fixtures.sh)), plus a splat-style `.s` rendering of the same function and a second, +different function. The tests prove: the same function at two link addresses signs identically while its raw words +differ; the relocated fields are present and are the only masked ones; a different function signs differently; a +one-bit register flip changes the signature and is classified as `register`; the two front-ends agree; the CLI's +`sign-objdump` + `cross` find exactly the two true pairs; `verify` accepts the pair and rejects the mismatch. + +## Limits + +- Same compiler, same flags, same conventions on both sides. Xenogears had to be signed under both its faithful `-G8` + and BFM's `-G0` convention; a different `-G` changes `gp`-relative addressing and the bytes. +- A function is identified, not understood: a hit says the *bytes* are shared. Small generic shapes (a two-instruction + epilogue) collide across unrelated functions — filter by instruction count, and read `verify`'s diff. +- MIPS I encodings; the relocation kinds handled are `R_MIPS_26`, `R_MIPS_HI16`, `R_MIPS_LO16` (PC16 is kept on purpose). + +## Provenance + +Written 2026-06-25 (BFM-decomp Phase 21, session "xdedup"), packaged 2026-09-07 (Phase 33). The method, the campaign it +served and the records it cites are in the BFM-decomp repository: the decision log, `docs/how-to-ai-decomp/`, and the +SETUP inventory row for `tools/xsig/`. + +## License + +MIT — see [`LICENSE`](LICENSE). diff --git a/tools/xsig/tests/fixture.c b/tools/xsig/tests/fixture.c new file mode 100644 index 000000000..b3c0b0b21 --- /dev/null +++ b/tools/xsig/tests/fixture.c @@ -0,0 +1,20 @@ +/* tests/fixture.c — the test fixture's source. NOT game code: two small functions written for the test, compiled with + * the toolchain xsig was built for (gcc-2.7.2-psx cc1 -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker → + * maspsx --aspsx-version=2.56 --expand-div → mipsel-linux-gnu-as) and linked TWICE at different addresses with + * --emit-relocs (see make_fixtures.sh). `fixture` exercises every field xsig masks — a jal (26-bit target), a %hi/%lo + * pair for a global (HI16/LO16) — and one it keeps (a PC-relative branch); `other` is the discrimination control. */ +extern int table[8]; +extern int helper(int); + +int fixture(int n) { + int i, acc = 0; + for (i = 0; i < n; i++) { + acc += table[i & 7]; + if (acc > 100) acc = helper(acc); + } + return acc; +} + +int other(int a, int b) { + return (a * 3) ^ (b >> 2); +} diff --git a/tools/xsig/tests/fixture_a.s b/tools/xsig/tests/fixture_a.s new file mode 100644 index 000000000..c9401f751 --- /dev/null +++ b/tools/xsig/tests/fixture_a.s @@ -0,0 +1,36 @@ +glabel fixture +/* 0 80010000 D8FFBD27 */ addiu sp,sp,-40 +/* 4 80010004 1C00B1AF */ sw s1,28(sp) +/* 8 80010008 21888000 */ move s1,a0 +/* C 8001000C 1800B0AF */ sw s0,24(sp) +/* 10 80010010 21800000 */ move s0,zero +/* 14 80010014 21200000 */ move a0,zero +/* 18 80010018 2400BFAF */ sw ra,36(sp) +/* 1C 8001001C 1300201A */ blez s1,8001006c +/* 20 80010020 2000B2AF */ sw s2,32(sp) +/* 24 80010024 0280123C */ lui s2,%hi(table) +/* 28 80010028 00005226 */ addiu s2,s2,%lo(table) +/* 2C 8001002C 07000232 */ andi v0,s0,0x7 +/* 30 80010030 80100200 */ sll v0,v0,0x2 +/* 34 80010034 21105200 */ addu v0,v0,s2 +/* 38 80010038 0000428C */ lw v0,0(v0) +/* 3C 8001003C 00000000 */ nop +/* 40 80010040 21208200 */ addu a0,a0,v0 +/* 44 80010044 65008228 */ slti v0,a0,101 +/* 48 80010048 04004014 */ bnez v0,8001005c +/* 4C 8001004C 00000000 */ nop +/* 50 80010050 0048000C */ jal 80012000 +/* 54 80010054 00000000 */ nop +/* 58 80010058 21204000 */ move a0,v0 +/* 5C 8001005C 01001026 */ addiu s0,s0,1 +/* 60 80010060 2A101102 */ slt v0,s0,s1 +/* 64 80010064 F2FF4014 */ bnez v0,80010030 +/* 68 80010068 07000232 */ andi v0,s0,0x7 +/* 6C 8001006C 21108000 */ move v0,a0 +/* 70 80010070 2400BF8F */ lw ra,36(sp) +/* 74 80010074 2000B28F */ lw s2,32(sp) +/* 78 80010078 1C00B18F */ lw s1,28(sp) +/* 7C 8001007C 1800B08F */ lw s0,24(sp) +/* 80 80010080 2800BD27 */ addiu sp,sp,40 +/* 84 80010084 0800E003 */ jr ra +/* 88 80010088 00000000 */ nop diff --git a/tools/xsig/tests/fixture_a.txt b/tools/xsig/tests/fixture_a.txt new file mode 100644 index 000000000..5b12aac33 --- /dev/null +++ b/tools/xsig/tests/fixture_a.txt @@ -0,0 +1,52 @@ + +/home/musashi/bfm-decomp/tools/xsig/tests/.fx.3YqO/a.elf: file format elf32-tradlittlemips + + +Disassembly of section .text: + +80010000 : +80010000: 27bdffd8 addiu sp,sp,-40 +80010004: afb1001c sw s1,28(sp) +80010008: 00808821 move s1,a0 +8001000c: afb00018 sw s0,24(sp) +80010010: 00008021 move s0,zero +80010014: 00002021 move a0,zero +80010018: afbf0024 sw ra,36(sp) +8001001c: 1a200013 blez s1,8001006c +80010020: afb20020 sw s2,32(sp) +80010024: 3c128002 lui s2,0x8002 + 80010024: R_MIPS_HI16 table +80010028: 26520000 addiu s2,s2,0 + 80010028: R_MIPS_LO16 table +8001002c: 32020007 andi v0,s0,0x7 +80010030: 00021080 sll v0,v0,0x2 +80010034: 00521021 addu v0,v0,s2 +80010038: 8c420000 lw v0,0(v0) +8001003c: 00000000 nop +80010040: 00822021 addu a0,a0,v0 +80010044: 28820065 slti v0,a0,101 +80010048: 14400004 bnez v0,8001005c +8001004c: 00000000 nop +80010050: 0c004800 jal 80012000 + 80010050: R_MIPS_26 helper +80010054: 00000000 nop +80010058: 00402021 move a0,v0 +8001005c: 26100001 addiu s0,s0,1 +80010060: 0211102a slt v0,s0,s1 +80010064: 1440fff2 bnez v0,80010030 +80010068: 32020007 andi v0,s0,0x7 +8001006c: 00801021 move v0,a0 +80010070: 8fbf0024 lw ra,36(sp) +80010074: 8fb20020 lw s2,32(sp) +80010078: 8fb1001c lw s1,28(sp) +8001007c: 8fb00018 lw s0,24(sp) +80010080: 27bd0028 addiu sp,sp,40 +80010084: 03e00008 jr ra +80010088: 00000000 nop + +8001008c : +8001008c: 00041040 sll v0,a0,0x1 +80010090: 00441021 addu v0,v0,a0 +80010094: 00052883 sra a1,a1,0x2 +80010098: 03e00008 jr ra +8001009c: 00451026 xor v0,v0,a1 diff --git a/tools/xsig/tests/fixture_b.txt b/tools/xsig/tests/fixture_b.txt new file mode 100644 index 000000000..bc8500c20 --- /dev/null +++ b/tools/xsig/tests/fixture_b.txt @@ -0,0 +1,52 @@ + +/home/musashi/bfm-decomp/tools/xsig/tests/.fx.3YqO/b.elf: file format elf32-tradlittlemips + + +Disassembly of section .text: + +80200000 : +80200000: 27bdffd8 addiu sp,sp,-40 +80200004: afb1001c sw s1,28(sp) +80200008: 00808821 move s1,a0 +8020000c: afb00018 sw s0,24(sp) +80200010: 00008021 move s0,zero +80200014: 00002021 move a0,zero +80200018: afbf0024 sw ra,36(sp) +8020001c: 1a200013 blez s1,8020006c +80200020: afb20020 sw s2,32(sp) +80200024: 3c128023 lui s2,0x8023 + 80200024: R_MIPS_HI16 table +80200028: 26520000 addiu s2,s2,0 + 80200028: R_MIPS_LO16 table +8020002c: 32020007 andi v0,s0,0x7 +80200030: 00021080 sll v0,v0,0x2 +80200034: 00521021 addu v0,v0,s2 +80200038: 8c420000 lw v0,0(v0) +8020003c: 00000000 nop +80200040: 00822021 addu a0,a0,v0 +80200044: 28820065 slti v0,a0,101 +80200048: 14400004 bnez v0,8020005c +8020004c: 00000000 nop +80200050: 0c084800 jal 80212000 + 80200050: R_MIPS_26 helper +80200054: 00000000 nop +80200058: 00402021 move a0,v0 +8020005c: 26100001 addiu s0,s0,1 +80200060: 0211102a slt v0,s0,s1 +80200064: 1440fff2 bnez v0,80200030 +80200068: 32020007 andi v0,s0,0x7 +8020006c: 00801021 move v0,a0 +80200070: 8fbf0024 lw ra,36(sp) +80200074: 8fb20020 lw s2,32(sp) +80200078: 8fb1001c lw s1,28(sp) +8020007c: 8fb00018 lw s0,24(sp) +80200080: 27bd0028 addiu sp,sp,40 +80200084: 03e00008 jr ra +80200088: 00000000 nop + +8020008c : +8020008c: 00041040 sll v0,a0,0x1 +80200090: 00441021 addu v0,v0,a0 +80200094: 00052883 sra a1,a1,0x2 +80200098: 03e00008 jr ra +8020009c: 00451026 xor v0,v0,a1 diff --git a/tools/xsig/tests/make_fixtures.sh b/tools/xsig/tests/make_fixtures.sh new file mode 100644 index 000000000..d3d90962c --- /dev/null +++ b/tools/xsig/tests/make_fixtures.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# tests/make_fixtures.sh — regenerate the committed fixtures from tests/fixture.c with the pinned toolchain. +# +# Needs the BFM-decomp checkout's toolchain (run from that repository's root): the vintage cc1 under tools/bin/, +# maspsx under tools/maspsx/, and mipsel-linux-gnu binutils. The OUTPUTS are what the tests read — plain objdump +# listings and a splat-style .s — so the tests themselves need no compiler and contain no game bytes. +# +# fixture_a.txt objdump -dr of fixture.o linked at 0x80010000 (helper @0x80012000, table @0x80020000) +# fixture_b.txt the same object linked at 0x80200000 (helper @0x80212000, table @0x80230000) +# fixture_a.s fixture() from fixture_a.txt rewritten in splat's `/* off vaddr WORD */ mnemonic operands` form, +# with %hi()/%lo() operands where the listing carries R_MIPS_HI16/LO16 +# The two listings differ ONLY in relocated fields (link addresses); xsig must sign them identically. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="${BFM_REPO:-$(cd "$HERE/../../.." && pwd)}" +CC1="$REPO/tools/bin/gcc-2.7.2-psx/cc1"; MASPSX="$REPO/tools/maspsx/maspsx.py"; PY="${PY:-$REPO/.venv/bin/python}" +[ -x "$CC1" ] || { echo "make_fixtures: $CC1 missing (tools/bootstrap.sh in the BFM checkout)"; exit 2; } +W="$(mktemp -d "$HERE/.fx.XXXX")"; trap 'rm -rf "$W"' EXIT +mipsel-linux-gnu-cpp -lang-c -undef -Wall -fno-builtin -Dmips -D__GNUC__=2 -D__OPTIMIZE__ -Dpsx -D_PSYQ -D_MIPSEL -D_LANGUAGE_C "$HERE/fixture.c" \ + | "$CC1" -quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker \ + | "$PY" "$MASPSX" --aspsx-version=2.56 --expand-div \ + | mipsel-linux-gnu-as -march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0 -o "$W/fixture.o" +link() { # $1 out.elf $2 text $3 helper $4 table + mipsel-linux-gnu-ld -q -e fixture -Ttext="$2" --defsym "helper=$3" --defsym "table=$4" -o "$1" "$W/fixture.o" +} +link "$W/a.elf" 0x80010000 0x80012000 0x80020000 +link "$W/b.elf" 0x80200000 0x80212000 0x80230000 +# objdump interleaves relocation records into the disassembly only for OBJECT files; for a linked ELF (--emit-relocs) +# it lists them separately (`-r`, section-relative offsets). Merge them in, exactly in the interleaved format an +# object listing has — the format xsig's front-end reads. +listing() { # $1 elf $2 text-vma $3 out + "$PY" - "$1" "$2" "$3" <<'PYEOF' +import re, subprocess, sys +elf, base, out = sys.argv[1], int(sys.argv[2], 16), sys.argv[3] +dis = subprocess.run(["mipsel-linux-gnu-objdump", "-d", elf], capture_output=True, text=True, check=True).stdout +rel = subprocess.run(["mipsel-linux-gnu-objdump", "-r", elf], capture_output=True, text=True, check=True).stdout +recs = {} +sect = None +for ln in rel.splitlines(): + m = re.match(r"RELOCATION RECORDS FOR \[(\S+)\]:", ln) + if m: + sect = m.group(1); continue + m = re.match(r"^([0-9a-f]+)\s+(R_MIPS_\w+)\s+(\S+)", ln) + if m and sect == ".text": + recs.setdefault(base + int(m.group(1), 16), []).append((m.group(2), m.group(3))) +lines = [] +for ln in dis.splitlines(): + lines.append(ln) + m = re.match(r"^\s*([0-9a-f]+):\s+[0-9a-f]{8}\s", ln) + if m: + for typ, sym in recs.get(int(m.group(1), 16), []): + lines.append(f"\t\t\t{m.group(1)}: {typ}\t{sym}") +open(out, "w").write("\n".join(lines) + "\n") +print(f"{out}: {sum(len(v) for v in recs.values())} relocation records merged") +PYEOF +} +listing "$W/a.elf" 0x80010000 "$HERE/fixture_a.txt" +listing "$W/b.elf" 0x80200000 "$HERE/fixture_b.txt" +# the splat-style .s of fixture() from listing A: /* off vaddr LEHEX */ mnemonic operands, %hi/%lo from the reloc lines +"$PY" - "$HERE/fixture_a.txt" "$HERE/fixture_a.s" <<'EOF' +import re, struct, sys +src, dst = sys.argv[1], sys.argv[2] +lines = open(src).read().splitlines() +out, cur, i = [], None, 0 +while i < len(lines): + ln = lines[i] + h = re.match(r"^([0-9a-f]+) <([^>]+)>:", ln) + if h: + cur = h.group(2); i += 1; continue + m = re.match(r"^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(\S+)\s*(.*)", ln) + if m and cur == "fixture": + addr, word, mn, ops = int(m.group(1), 16), int(m.group(2), 16), m.group(3), m.group(4).strip() + reloc = lines[i + 1] if i + 1 < len(lines) and "R_MIPS" in lines[i + 1] else "" + if "_HI16" in reloc: + ops = re.sub(r"0x[0-9a-f]+|-?\d+$", "%hi(table)", ops) if ops else "%hi(table)" + elif "_LO16" in reloc: + ops = re.sub(r"(-?0x[0-9a-f]+|-?\d+)(\(\$\w+\))?$", r"%lo(table)\2", ops) if ops else "%lo(table)" + le = struct.pack("> 26) in xsig.J_OPS for w, _ in pa), "the fixture carries a jal") + c = xsig.compare(pa, fn(B, "fixture")[0]) + self.assertTrue(c["same"]); self.assertEqual(c["diffs"], []); self.assertGreaterEqual(c["relocated"], 3) + + def test_different_function_different_signature(self): + self.assertNotEqual(xsig.signature(fn(A, "fixture")[0])["sig"], xsig.signature(fn(A, "other")[0])["sig"]) + + def test_register_flip_changes_signature_and_is_classified(self): + pa, _ = fn(A, "fixture") + idx = next(i for i, (w, a) in enumerate(pa) if not a and (w >> 26) not in xsig.J_OPS and (w >> 26) != 0) + flipped = list(pa); w, a = flipped[idx]; flipped[idx] = (w ^ (1 << 16), a) + self.assertNotEqual(xsig.signature(flipped)["sig"], xsig.signature(pa)["sig"]) + c = xsig.compare(pa, flipped) + self.assertEqual([d[3] for d in c["diffs"]], ["register"]) + + def test_splat_s_frontend_agrees_with_objdump(self): + ps, _ = xsig.parse_splat_s(S) + self.assertEqual(xsig.signature(ps)["sig"], xsig.signature(fn(A, "fixture")[0])["sig"]) + + def test_cli_sign_and_cross_find_the_pair(self): + out_a, out_b = os.path.join(HERE, ".a.jsonl"), os.path.join(HERE, ".b.jsonl") + try: + with redirect_stderr(io.StringIO()): + self.assertEqual(xsig.main(["sign-objdump", A, "--project", "left", "-o", out_a]), 0) + self.assertEqual(xsig.main(["sign-objdump", B, "--project", "right", "-o", out_b]), 0) + buf = io.StringIO() + with redirect_stdout(buf): + self.assertEqual(xsig.main(["cross", out_a, out_b, "--min-ins", "4"]), 0) + hits = [l for l in buf.getvalue().splitlines() if l and not l.startswith("#") and not l.startswith("nins")] + names = {(l.split("\t")[3], l.split("\t")[6]) for l in hits} + self.assertIn(("fixture", "fixture"), names); self.assertIn(("other", "other"), names) + self.assertNotIn(("fixture", "other"), names) + finally: + for p in (out_a, out_b): + if os.path.exists(p): + os.unlink(p) + + def test_cli_verify(self): + with redirect_stdout(io.StringIO()): + self.assertEqual(xsig.main(["verify", A, B, "--name-a", "fixture", "--name-b", "fixture"]), 0) + self.assertEqual(xsig.main(["verify", A, B, "--name-a", "fixture", "--name-b", "other"]), 1) + + def test_selftest(self): + with redirect_stdout(io.StringIO()): + self.assertEqual(xsig.selftest(), 0) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/tools/xsig/xsig.py b/tools/xsig/xsig.py new file mode 100644 index 000000000..699e0b581 --- /dev/null +++ b/tools/xsig/xsig.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""xsig — relocation-masked per-function signatures for CROSS-PROJECT code identification (MIPS, PlayStation-era). + +Two projects that compile the SAME source with the SAME compiler emit byte-identical machine code EXCEPT in the +relocated operand fields — `j`/`jal` 26-bit targets and `%hi`/`%lo` 16-bit immediates — which differ by link address. +xsig hashes a signature that MASKS exactly those fields and keeps everything else (opcodes, registers, true constants, +PC-relative branch offsets) verbatim, so a shared signature means "the same function, linked somewhere else". + + sig = sha1( little-endian masked word stream, trailing all-zero (nop) words stripped ) + +Masking, driven by EXPLICIT relocation information on both sides: + * j / jal (opcode 2/3): keep the 6-bit opcode, mask the 26-bit target — by opcode, no reloc record needed; + * an instruction carrying a HI16/LO16 relocation: keep opcode+rs+rt, mask the low 16 bits. The relocation is read + from the input: a splat-style `.s` line whose operands contain `%hi(` / `%lo(`, or an `objdump -dr` listing whose + next line names R_MIPS_HI16 / R_MIPS_LO16; + * everything else is kept — registers are NEVER masked (that is too loose), a constant `lui` stays a constant. +Trailing nops are inter-function alignment padding; stripping them on both sides removes a boundary-dependent length +ambiguity and cannot forge a match. Symmetric and tighter than a heuristic that tracks `lui` pairs, which over-masks. + +Library: signature(pairs) · parse_splat_s(path) · parse_objdump_dr(text) · compare(pairs_a, pairs_b) +CLI: + xsig.py sign-s PATH... [--project P] [--dedup] [-o out.jsonl] one function per splat-style .s file + xsig.py sign-objdump FILE... [--project P] [-o out.jsonl] every function in `objdump -dr` texts + xsig.py cross A.jsonl B.jsonl [--min-ins N] join two signature sets on `sig` + xsig.py verify A B [--name-a F] [--name-b G] compare two functions instruction by instruction + xsig.py selftest the built-in property checks (no files) +Input files may be splat-style `.s` (one function) or `objdump -dr` text (many functions, --name to pick one). + +Written for the Brave Fencer Musashi decompilation (github.com/Druthulu/BFM-decomp), where it compared the unmatched +functions against three sibling decompilations built with the same compiler; see README.md. MIT license. +""" +import argparse +import glob +import hashlib +import json +import os +import re +import struct +import sys + +__version__ = "1.0.0" +J_OPS = (2, 3) # j, jal + + +# ----------------------------------------------------------------------------- the signature +def mask_word(word, is_addr_imm): + op = word >> 26 + if op in J_OPS: + return word & 0xFC000000 # keep opcode, mask the 26-bit target + if is_addr_imm: + return word & 0xFFFF0000 # keep opcode + rs + rt, mask the 16-bit immediate + return word & 0xFFFFFFFF # keep everything (registers, constants, branch offsets) + + +def _strip_trailing_nops(masked): + n = len(masked) + while n > 0 and masked[n - 1] == 0: + n -= 1 + return masked[:n] + + +def signature(pairs): + """pairs: [(raw_word, is_addr_imm)] → {sig, nins, nins_raw, masked}; `masked` is the trailing-nop-stripped + masked word list, kept for diffing near-misses.""" + masked = [mask_word(w, a) for (w, a) in pairs] + core = _strip_trailing_nops(masked) + buf = b"".join(struct.pack(" */ +_SLINE = re.compile(r"/\*\s*[0-9A-Fa-f]+\s+[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})\s*\*/\s+(\S.*)") + + +def parse_splat_s(path): + """One function per file (splat's `nonmatchings/*.s`). Returns (pairs, mnemonics).""" + pairs, mnem = [], [] + with open(path, "r", errors="replace") as fh: + for line in fh: + m = _SLINE.search(line) + if not m: + continue + word = struct.unpack("]+)>:") +_INS = re.compile(r"^\s*[0-9a-f]+:\s+([0-9a-f]{8})\s+(.*)") # an object file indents the address, a linked ELF does not + + +def parse_objdump_dr(text): + """Every function in an `objdump -dr` listing → [(name, pairs, mnemonics)]. HI16/LO16 relocation lines flip the + preceding instruction's is_addr flag; R_MIPS_26 is handled by opcode; R_MIPS_PC16 (branches) is kept.""" + out, cur, pairs, mnem = [], None, [], [] + + def flush(): + if cur is not None and pairs: + out.append((cur, [(w, bool(a)) for (w, a) in pairs], list(mnem))) + + for line in text.splitlines(): + h = _HDR.match(line) + if h: + flush() + cur, pairs, mnem = h.group(1), [], [] + continue + mi = _INS.match(line) + if mi: + pairs.append([int(mi.group(1), 16), False]) + mnem.append(mi.group(2).split()[0] if mi.group(2).split() else "") + continue + if "R_MIPS" in line and pairs and (("_HI16" in line) or ("_LO16" in line)): + pairs[-1][1] = True + flush() + return out + + +def load_functions(path, name=None): + """A .s file (one function, named after the file) or an objdump text (many). Returns [(name, pairs, mnem)].""" + if path.endswith(".s"): + pairs, mnem = parse_splat_s(path) + return [(os.path.splitext(os.path.basename(path))[0], pairs, mnem)] + fns = parse_objdump_dr(open(path, errors="replace").read()) + if name is not None: + fns = [f for f in fns if f[0] == name] + if not fns: + sys.exit(f"xsig: no function named {name!r} in {path}") + return fns + + +# ----------------------------------------------------------------------------- compare (the diff a score cannot replace) +def compare(pairs_a, pairs_b): + """Instruction-by-instruction comparison of two functions' masked streams. Returns a dict: + same (bool), nins_a/nins_b, relocated (fields masked on the A side), diffs = [(index, word_a, word_b, kind)] + with kind ∈ {'opcode', 'register', 'immediate', 'length'} — read it before believing any score.""" + sa, sb = signature(pairs_a), signature(pairs_b) + diffs = [] + n = max(len(sa["masked"]), len(sb["masked"])) + for i in range(n): + if i >= len(sa["masked"]) or i >= len(sb["masked"]): + diffs.append((i, sa["masked"][i] if i < len(sa["masked"]) else None, + sb["masked"][i] if i < len(sb["masked"]) else None, "length")) + continue + wa, wb = sa["masked"][i], sb["masked"][i] + if wa == wb: + continue + if (wa >> 26) != (wb >> 26) or ((wa >> 26) == 0 and (wa & 0x3F) != (wb & 0x3F)): + kind = "opcode" + elif (wa & 0x03E00000) != (wb & 0x03E00000) or (wa & 0x001F0000) != (wb & 0x001F0000) \ + or ((wa >> 26) == 0 and (wa & 0x0000F800) != (wb & 0x0000F800)): + kind = "register" + else: + kind = "immediate" + diffs.append((i, wa, wb, kind)) + relocated = sum(1 for (w, a) in pairs_a if a or (w >> 26) in J_OPS) + return {"same": sa["sig"] == sb["sig"], "sig_a": sa["sig"], "sig_b": sb["sig"], "nins_a": sa["nins"], + "nins_b": sb["nins"], "relocated": relocated, "diffs": diffs} + + +# ----------------------------------------------------------------------------- CLI +def _record(project, tu, name, pairs, mnem, path=""): + s = signature(pairs) + return {"project": project, "tu": tu, "name": name, "kind": "c_function", "sig": s["sig"], "nins": s["nins"], + "nins_raw": s["nins_raw"], "mnem_head": " ".join(mnem[:6]), "path": path} + + +def _write(records, out): + fh = open(out, "w") if out else sys.stdout + for r in records: + fh.write(json.dumps(r, separators=(",", ":")) + "\n") + if out: + fh.close() + + +def cmd_sign_s(a): + paths = [] + for p in a.paths: + paths += sorted(glob.glob(os.path.join(p, "**", "*.s"), recursive=True)) if os.path.isdir(p) else [p] + recs, by_sig, empty = [], {}, 0 + for p in paths: + pairs, mnem = parse_splat_s(p) + if not pairs: + empty += 1 + continue + r = _record(a.project, os.path.dirname(p), os.path.splitext(os.path.basename(p))[0], pairs, mnem, p) + if a.dedup: + e = by_sig.get(r["sig"]) + if e is None: + r["n_loc"], r["locs"] = 1, [{"tu": r["tu"], "name": r["name"]}] + by_sig[r["sig"]] = r + else: + e["n_loc"] += 1 + if len(e["locs"]) < 40: + e["locs"].append({"tu": r["tu"], "name": r["name"]}) + else: + recs.append(r) + recs = list(by_sig.values()) if a.dedup else recs + _write(recs, a.out) + print(f"xsig sign-s: {len(paths)} files ({empty} without instructions) -> {len(recs)} records" + + (f" ({len(by_sig)} unique signatures)" if a.dedup else "") + (f" -> {a.out}" if a.out else ""), file=sys.stderr) + return 0 if paths else 2 + + +def cmd_sign_objdump(a): + recs = [] + for f in a.files: + tu = os.path.splitext(os.path.basename(f))[0] + for name, pairs, mnem in parse_objdump_dr(open(f, errors="replace").read()): + recs.append(_record(a.project, tu, name, pairs, mnem, f)) + _write(recs, a.out) + print(f"xsig sign-objdump: {len(a.files)} listings -> {len(recs)} functions" + (f" -> {a.out}" if a.out else ""), + file=sys.stderr) + return 0 if recs else 2 + + +def _load_jsonl(path): + return [json.loads(l) for l in open(path) if l.strip()] + + +def cmd_cross(a): + A, B = _load_jsonl(a.a), _load_jsonl(a.b) + by_sig = {} + for r in B: + by_sig.setdefault(r["sig"], []).append(r) + hits = [(ra, rb) for ra in A for rb in by_sig.get(ra["sig"], []) if ra["nins"] >= a.min_ins] + hits.sort(key=lambda h: -h[0]["nins"]) + print(f"# xsig cross: {len(A)} records ({len(set(r['sig'] for r in A))} unique sigs) x {len(B)} records " + f"({len(by_sig)} unique sigs), min-ins {a.min_ins} -> {len(hits)} hits") + print("nins\tsig\tproject_a\tname_a\ttu_a\tproject_b\tname_b\ttu_b") + for ra, rb in hits: + print(f"{ra['nins']}\t{ra['sig'][:12]}\t{ra.get('project','')}\t{ra['name']}\t{ra.get('tu','')}\t" + f"{rb.get('project','')}\t{rb['name']}\t{rb.get('tu','')}") + return 0 + + +def cmd_verify(a): + fa = load_functions(a.a, a.name_a) + fb = load_functions(a.b, a.name_b) + if len(fa) != 1 or len(fb) != 1: + sys.exit(f"xsig verify: pick one function per side (--name-a / --name-b): A has {len(fa)}, B has {len(fb)}") + (na, pa, ma), (nb, pb, mb) = fa[0], fb[0] + c = compare(pa, pb) + print(f"xsig verify: {na} ({c['nins_a']} ins) vs {nb} ({c['nins_b']} ins) — {c['relocated']} relocated fields masked on A") + for i, wa, wb, kind in c["diffs"]: + fa_ = f"{wa:08x}" if wa is not None else "--------" + fb_ = f"{wb:08x}" if wb is not None else "--------" + print(f" #{i:<4} {fa_} {fb_} {kind:9s} {ma[i] if i < len(ma) else ''}") + kinds = {} + for _, _, _, k in c["diffs"]: + kinds[k] = kinds.get(k, 0) + 1 + print("xsig verify: " + ("IDENTICAL up to relocation" if c["same"] else + "DIFFERENT — " + ", ".join(f"{v} {k}" for k, v in sorted(kinds.items())))) + return 0 if c["same"] else 1 + + +def selftest(): + """Property checks on a synthetic stream (no files, no game bytes): self-match, relocation invariance, + register sensitivity, discrimination, trailing-nop stripping, and the two front-ends agreeing on one function.""" + import random + rng = random.Random(1) + # a synthetic 12-instruction function: lui/addiu %hi/%lo pair, a jal, a branch, ALU, loads, a nop tail + words = [(0x27BDFFE8, False), (0xAFBF0014, False), (0x3C028001, True), (0x24422A40, True), (0x8C430000, False), + (0x0C004A10, False), (0x00000000, False), (0x10400003, False), (0x24630001, False), (0x8FBF0014, False), + (0x03E00008, False), (0x27BD0018, False), (0x00000000, False), (0x00000000, False)] + base = signature(words) + assert signature(words)["sig"] == base["sig"], "self-match" + assert base["nins"] == 12 and base["nins_raw"] == 14, "trailing nops stripped, the delay-slot nop kept" + pert = [] + for w, a in words: + if (w >> 26) in J_OPS: + pert.append(((w & 0xFC000000) | rng.randint(1, 0x3FFFFFF), a)) + elif a: + pert.append(((w & 0xFFFF0000) | rng.randint(1, 0xFFFF), a)) + else: + pert.append((w, a)) + assert signature(pert)["sig"] == base["sig"], "relocation invariance" + reg = list(words); w, a = reg[4]; reg[4] = (w ^ (1 << 16), a) # flip rt of the lw + assert signature(reg)["sig"] != base["sig"], "register sensitivity" + other = [(0x00851021, False), (0x03E00008, False), (0x00000000, False)] + assert signature(other)["sig"] != base["sig"], "discrimination" + c = compare(words, reg) + assert not c["same"] and c["diffs"] and c["diffs"][0][3] == "register", "compare classifies a register flip" + # the two front-ends on the same function + s_lines, od_lines = [], ["00000000 :"] + for i, (w, a) in enumerate(words): + le = struct.pack("