phase7: census duplicate bodies and expose a zero band of false positives

Matching conventions require a duplicate check before registering, because a
shared body is matched once and registered once per address. Phase 6 did that
check by hand and found one 12-byte pair. tools/sf3_dupes now hashes every
derived extent body and groups exact duplicates.

Results: 2284 extents, 65 multi-address groups, 2104 singletons. Only 10 groups
contain code (24 addresses, all exact-graded); 55 are all-zero bodies. The
hand-found pair 0x800262E0/0x800262EC is reproduced as g0002, which is the check
that the census measures what it claims. The largest real groups are 712 bytes
(0x8001084C/0x800189E8) and 436 bytes.

The zero groups are a real finding: 252 extents have all-zero bodies, 245 inside
the zero band 0x80147000..0x80170000. The cause is the inventory's jal grade,
which decodes every word as an instruction -- in a data region a word with
opcode 3 is graded as a call whose target lands in the zero band. The census
flags those groups rather than hiding them, and the worklist must exclude
degenerate bodies.

The census is tracked rather than ignored as the plan said, because it holds
addresses, sizes and grades only (the same class as the tracked inventory and
extents tables) and the worklist must be reproducible from tracked inputs. The
content hash is computed and never written.

Makefile

@@ -27,12 +27,14 @@ REGIONS := config/regions.tsv
 SYMBOLS := config/symbols.tsv
 INVENTORY := config/function_inventory.tsv
 EXTENTS := config/function_extents.tsv
+DUPES := config/duplicate_bodies.tsv
 MATCH := tools/sf3_match
 EXTENTS_TOOL := tools/sf3_extents
+DUPES_TOOL := tools/sf3_dupes
 CODE_OUT := build/code
 EXPECTED_SHA1 := e173426c157384ebf1b6caf8c6fea18a85a14af9
 
-.PHONY: all validate split assemble link binary code gate extents extents-verify test check clean
+.PHONY: all validate split assemble link binary code gate extents extents-verify dupes test check clean
 
 all: binary
 
@@ -102,6 +104,13 @@ extents-verify: validate
 	@"$(EXTENTS_TOOL)" verify --exe "$(EXE)" --inventory "$(INVENTORY)" \
 		--extents "$(EXTENTS)" --regions "$(REGIONS)"
 
+# Duplicate-body census (Phase 7): groups byte-identical bodies so a shared body
+# is matched once and registered once per address.
+dupes: validate
+	@test -f "$(DUPES_TOOL)"
+	@test -f "$(EXTENTS)"
+	@"$(DUPES_TOOL)" census --exe "$(EXE)" --extents "$(EXTENTS)" --out "$(DUPES)" --force
+
 # Verification gates. `test` is synthetic-only and needs no game input;
 # `check` adds the extents check and the full-binary byte gate (which do).
 test:

config/duplicate_bodies.tsv

@@ -0,0 +1,74 @@
+# Syphon Filter 3 (USA) duplicate-body census.
+# Columns: group<TAB>size<TAB>count<TAB>addresses<TAB>grades<TAB>flags.
+# Multi-address groups only; addresses, sizes and grades only; no bytes.
+# A group is evidence of a shared body, not a claim that every member is
+# a real function start: verify a member before registering it.
+# flags=zero means the body is all zero bytes -- a zero-filled region the
+# walk ran through, not a function. Exclude those from matching.
+# Regenerate: ./tools/sf3_dupes census --exe '<exe>' \
+#   --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force
+g0001	8	4	0x80042088,0x80089C4C,0x80089E74,0x800B5AF0	exact,exact,exact,exact	-
+g0002	12	2	0x800262E0,0x800262EC	exact,exact	-
+g0003	12	2	0x80147D74,0x801515F4	fallthrough,fallthrough	zero
+g0004	16	8	0x80147650,0x80149A00,0x8014A170,0x8014A1C0,0x8014AC60,0x8014AC70,0x8014AC80,0x8014AEB0	fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0005	20	2	0x80149478,0x80168288	fallthrough,fallthrough	zero
+g0006	32	3	0x801478A8,0x80147DFC,0x80151660	fallthrough,fallthrough,fallthrough	zero
+g0007	36	3	0x800F7FB4,0x80103C7C,0x80103F84	exact,exact,exact	-
+g0008	36	2	0x800F8F9C,0x80109314	exact,exact	-
+g0009	36	4	0x8014762C,0x80147E1C,0x80169448,0x8016946C	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0010	40	2	0x801540D4,0x8016E4DC	fallthrough,fallthrough	zero
+g0011	48	2	0x80148210,0x8016C808	fallthrough,fallthrough	zero
+g0012	52	2	0x80147D40,0x8016CA70	fallthrough,fallthrough	zero
+g0013	60	2	0x80010810,0x8009C69C	exact,exact	-
+g0014	60	4	0x8014A0F8,0x8014A134,0x801522D0,0x8015F554	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0015	64	6	0x801482B0,0x8014A180,0x8014AE70,0x80151294,0x8016ADF0,0x8016B524	fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0016	72	4	0x801499B8,0x80164824,0x8016ADA8,0x8016E494	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0017	80	2	0x80146D00,0x80169944	fallthrough,fallthrough	zero
+g0018	84	2	0x801694EC,0x8016E2D8	fallthrough,fallthrough	zero
+g0019	88	2	0x8016A004,0x8016C47C	fallthrough,fallthrough	zero
+g0020	92	3	0x801498CC,0x80169490,0x80169994	fallthrough,fallthrough,fallthrough	zero
+g0021	96	4	0x80147CE0,0x80151600,0x8015230C,0x8016C164	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0022	104	2	0x80162AA8,0x80162FCC	fallthrough,fallthrough	zero
+g0023	108	3	0x80149A8C,0x801521EC,0x80153868	fallthrough,fallthrough,fallthrough	zero
+g0024	112	4	0x80146C90,0x80148240,0x8014AE00,0x80151AF4	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0025	116	4	0x80147C6C,0x80154060,0x8016D364,0x8016D818	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0026	120	3	0x80018CB0,0x8001ADC4,0x8009C6D8	exact,exact,exact	-
+g0027	120	2	0x80152258,0x8016D090	fallthrough,fallthrough	zero
+g0028	124	4	0x801471CC,0x80147D80,0x80149A10,0x8016486C	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0029	128	2	0x800FB13C,0x800FBE04	exact,exact	-
+g0030	140	2	0x8009E8D0,0x800A45E0	exact,exact	-
+g0031	144	3	0x80147F30,0x80149928,0x80165FF8	fallthrough,fallthrough,fallthrough	zero
+g0032	148	6	0x8014804C,0x8014AD6C,0x80151CEC,0x801564B4,0x8016AA20,0x8016C850	fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0033	152	4	0x801478C8,0x801520AC,0x80155F64,0x80164AFC	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0034	160	3	0x80151968,0x8016729C,0x8016C0C4	fallthrough,fallthrough,fallthrough	zero
+g0035	164	2	0x801648E8,0x8016CD48	fallthrough,fallthrough	zero
+g0036	168	3	0x80152144,0x8015F4AC,0x8016C27C	fallthrough,fallthrough,fallthrough	zero
+g0037	172	4	0x801480E0,0x80163D78,0x8016A974,0x8016C75C	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0038	176	2	0x801540FC,0x8016B298	fallthrough,fallthrough	zero
+g0039	184	2	0x80149AF8,0x8016C1C4	fallthrough,fallthrough	zero
+g0040	188	3	0x801644AC,0x80164568,0x80166528	fallthrough,fallthrough,fallthrough	zero
+g0041	212	2	0x80169374,0x8016D9AC	fallthrough,fallthrough	zero
+g0042	216	2	0x80149F48,0x8014A020	fallthrough,fallthrough	zero
+g0043	220	2	0x8014AC90,0x801512D4	fallthrough,fallthrough	zero
+g0044	252	5	0x801495A8,0x80161978,0x80161A74,0x801643B0,0x80164F5C	fallthrough,fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0045	256	2	0x801497CC,0x80151B64	fallthrough,fallthrough	zero
+g0046	264	3	0x80149E40,0x80164CF4,0x8016E504	fallthrough,fallthrough,fallthrough	zero
+g0047	268	2	0x80153034,0x8016CF84	fallthrough,fallthrough	zero
+g0048	296	2	0x801496A4,0x80167174	fallthrough,fallthrough	zero
+g0049	312	2	0x80147400,0x80168910	fallthrough,fallthrough	zero
+g0050	352	4	0x80164B94,0x80164DFC,0x801687B0,0x80169540	fallthrough,fallthrough,fallthrough,fallthrough	zero
+g0051	396	2	0x80156328,0x8016C8E4	fallthrough,fallthrough	zero
+g0052	416	2	0x80162690,0x801696A0	fallthrough,fallthrough	zero
+g0053	436	2	0x8001D98C,0x8009C750	exact,exact	-
+g0054	436	2	0x80152C88,0x801532A4	fallthrough,fallthrough	zero
+g0055	440	2	0x80147248,0x80166FBC	fallthrough,fallthrough	zero
+g0056	476	2	0x80149BB0,0x8016B348	fallthrough,fallthrough	zero
+g0057	488	3	0x80150EB0,0x801538D4,0x80159DE8	fallthrough,fallthrough,fallthrough	zero
+g0058	600	2	0x80148D6C,0x80148FC4	fallthrough,fallthrough	zero
+g0059	652	2	0x80147960,0x80163E24	fallthrough,fallthrough	zero
+g0060	656	2	0x80153458,0x80153DD0	fallthrough,fallthrough	zero
+g0061	712	2	0x8001084C,0x800189E8	exact,exact	-
+g0062	812	2	0x80151D80,0x80155FFC	fallthrough,fallthrough	zero
+g0063	816	2	0x80169CD4,0x8016E60C	fallthrough,fallthrough	zero
+g0064	1184	2	0x8014F07C,0x80166088	fallthrough,fallthrough	zero
+g0065	2000	2	0x80165828,0x8016B8F4	fallthrough,fallthrough	zero

docs/PHASE7_DUPES.md

@@ -0,0 +1,112 @@
+# Phase 7 — Duplicate-Body Census
+
+**Scope:** P7-T3 — find every body the executable contains more than once, so a shared body is matched
+once and registered once per address, instead of being found by hand.
+**Status:** complete. Tracked output: `config/duplicate_bodies.tsv`. Tool: `tools/sf3_dupes`
+(20 synthetic tests).
+
+## Why
+
+`docs/MATCHING_CONVENTIONS.md` requires a duplicate check **before** registering a match, because the
+registry's N-rows-to-one-source mechanism exists precisely so a shared body is written once. Phase 6
+performed that check by hand and found exactly one shared body (`0x800262E0` / `0x800262EC`). There was
+no way to know how many others exist, or how large they are.
+
+## Method
+
+Every extent row in `config/function_extents.tsv` is hashed over exactly the bytes
+`[address, end)` that `tools/sf3_extents` derived, and rows are grouped by `(size, body)`. Each member
+keeps its grade, so a consumer can prefer groups whose members are all `exact`. `--exact-only` and
+`--min-size N` restrict the census.
+
+The content hash is computed and **never written**. Grouping is the finding; addresses, sizes and
+grades are the only ROM-derived quantities this project tracks — the same rule
+`config/function_inventory.tsv` and `config/function_extents.tsv` follow. No instruction bytes appear
+in the output.
+
+## Results
+
+| Measure | Count |
+|---|---|
+| Extents censused | 2,284 |
+| Multi-address groups | 65 |
+| — of which bodies are entirely zero bytes (`flags=zero`) | 55 |
+| — of which bodies contain code | **10** |
+| Addresses inside groups | 180 |
+| Singleton bodies | 2,104 |
+| Largest group | 8 addresses |
+
+Restricting to `exact` extents yields **the same 10 groups**, so every real duplicate group consists of
+`exact`-graded members.
+
+### The ten real duplicate groups
+
+| Group | Size | Members | Grades |
+|---|---|---|---|
+| `g0001` | 8 | `0x80042088`, `0x80089C4C`, `0x80089E74`, `0x800B5AF0` | all `exact` |
+| `g0002` | 12 | `0x800262E0`, `0x800262EC` | all `exact` |
+| `g0007` | 36 | `0x800F7FB4`, `0x80103C7C`, `0x80103F84` | all `exact` |
+| `g0008` | 36 | `0x800F8F9C`, `0x80109314` | all `exact` |
+| `g0013` | 60 | `0x80010810`, `0x8009C69C` | all `exact` |
+| `g0026` | 120 | `0x80018CB0`, `0x8001ADC4`, `0x8009C6D8` | all `exact` |
+| `g0029` | 128 | `0x800FB13C`, `0x800FBE04` | all `exact` |
+| `g0030` | 140 | `0x8009E8D0`, `0x800A45E0` | all `exact` |
+| `g0053` | 436 | `0x8001D98C`, `0x8009C750` | all `exact` |
+| `g0061` | 712 | `0x8001084C`, `0x800189E8` | all `exact` |
+
+`g0002` is the pair Phase 6 found by hand: the census reproduces it, which is the check that the
+method is measuring what it claims to measure. `g0061` (712 bytes) and `g0053` (436 bytes) are
+substantial shared bodies — matching one of those registers two functions for the price of one, which
+is a materially better return than the 12-byte pair.
+
+## The finding this census produced: a zero band of false positives
+
+55 of the 65 groups are bodies consisting **entirely of zero bytes**, and they are concentrated in one
+place: 252 extents overall have all-zero bodies, 245 of them inside `0x80147000`–`0x80170000`, a
+160 KB zero-filled band. In that band the grades are 245 `fallthrough` and 1 `indirect`.
+
+The cause is a false positive in the **inventory**, not in the extents walk. `sf3_boundaries` grades
+`jal` by decoding every word in the payload as an instruction; in a data region a word that happens to
+have opcode `3` is graded as a call, and its "target" is an address inside the zero band. The walk then
+starts at that address, decodes zeros (which are `nop`s), continues linearly, and reports a
+`fallthrough` extent ending at the next such candidate. Two such extents of equal length therefore
+contain identical all-zero bytes and group together.
+
+Consequences, recorded rather than fixed in place:
+
+- The inventory's `jal` grade is **not** sufficient on its own in a data region. This is a known limit
+  of Phase 6's grade definition (`docs/PHASE6_BOUNDARIES.md` §Grades) and is now demonstrated with
+  addresses, not asserted.
+- The census flags these groups (`flags=zero`) instead of hiding them, so the finding stays visible.
+- The worklist (P7-T4) must exclude candidates whose body is degenerate; a zero body is not code.
+
+## Limits
+
+- A duplicate group is **evidence of a shared body**, not a claim that every member is a real function
+  start. A shared *tail* reached by several functions is not a duplicate function, and a member still
+  has to be verified through the ordinary workflow.
+- The census inherits the extents' limits: a group built on `fallthrough`/`indirect`/`escape` extents
+  is only as reliable as those bounds. In practice every real group here is all-`exact`.
+- Two different functions that happen to compile to identical bytes are indistinguishable from a
+  shared body by this method. That is the correct behaviour for the registry (one source, N rows), but
+  it is a statement about bytes, not about intent.
+- `--min-size` exists because very short bodies collide for boring reasons; the default is 1, so the
+  tracked census is unfiltered.
+
+## Regenerate
+
+```bash
+make dupes
+# or, explicitly:
+./tools/sf3_dupes census --exe 'extracted/SCUS_946.40;1' \
+  --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force
+```
+
+## Deviation from the plan
+
+`phase-ends/Phase7_PLAN.md` said the census artifact would be **ignored** and only aggregate counts
+tracked. It is tracked instead, with the content hash still never written. The reason: the tracked file
+contains addresses, sizes and grades only — the same class of information as the already-tracked
+inventory and extents tables — and P7-T4's worklist must be reproducible from tracked inputs alone.
+Keeping the grouping in an ignored file would have made the worklist unreproducible for exactly the
+reason the plan wanted it reproducible. The plan's intent (no ROM content committed) is preserved.

phase-ends/CURRENT_PHASE.md

@@ -11,7 +11,7 @@ function bodies, bringing the project past **thirty** distinct byte-identical fu
 
 - [x] **P7-T1 — Phase control records, baseline revalidation, and open-item triage** (complete)
 - [x] **P7-T2 — Evidence-graded function extents** (complete)
-- [ ] **P7-T3 — Duplicate-body census**
+- [x] **P7-T3 — Duplicate-body census** (complete)
 - [ ] **P7-T4 — Candidate triage worklist**
 - [ ] **Rules check**
 - [ ] **P7-T5 — Symbol rows at scale**
@@ -115,6 +115,47 @@ of a called entry", equally true of library and game code); `standalone` (438) i
 15 `escape` cases are clipped conservatively with the crossing instruction recorded but the cause not
 determined per case; and nothing here promotes a candidate to a match.
 
+## P7-T3 — Duplicate-body census (2026-09-23)
+
+**Delivered:**
+
+- `tools/sf3_dupes` — hashes the body of every derived extent and groups byte-identical bodies;
+  20 synthetic tests.
+- `config/duplicate_bodies.tsv` — tracked census of multi-address groups,
+  `group<TAB>size<TAB>count<TAB>addresses<TAB>grades<TAB>flags`. Addresses, sizes and grades only; the
+  content hash is computed and never written.
+- `make dupes`; `docs/PHASE7_DUPES.md`.
+
+**Results:** 2,284 extents → 65 multi-address groups, of which **10 contain code** (24 addresses) and
+55 are all-zero bodies; 2,104 singleton bodies. Restricting to `exact` extents yields the same 10
+groups. `g0002` is the `0x800262E0`/`0x800262EC` pair Phase 6 found by hand — the census reproduces it.
+The largest real groups are `g0061` (712 bytes, `0x8001084C`/`0x800189E8`) and `g0053` (436 bytes).
+
+**Finding:** 252 extents have all-zero bodies, 245 of them inside the zero band
+`0x80147000`–`0x80170000` (160 KB), graded `fallthrough`. Cause: the **inventory's** `jal` grade comes
+from decoding every word as an instruction, so a data word with opcode `3` is graded as a call and its
+"target" lands in the zero band. The walk then decodes zeros as `nop`s and reports a `fallthrough`
+extent. Recorded, not hidden: the census flags those groups `flags=zero`, and the P7-T4 worklist must
+exclude degenerate bodies. This demonstrates a Phase 6 grade limit with addresses rather than asserting
+it.
+
+**Verification:**
+
+| Check | Command | Result |
+|---|---|---|
+| Known shared body found | `grep 0x800262E0 config/duplicate_bodies.tsv` | `g0002  12  2  0x800262E0,0x800262EC  exact,exact  -` |
+| Synthetic suite | `python3 -m unittest discover -s tools/tests` | 20 new tests pass |
+| Firewall | output columns | no body bytes and no hash written |
+| Determinism | two `census` runs | identical (group labels ordered by size then first address) |
+
+**Deviation from the plan:** the plan said the census artifact would be ignored and only aggregate
+counts tracked. It is tracked instead, hash still never written: the file holds addresses, sizes and
+grades only (the same class as the tracked inventory and extents tables), and P7-T4's worklist must be
+reproducible from tracked inputs alone. The plan's intent — no ROM content committed — is preserved.
+
+**Limits:** a group is evidence of a shared body, not that every member is a function start; a shared
+*tail* is not a duplicate function; and the census inherits the extents' bounds.
+
 ## Notes and limits
 
 - The four class items are the plan's P7-T2..T5 work; the single-function items are explicitly out of

tools/sf3_dupes

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+"""Duplicate-body census for the USA executable.
+
+Matching conventions require checking for duplicates *before* registering a
+match, because a body shared by several addresses is matched **once** and
+registered once per address (`config/regions.tsv`, N rows to one source). Phase 6
+did that check by hand and found one shared body; there was no way to know how
+many others exist.
+
+This tool hashes the body of every derived extent in
+`config/function_extents.tsv` and groups exact duplicates.
+
+## What is grouped
+
+Every extent row -- any grade -- is hashed over exactly the bytes
+`[address, end)` that `tools/sf3_extents` derived. Each member keeps its grade so
+a consumer can prefer groups whose members are all `exact`. `--exact-only`
+restricts the census to `exact` extents, and `--min-size` drops bodies shorter
+than N bytes (very short bodies collide for boring reasons, e.g. `jr ra; nop`).
+
+## What is written
+
+A tracked, sorted TSV of **multi-address groups only**:
+
+    group<TAB>size<TAB>count<TAB>addresses<TAB>grades<TAB>flags
+
+  * `group` is a stable label (`g0001`, ...) assigned in `(size, first address)`
+    order, so the file is deterministic;
+  * `addresses` is a comma-separated list in ascending order;
+  * `grades` is the comma-separated grade of each address, in the same order;
+  * `flags` is `zero` when the shared body is nothing but zero bytes, otherwise
+    `-`. An all-zero "body" is not a function: it is a zero-filled region the
+    walk ran through, which happens when a data word decodes as a `jal` whose
+    target lands in the zero band. Those groups are recorded, not hidden, so the
+    finding stays visible and the worklist can exclude them.
+
+The content hash is computed and **never written**: grouping is the finding, and
+addresses, sizes and grades are the only ROM-derived quantities this project
+tracks (the same rule `config/function_inventory.tsv` and
+`config/function_extents.tsv` follow). No instruction bytes are read into the
+output, and the executable is read only from the caller-supplied path.
+
+The census is a measurement, not a match claim: two identical bodies are
+evidence of shared code, and nothing more. A shared *tail* is not a duplicate
+function, so a group member still has to be verified as a real function start by
+the ordinary workflow.
+
+Exit codes: 0 success, 2 usage or environment error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+from pathlib import Path
+import struct
+import sys
+from typing import Sequence
+
+
+EXE_MAGIC = b"PS-X EXE"
+HEADER_SIZE = 0x800
+PAYLOAD_LMA = 0x800
+
+EXTENT_GRADES = frozenset({"exact", "fallthrough", "indirect", "escape"})
+
+HEADER_LINES = (
+    "# Syphon Filter 3 (USA) duplicate-body census.",
+    "# Columns: group<TAB>size<TAB>count<TAB>addresses<TAB>grades<TAB>flags.",
+    "# Multi-address groups only; addresses, sizes and grades only; no bytes.",
+    "# A group is evidence of a shared body, not a claim that every member is",
+    "# a real function start: verify a member before registering it.",
+    "# flags=zero means the body is all zero bytes -- a zero-filled region the",
+    "# walk ran through, not a function. Exclude those from matching.",
+    "# Regenerate: ./tools/sf3_dupes census --exe '<exe>' \\",
+    "#   --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force",
+)
+
+
+class ToolError(Exception):
+    """A usage or environment problem; maps to exit code 2."""
+
+
+def parse_hex(text: str, label: str) -> int:
+    try:
+        return int(text, 16)
+    except ValueError as exc:
+        raise ToolError(f"{label}: not a hex address: {text!r}") from exc
+
+
+def require_file(path: Path, label: str) -> Path:
+    if not path.is_file():
+        raise ToolError(f"{label} is not a regular file: {path}")
+    return path
+
+
+def resolve_output(path: Path, force: bool) -> Path:
+    if path.exists() or path.is_symlink():
+        if not force:
+            raise ToolError(f"output already exists (use --force to overwrite): {path}")
+        if not path.is_file() or path.is_symlink():
+            raise ToolError(f"output is not a regular file: {path}")
+    return path
+
+
+def parse_psx_exe(header: bytes) -> tuple[int, int, int]:
+    if len(header) < HEADER_SIZE:
+        raise ToolError("executable is smaller than a PS-X EXE header")
+    if header[:8] != EXE_MAGIC:
+        raise ToolError("executable does not carry the PS-X EXE magic")
+    entry, _gp, text_address, text_size = struct.unpack_from("<IIII", header, 0x10)
+    if text_size == 0:
+        raise ToolError("PS-X EXE header declares an empty payload")
+    return entry, text_address, text_size
+
+
+def load_extents(path: Path) -> list[tuple[int, int, str]]:
+    """Read a generated extents table; keep rows that carry an extent."""
+    rows: list[tuple[int, int, str]] = []
+    for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        line = raw.split("#", 1)[0].strip()
+        if not line:
+            continue
+        fields = line.split("\t")
+        if len(fields) != 7:
+            raise ToolError(f"extents line {number}: expected seven fields")
+        address = parse_hex(fields[0], f"extents line {number}")
+        grade = fields[5]
+        if fields[1] == "-":
+            continue
+        end = parse_hex(fields[1], f"extents line {number}")
+        if end <= address:
+            raise ToolError(f"extents line {number}: end is not after address")
+        if grade not in EXTENT_GRADES:
+            raise ToolError(f"extents line {number}: {grade!r} carries no extent")
+        rows.append((address, end, grade))
+    if not rows:
+        raise ToolError("extents table contains no rows with an extent")
+    return rows
+
+
+class Group:
+    """One set of addresses whose bodies are byte-identical."""
+
+    __slots__ = ("size", "members", "all_zero")
+
+    def __init__(self, size: int, all_zero: bool) -> None:
+        self.size = size
+        self.members: list[tuple[int, str]] = []
+        self.all_zero = all_zero
+
+
+def census(payload: bytes, text_address: int, rows: Sequence[tuple[int, int, str]],
+           min_size: int, exact_only: bool) -> tuple[list[Group], int]:
+    """Group rows by (size, body bytes). Returns the groups and the singleton count."""
+    buckets: dict[tuple[int, bytes], Group] = {}
+    for address, end, grade in rows:
+        if exact_only and grade != "exact":
+            continue
+        size = end - address
+        if size < min_size:
+            continue
+        start = address - text_address
+        body = payload[start:start + size]
+        if len(body) != size:
+            raise ToolError(f"extent 0x{address:08X}..0x{end:08X} is outside the payload")
+        key = (size, hashlib.sha1(body).digest())
+        group = buckets.get(key)
+        if group is None:
+            group = buckets[key] = Group(size, not any(body))
+        group.members.append((address, grade))
+
+    groups = [group for group in buckets.values() if len(group.members) > 1]
+    singletons = sum(1 for group in buckets.values() if len(group.members) == 1)
+    for group in groups:
+        group.members.sort()
+    groups.sort(key=lambda group: (group.size, group.members[0][0]))
+    return groups, singletons
+
+
+def format_census(groups: Sequence[Group]) -> str:
+    lines = list(HEADER_LINES)
+    for index, group in enumerate(groups, 1):
+        addresses = ",".join(f"0x{address:08X}" for address, _grade in group.members)
+        grades = ",".join(grade for _address, grade in group.members)
+        flags = "zero" if group.all_zero else "-"
+        lines.append("\t".join((
+            f"g{index:04d}", str(group.size), str(len(group.members)), addresses, grades, flags,
+        )))
+    return "\n".join(lines) + "\n"
+
+
+def command_census(args: argparse.Namespace) -> int:
+    exe_path = require_file(args.exe, "executable")
+    extents_path = require_file(args.extents, "function extents")
+    out = resolve_output(args.out, args.force)
+    if args.min_size < 0:
+        raise ToolError("--min-size cannot be negative")
+
+    data = exe_path.read_bytes()
+    _entry, text_address, text_size = parse_psx_exe(data)
+    payload = data[PAYLOAD_LMA:PAYLOAD_LMA + text_size]
+    rows = load_extents(extents_path)
+    groups, singletons = census(payload, text_address, rows, args.min_size, args.exact_only)
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(format_census(groups), encoding="ascii")
+
+    addresses = sum(len(group.members) for group in groups)
+    duplicated = sum(group.size * (len(group.members) - 1) for group in groups)
+    zero_groups = sum(1 for group in groups if group.all_zero)
+    print(f"extents={len(rows)}")
+    print(f"groups={len(groups)}")
+    print(f"groups_zero_body={zero_groups}")
+    print(f"groups_with_code={len(groups) - zero_groups}")
+    print(f"addresses_in_groups={addresses}")
+    print(f"singletons={singletons}")
+    print(f"duplicated_bytes={duplicated}")
+    if groups:
+        print(f"largest_group={max(len(group.members) for group in groups)}")
+    print(f"output={out}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    census_parser = subparsers.add_parser("census", help="group byte-identical bodies")
+    census_parser.add_argument("--exe", required=True, type=Path)
+    census_parser.add_argument("--extents", required=True, type=Path,
+                               help="derived function extents (tools/sf3_extents output)")
+    census_parser.add_argument("--out", required=True, type=Path)
+    census_parser.add_argument("--force", action="store_true",
+                               help="overwrite an existing output file")
+    census_parser.add_argument("--min-size", type=int, default=1, dest="min_size",
+                               help="ignore bodies shorter than N bytes (default 1)")
+    census_parser.add_argument("--exact-only", action="store_true",
+                               help="census only extents graded exact")
+    census_parser.set_defaults(handler=command_census)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.handler(args)
+    except ToolError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    except OSError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/tests/test_sf3_dupes.py

@@ -0,0 +1,262 @@
+"""Synthetic-only tests for the duplicate-body census tool.
+
+Fixtures are self-authored bytes in temporary directories. They never read the
+local disc or the extracted game executable.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.machinery
+import importlib.util
+import io
+from pathlib import Path
+import struct
+import sys
+import tempfile
+import unittest
+
+
+TOOL_PATH = Path(__file__).resolve().parents[1] / "sf3_dupes"
+
+
+def _load_tool() -> object:
+    loader = importlib.machinery.SourceFileLoader("sf3_dupes_under_test", str(TOOL_PATH))
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    if spec is None:
+        raise RuntimeError("could not create an import specification")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    loader.exec_module(module)
+    return module
+
+
+sf3_dupes = _load_tool()
+
+PAYLOAD = 0x80010000
+
+# Payload-relative layout of the fixture.
+SHARED_A = 0x00          # 8 bytes of code, duplicated at SHARED_B
+SHARED_B = 0x10
+UNIQUE = 0x20            # 12 bytes, no twin
+ZERO_A = 0x30            # 16 zero bytes, duplicated at ZERO_B
+ZERO_B = 0x40
+
+
+def _payload() -> bytes:
+    words = [0] * (0x50 // 4)
+    words[SHARED_A // 4] = 0x03E00008          # jr ra
+    words[SHARED_B // 4] = 0x03E00008          # jr ra
+    words[UNIQUE // 4] = 0x24020001
+    words[UNIQUE // 4 + 1] = 0x24030002
+    words[UNIQUE // 4 + 2] = 0x03E00008
+    return b"".join(struct.pack("<I", word) for word in words)
+
+
+def _synthetic_exe(payload: bytes, entry: int) -> bytes:
+    header = bytearray(0x800)
+    header[:8] = b"PS-X EXE"
+    struct.pack_into("<IIII", header, 0x10, entry, 0, PAYLOAD, len(payload))
+    return bytes(header) + payload
+
+
+def _extents_text() -> str:
+    lines = [
+        "# synthetic extents",
+        "# Columns: address<TAB>end<TAB>size<TAB>next<TAB>gap<TAB>grade<TAB>evidence.",
+    ]
+
+    def row(offset: int, size: int, grade: str, evidence: str) -> str:
+        start = PAYLOAD + offset
+        end = start + size
+        return f"0x{start:08X}\t0x{end:08X}\t{size}\t-\t-\t{grade}\t{evidence}"
+
+    lines.append(row(SHARED_A, 8, "exact", "term=jr_ra"))
+    lines.append(row(SHARED_B, 8, "exact", "term=jr_ra"))
+    lines.append(row(UNIQUE, 12, "exact", "term=jr_ra"))
+    lines.append(row(ZERO_A, 16, "fallthrough", "hit=0x80010040"))
+    lines.append(row(ZERO_B, 16, "fallthrough", "hit=0x80010050"))
+    lines.append(f"0x{PAYLOAD + 0x60:08X}\t-\t-\t-\t-\tcontained\tinside=0x80010000")
+    return "\n".join(lines) + "\n"
+
+
+def _rows() -> list[tuple[int, int, str]]:
+    return [
+        (PAYLOAD + SHARED_A, PAYLOAD + SHARED_A + 8, "exact"),
+        (PAYLOAD + SHARED_B, PAYLOAD + SHARED_B + 8, "exact"),
+        (PAYLOAD + UNIQUE, PAYLOAD + UNIQUE + 12, "exact"),
+        (PAYLOAD + ZERO_A, PAYLOAD + ZERO_A + 16, "fallthrough"),
+        (PAYLOAD + ZERO_B, PAYLOAD + ZERO_B + 16, "fallthrough"),
+    ]
+
+
+class ParseTests(unittest.TestCase):
+    def test_rejects_missing_magic(self) -> None:
+        with self.assertRaises(sf3_dupes.ToolError):
+            sf3_dupes.parse_psx_exe(bytes(0x800))
+
+    def test_rejects_empty_payload(self) -> None:
+        header = bytearray(0x800)
+        header[:8] = b"PS-X EXE"
+        with self.assertRaises(sf3_dupes.ToolError):
+            sf3_dupes.parse_psx_exe(bytes(header))
+
+
+class ExtentLoadingTests(unittest.TestCase):
+    def test_skips_rows_without_an_extent(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "extents.tsv"
+            path.write_text(_extents_text(), encoding="ascii")
+            rows = sf3_dupes.load_extents(path)
+            self.assertEqual(len(rows), 5)
+
+    def test_rejects_a_row_whose_grade_carries_no_extent(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "extents.tsv"
+            path.write_text(
+                f"0x{PAYLOAD:08X}\t0x{PAYLOAD + 8:08X}\t8\t-\t-\tcontained\tinside=0x1\n",
+                encoding="ascii")
+            with self.assertRaises(sf3_dupes.ToolError):
+                sf3_dupes.load_extents(path)
+
+    def test_rejects_an_empty_extents_table(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "extents.tsv"
+            path.write_text("# nothing\n", encoding="ascii")
+            with self.assertRaises(sf3_dupes.ToolError):
+                sf3_dupes.load_extents(path)
+
+
+class CensusTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.payload = _payload()
+
+    def _groups(self, **kwargs: object) -> list[object]:
+        options = {"min_size": 1, "exact_only": False}
+        options.update(kwargs)
+        groups, _singletons = sf3_dupes.census(
+            self.payload, PAYLOAD, _rows(), options["min_size"], options["exact_only"])
+        return groups
+
+    def test_identical_bodies_are_grouped(self) -> None:
+        groups = self._groups()
+        addresses = [[address for address, _grade in group.members] for group in groups]
+        self.assertIn([PAYLOAD + SHARED_A, PAYLOAD + SHARED_B], addresses)
+        self.assertIn([PAYLOAD + ZERO_A, PAYLOAD + ZERO_B], addresses)
+
+    def test_unique_bodies_are_not_grouped(self) -> None:
+        groups = self._groups()
+        for group in groups:
+            self.assertNotIn(PAYLOAD + UNIQUE, [address for address, _g in group.members])
+
+    def test_a_zero_body_is_flagged(self) -> None:
+        groups = {group.members[0][0]: group for group in self._groups()}
+        self.assertTrue(groups[PAYLOAD + ZERO_A].all_zero)
+        self.assertFalse(groups[PAYLOAD + SHARED_A].all_zero)
+
+    def test_min_size_drops_short_bodies(self) -> None:
+        groups = self._groups(min_size=10)
+        sizes = {group.size for group in groups}
+        self.assertEqual(sizes, {16})
+
+    def test_exact_only_drops_other_grades(self) -> None:
+        groups = self._groups(exact_only=True)
+        self.assertEqual([group.size for group in groups], [8])
+        for group in groups:
+            for _address, grade in group.members:
+                self.assertEqual(grade, "exact")
+
+    def test_singletons_are_counted_not_listed(self) -> None:
+        groups, singletons = sf3_dupes.census(self.payload, PAYLOAD, _rows(), 1, False)
+        self.assertEqual(singletons, 1)
+        self.assertEqual(len(groups), 2)
+
+    def test_groups_are_ordered_by_size_then_address(self) -> None:
+        groups = self._groups()
+        keys = [(group.size, group.members[0][0]) for group in groups]
+        self.assertEqual(keys, sorted(keys))
+
+    def test_an_extent_outside_the_payload_is_rejected(self) -> None:
+        rows = [(PAYLOAD + 0x1000, PAYLOAD + 0x1008, "exact")]
+        with self.assertRaises(sf3_dupes.ToolError):
+            sf3_dupes.census(self.payload, PAYLOAD, rows, 1, False)
+
+
+class FormatTests(unittest.TestCase):
+    def test_header_and_column_count(self) -> None:
+        groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False)
+        text = sf3_dupes.format_census(groups)
+        body = [line for line in text.splitlines() if not line.startswith("#")]
+        self.assertTrue(body)
+        for line in body:
+            self.assertEqual(len(line.split("\t")), 6)
+
+    def test_group_labels_are_stable_and_sorted(self) -> None:
+        groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False)
+        labels = [line.split("\t")[0] for line in sf3_dupes.format_census(groups).splitlines()
+                  if not line.startswith("#")]
+        self.assertEqual(labels, ["g0001", "g0002"])
+
+    def test_no_body_bytes_are_written(self) -> None:
+        groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False)
+        text = sf3_dupes.format_census(groups)
+        self.assertNotIn("03e00008", text.lower())
+        self.assertNotIn("24020001", text.lower())
+
+
+class MainTests(unittest.TestCase):
+    def _write_inputs(self, root: Path) -> tuple[Path, Path]:
+        exe = root / "synthetic.exe"
+        exe.write_bytes(_synthetic_exe(_payload(), PAYLOAD))
+        extents = root / "extents.tsv"
+        extents.write_text(_extents_text(), encoding="ascii")
+        return exe, extents
+
+    def test_census_writes_the_table(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            exe, extents = self._write_inputs(root)
+            out = root / "dupes.tsv"
+            stdout = io.StringIO()
+            with contextlib.redirect_stdout(stdout):
+                rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents),
+                                     "--out", str(out)])
+            self.assertEqual(rc, 0)
+            text = out.read_text(encoding="ascii")
+            self.assertIn(f"0x{PAYLOAD + SHARED_A:08X},0x{PAYLOAD + SHARED_B:08X}", text)
+            self.assertIn("groups=2", stdout.getvalue())
+            self.assertIn("groups_zero_body=1", stdout.getvalue())
+
+    def test_census_refuses_an_existing_output(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            exe, extents = self._write_inputs(root)
+            out = root / "dupes.tsv"
+            out.write_text("", encoding="ascii")
+            rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents),
+                                 "--out", str(out)])
+            self.assertEqual(rc, 2)
+
+    def test_census_force_overwrites(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            exe, extents = self._write_inputs(root)
+            out = root / "dupes.tsv"
+            out.write_text("stale\n", encoding="ascii")
+            with contextlib.redirect_stdout(io.StringIO()):
+                rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents),
+                                     "--out", str(out), "--force"])
+            self.assertEqual(rc, 0)
+            self.assertNotIn("stale", out.read_text(encoding="ascii"))
+
+    def test_rejects_a_negative_min_size(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            exe, extents = self._write_inputs(root)
+            rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents),
+                                 "--out", str(root / "dupes.tsv"), "--min-size", "-1"])
+            self.assertEqual(rc, 2)
+
+
+if __name__ == "__main__":
+    unittest.main()