diff --git a/Makefile b/Makefile index e2bbdf8..d6766a8 100644 --- a/Makefile +++ b/Makefile @@ -27,12 +27,14 @@ REGIONS := config/regions.tsv SYMBOLS := config/symbols.tsv INVENTORY := config/function_inventory.tsv EXTENTS := config/function_extents.tsv +DUPES := config/duplicate_bodies.tsv MATCH := tools/sf3_match EXTENTS_TOOL := tools/sf3_extents +DUPES_TOOL := tools/sf3_dupes CODE_OUT := build/code EXPECTED_SHA1 := e173426c157384ebf1b6caf8c6fea18a85a14af9 -.PHONY: all validate split assemble link binary code gate extents extents-verify test check clean +.PHONY: all validate split assemble link binary code gate extents extents-verify dupes test check clean all: binary @@ -102,6 +104,13 @@ extents-verify: validate @"$(EXTENTS_TOOL)" verify --exe "$(EXE)" --inventory "$(INVENTORY)" \ --extents "$(EXTENTS)" --regions "$(REGIONS)" +# Duplicate-body census (Phase 7): groups byte-identical bodies so a shared body +# is matched once and registered once per address. +dupes: validate + @test -f "$(DUPES_TOOL)" + @test -f "$(EXTENTS)" + @"$(DUPES_TOOL)" census --exe "$(EXE)" --extents "$(EXTENTS)" --out "$(DUPES)" --force + # Verification gates. `test` is synthetic-only and needs no game input; # `check` adds the extents check and the full-binary byte gate (which do). test: diff --git a/config/duplicate_bodies.tsv b/config/duplicate_bodies.tsv new file mode 100644 index 0000000..d39fdb4 --- /dev/null +++ b/config/duplicate_bodies.tsv @@ -0,0 +1,74 @@ +# Syphon Filter 3 (USA) duplicate-body census. +# Columns: groupsizecountaddressesgradesflags. +# Multi-address groups only; addresses, sizes and grades only; no bytes. +# A group is evidence of a shared body, not a claim that every member is +# a real function start: verify a member before registering it. +# flags=zero means the body is all zero bytes -- a zero-filled region the +# walk ran through, not a function. Exclude those from matching. +# Regenerate: ./tools/sf3_dupes census --exe '' \ +# --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force +g0001 8 4 0x80042088,0x80089C4C,0x80089E74,0x800B5AF0 exact,exact,exact,exact - +g0002 12 2 0x800262E0,0x800262EC exact,exact - +g0003 12 2 0x80147D74,0x801515F4 fallthrough,fallthrough zero +g0004 16 8 0x80147650,0x80149A00,0x8014A170,0x8014A1C0,0x8014AC60,0x8014AC70,0x8014AC80,0x8014AEB0 fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough zero +g0005 20 2 0x80149478,0x80168288 fallthrough,fallthrough zero +g0006 32 3 0x801478A8,0x80147DFC,0x80151660 fallthrough,fallthrough,fallthrough zero +g0007 36 3 0x800F7FB4,0x80103C7C,0x80103F84 exact,exact,exact - +g0008 36 2 0x800F8F9C,0x80109314 exact,exact - +g0009 36 4 0x8014762C,0x80147E1C,0x80169448,0x8016946C fallthrough,fallthrough,fallthrough,fallthrough zero +g0010 40 2 0x801540D4,0x8016E4DC fallthrough,fallthrough zero +g0011 48 2 0x80148210,0x8016C808 fallthrough,fallthrough zero +g0012 52 2 0x80147D40,0x8016CA70 fallthrough,fallthrough zero +g0013 60 2 0x80010810,0x8009C69C exact,exact - +g0014 60 4 0x8014A0F8,0x8014A134,0x801522D0,0x8015F554 fallthrough,fallthrough,fallthrough,fallthrough zero +g0015 64 6 0x801482B0,0x8014A180,0x8014AE70,0x80151294,0x8016ADF0,0x8016B524 fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough zero +g0016 72 4 0x801499B8,0x80164824,0x8016ADA8,0x8016E494 fallthrough,fallthrough,fallthrough,fallthrough zero +g0017 80 2 0x80146D00,0x80169944 fallthrough,fallthrough zero +g0018 84 2 0x801694EC,0x8016E2D8 fallthrough,fallthrough zero +g0019 88 2 0x8016A004,0x8016C47C fallthrough,fallthrough zero +g0020 92 3 0x801498CC,0x80169490,0x80169994 fallthrough,fallthrough,fallthrough zero +g0021 96 4 0x80147CE0,0x80151600,0x8015230C,0x8016C164 fallthrough,fallthrough,fallthrough,fallthrough zero +g0022 104 2 0x80162AA8,0x80162FCC fallthrough,fallthrough zero +g0023 108 3 0x80149A8C,0x801521EC,0x80153868 fallthrough,fallthrough,fallthrough zero +g0024 112 4 0x80146C90,0x80148240,0x8014AE00,0x80151AF4 fallthrough,fallthrough,fallthrough,fallthrough zero +g0025 116 4 0x80147C6C,0x80154060,0x8016D364,0x8016D818 fallthrough,fallthrough,fallthrough,fallthrough zero +g0026 120 3 0x80018CB0,0x8001ADC4,0x8009C6D8 exact,exact,exact - +g0027 120 2 0x80152258,0x8016D090 fallthrough,fallthrough zero +g0028 124 4 0x801471CC,0x80147D80,0x80149A10,0x8016486C fallthrough,fallthrough,fallthrough,fallthrough zero +g0029 128 2 0x800FB13C,0x800FBE04 exact,exact - +g0030 140 2 0x8009E8D0,0x800A45E0 exact,exact - +g0031 144 3 0x80147F30,0x80149928,0x80165FF8 fallthrough,fallthrough,fallthrough zero +g0032 148 6 0x8014804C,0x8014AD6C,0x80151CEC,0x801564B4,0x8016AA20,0x8016C850 fallthrough,fallthrough,fallthrough,fallthrough,fallthrough,fallthrough zero +g0033 152 4 0x801478C8,0x801520AC,0x80155F64,0x80164AFC fallthrough,fallthrough,fallthrough,fallthrough zero +g0034 160 3 0x80151968,0x8016729C,0x8016C0C4 fallthrough,fallthrough,fallthrough zero +g0035 164 2 0x801648E8,0x8016CD48 fallthrough,fallthrough zero +g0036 168 3 0x80152144,0x8015F4AC,0x8016C27C fallthrough,fallthrough,fallthrough zero +g0037 172 4 0x801480E0,0x80163D78,0x8016A974,0x8016C75C fallthrough,fallthrough,fallthrough,fallthrough zero +g0038 176 2 0x801540FC,0x8016B298 fallthrough,fallthrough zero +g0039 184 2 0x80149AF8,0x8016C1C4 fallthrough,fallthrough zero +g0040 188 3 0x801644AC,0x80164568,0x80166528 fallthrough,fallthrough,fallthrough zero +g0041 212 2 0x80169374,0x8016D9AC fallthrough,fallthrough zero +g0042 216 2 0x80149F48,0x8014A020 fallthrough,fallthrough zero +g0043 220 2 0x8014AC90,0x801512D4 fallthrough,fallthrough zero +g0044 252 5 0x801495A8,0x80161978,0x80161A74,0x801643B0,0x80164F5C fallthrough,fallthrough,fallthrough,fallthrough,fallthrough zero +g0045 256 2 0x801497CC,0x80151B64 fallthrough,fallthrough zero +g0046 264 3 0x80149E40,0x80164CF4,0x8016E504 fallthrough,fallthrough,fallthrough zero +g0047 268 2 0x80153034,0x8016CF84 fallthrough,fallthrough zero +g0048 296 2 0x801496A4,0x80167174 fallthrough,fallthrough zero +g0049 312 2 0x80147400,0x80168910 fallthrough,fallthrough zero +g0050 352 4 0x80164B94,0x80164DFC,0x801687B0,0x80169540 fallthrough,fallthrough,fallthrough,fallthrough zero +g0051 396 2 0x80156328,0x8016C8E4 fallthrough,fallthrough zero +g0052 416 2 0x80162690,0x801696A0 fallthrough,fallthrough zero +g0053 436 2 0x8001D98C,0x8009C750 exact,exact - +g0054 436 2 0x80152C88,0x801532A4 fallthrough,fallthrough zero +g0055 440 2 0x80147248,0x80166FBC fallthrough,fallthrough zero +g0056 476 2 0x80149BB0,0x8016B348 fallthrough,fallthrough zero +g0057 488 3 0x80150EB0,0x801538D4,0x80159DE8 fallthrough,fallthrough,fallthrough zero +g0058 600 2 0x80148D6C,0x80148FC4 fallthrough,fallthrough zero +g0059 652 2 0x80147960,0x80163E24 fallthrough,fallthrough zero +g0060 656 2 0x80153458,0x80153DD0 fallthrough,fallthrough zero +g0061 712 2 0x8001084C,0x800189E8 exact,exact - +g0062 812 2 0x80151D80,0x80155FFC fallthrough,fallthrough zero +g0063 816 2 0x80169CD4,0x8016E60C fallthrough,fallthrough zero +g0064 1184 2 0x8014F07C,0x80166088 fallthrough,fallthrough zero +g0065 2000 2 0x80165828,0x8016B8F4 fallthrough,fallthrough zero diff --git a/docs/PHASE7_DUPES.md b/docs/PHASE7_DUPES.md new file mode 100644 index 0000000..b4c472f --- /dev/null +++ b/docs/PHASE7_DUPES.md @@ -0,0 +1,112 @@ +# Phase 7 — Duplicate-Body Census + +**Scope:** P7-T3 — find every body the executable contains more than once, so a shared body is matched +once and registered once per address, instead of being found by hand. +**Status:** complete. Tracked output: `config/duplicate_bodies.tsv`. Tool: `tools/sf3_dupes` +(20 synthetic tests). + +## Why + +`docs/MATCHING_CONVENTIONS.md` requires a duplicate check **before** registering a match, because the +registry's N-rows-to-one-source mechanism exists precisely so a shared body is written once. Phase 6 +performed that check by hand and found exactly one shared body (`0x800262E0` / `0x800262EC`). There was +no way to know how many others exist, or how large they are. + +## Method + +Every extent row in `config/function_extents.tsv` is hashed over exactly the bytes +`[address, end)` that `tools/sf3_extents` derived, and rows are grouped by `(size, body)`. Each member +keeps its grade, so a consumer can prefer groups whose members are all `exact`. `--exact-only` and +`--min-size N` restrict the census. + +The content hash is computed and **never written**. Grouping is the finding; addresses, sizes and +grades are the only ROM-derived quantities this project tracks — the same rule +`config/function_inventory.tsv` and `config/function_extents.tsv` follow. No instruction bytes appear +in the output. + +## Results + +| Measure | Count | +|---|---| +| Extents censused | 2,284 | +| Multi-address groups | 65 | +| — of which bodies are entirely zero bytes (`flags=zero`) | 55 | +| — of which bodies contain code | **10** | +| Addresses inside groups | 180 | +| Singleton bodies | 2,104 | +| Largest group | 8 addresses | + +Restricting to `exact` extents yields **the same 10 groups**, so every real duplicate group consists of +`exact`-graded members. + +### The ten real duplicate groups + +| Group | Size | Members | Grades | +|---|---|---|---| +| `g0001` | 8 | `0x80042088`, `0x80089C4C`, `0x80089E74`, `0x800B5AF0` | all `exact` | +| `g0002` | 12 | `0x800262E0`, `0x800262EC` | all `exact` | +| `g0007` | 36 | `0x800F7FB4`, `0x80103C7C`, `0x80103F84` | all `exact` | +| `g0008` | 36 | `0x800F8F9C`, `0x80109314` | all `exact` | +| `g0013` | 60 | `0x80010810`, `0x8009C69C` | all `exact` | +| `g0026` | 120 | `0x80018CB0`, `0x8001ADC4`, `0x8009C6D8` | all `exact` | +| `g0029` | 128 | `0x800FB13C`, `0x800FBE04` | all `exact` | +| `g0030` | 140 | `0x8009E8D0`, `0x800A45E0` | all `exact` | +| `g0053` | 436 | `0x8001D98C`, `0x8009C750` | all `exact` | +| `g0061` | 712 | `0x8001084C`, `0x800189E8` | all `exact` | + +`g0002` is the pair Phase 6 found by hand: the census reproduces it, which is the check that the +method is measuring what it claims to measure. `g0061` (712 bytes) and `g0053` (436 bytes) are +substantial shared bodies — matching one of those registers two functions for the price of one, which +is a materially better return than the 12-byte pair. + +## The finding this census produced: a zero band of false positives + +55 of the 65 groups are bodies consisting **entirely of zero bytes**, and they are concentrated in one +place: 252 extents overall have all-zero bodies, 245 of them inside `0x80147000`–`0x80170000`, a +160 KB zero-filled band. In that band the grades are 245 `fallthrough` and 1 `indirect`. + +The cause is a false positive in the **inventory**, not in the extents walk. `sf3_boundaries` grades +`jal` by decoding every word in the payload as an instruction; in a data region a word that happens to +have opcode `3` is graded as a call, and its "target" is an address inside the zero band. The walk then +starts at that address, decodes zeros (which are `nop`s), continues linearly, and reports a +`fallthrough` extent ending at the next such candidate. Two such extents of equal length therefore +contain identical all-zero bytes and group together. + +Consequences, recorded rather than fixed in place: + +- The inventory's `jal` grade is **not** sufficient on its own in a data region. This is a known limit + of Phase 6's grade definition (`docs/PHASE6_BOUNDARIES.md` §Grades) and is now demonstrated with + addresses, not asserted. +- The census flags these groups (`flags=zero`) instead of hiding them, so the finding stays visible. +- The worklist (P7-T4) must exclude candidates whose body is degenerate; a zero body is not code. + +## Limits + +- A duplicate group is **evidence of a shared body**, not a claim that every member is a real function + start. A shared *tail* reached by several functions is not a duplicate function, and a member still + has to be verified through the ordinary workflow. +- The census inherits the extents' limits: a group built on `fallthrough`/`indirect`/`escape` extents + is only as reliable as those bounds. In practice every real group here is all-`exact`. +- Two different functions that happen to compile to identical bytes are indistinguishable from a + shared body by this method. That is the correct behaviour for the registry (one source, N rows), but + it is a statement about bytes, not about intent. +- `--min-size` exists because very short bodies collide for boring reasons; the default is 1, so the + tracked census is unfiltered. + +## Regenerate + +```bash +make dupes +# or, explicitly: +./tools/sf3_dupes census --exe 'extracted/SCUS_946.40;1' \ + --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force +``` + +## Deviation from the plan + +`phase-ends/Phase7_PLAN.md` said the census artifact would be **ignored** and only aggregate counts +tracked. It is tracked instead, with the content hash still never written. The reason: the tracked file +contains addresses, sizes and grades only — the same class of information as the already-tracked +inventory and extents tables — and P7-T4's worklist must be reproducible from tracked inputs alone. +Keeping the grouping in an ignored file would have made the worklist unreproducible for exactly the +reason the plan wanted it reproducible. The plan's intent (no ROM content committed) is preserved. diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index d68a2e9..5671098 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -11,7 +11,7 @@ function bodies, bringing the project past **thirty** distinct byte-identical fu - [x] **P7-T1 — Phase control records, baseline revalidation, and open-item triage** (complete) - [x] **P7-T2 — Evidence-graded function extents** (complete) -- [ ] **P7-T3 — Duplicate-body census** +- [x] **P7-T3 — Duplicate-body census** (complete) - [ ] **P7-T4 — Candidate triage worklist** - [ ] **Rules check** - [ ] **P7-T5 — Symbol rows at scale** @@ -115,6 +115,47 @@ of a called entry", equally true of library and game code); `standalone` (438) i 15 `escape` cases are clipped conservatively with the crossing instruction recorded but the cause not determined per case; and nothing here promotes a candidate to a match. +## P7-T3 — Duplicate-body census (2026-09-23) + +**Delivered:** + +- `tools/sf3_dupes` — hashes the body of every derived extent and groups byte-identical bodies; + 20 synthetic tests. +- `config/duplicate_bodies.tsv` — tracked census of multi-address groups, + `groupsizecountaddressesgradesflags`. Addresses, sizes and grades only; the + content hash is computed and never written. +- `make dupes`; `docs/PHASE7_DUPES.md`. + +**Results:** 2,284 extents → 65 multi-address groups, of which **10 contain code** (24 addresses) and +55 are all-zero bodies; 2,104 singleton bodies. Restricting to `exact` extents yields the same 10 +groups. `g0002` is the `0x800262E0`/`0x800262EC` pair Phase 6 found by hand — the census reproduces it. +The largest real groups are `g0061` (712 bytes, `0x8001084C`/`0x800189E8`) and `g0053` (436 bytes). + +**Finding:** 252 extents have all-zero bodies, 245 of them inside the zero band +`0x80147000`–`0x80170000` (160 KB), graded `fallthrough`. Cause: the **inventory's** `jal` grade comes +from decoding every word as an instruction, so a data word with opcode `3` is graded as a call and its +"target" lands in the zero band. The walk then decodes zeros as `nop`s and reports a `fallthrough` +extent. Recorded, not hidden: the census flags those groups `flags=zero`, and the P7-T4 worklist must +exclude degenerate bodies. This demonstrates a Phase 6 grade limit with addresses rather than asserting +it. + +**Verification:** + +| Check | Command | Result | +|---|---|---| +| Known shared body found | `grep 0x800262E0 config/duplicate_bodies.tsv` | `g0002 12 2 0x800262E0,0x800262EC exact,exact -` | +| Synthetic suite | `python3 -m unittest discover -s tools/tests` | 20 new tests pass | +| Firewall | output columns | no body bytes and no hash written | +| Determinism | two `census` runs | identical (group labels ordered by size then first address) | + +**Deviation from the plan:** the plan said the census artifact would be ignored and only aggregate +counts tracked. It is tracked instead, hash still never written: the file holds addresses, sizes and +grades only (the same class as the tracked inventory and extents tables), and P7-T4's worklist must be +reproducible from tracked inputs alone. The plan's intent — no ROM content committed — is preserved. + +**Limits:** a group is evidence of a shared body, not that every member is a function start; a shared +*tail* is not a duplicate function; and the census inherits the extents' bounds. + ## Notes and limits - The four class items are the plan's P7-T2..T5 work; the single-function items are explicitly out of diff --git a/tools/sf3_dupes b/tools/sf3_dupes new file mode 100755 index 0000000..3375a3c --- /dev/null +++ b/tools/sf3_dupes @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Duplicate-body census for the USA executable. + +Matching conventions require checking for duplicates *before* registering a +match, because a body shared by several addresses is matched **once** and +registered once per address (`config/regions.tsv`, N rows to one source). Phase 6 +did that check by hand and found one shared body; there was no way to know how +many others exist. + +This tool hashes the body of every derived extent in +`config/function_extents.tsv` and groups exact duplicates. + +## What is grouped + +Every extent row -- any grade -- is hashed over exactly the bytes +`[address, end)` that `tools/sf3_extents` derived. Each member keeps its grade so +a consumer can prefer groups whose members are all `exact`. `--exact-only` +restricts the census to `exact` extents, and `--min-size` drops bodies shorter +than N bytes (very short bodies collide for boring reasons, e.g. `jr ra; nop`). + +## What is written + +A tracked, sorted TSV of **multi-address groups only**: + + groupsizecountaddressesgradesflags + + * `group` is a stable label (`g0001`, ...) assigned in `(size, first address)` + order, so the file is deterministic; + * `addresses` is a comma-separated list in ascending order; + * `grades` is the comma-separated grade of each address, in the same order; + * `flags` is `zero` when the shared body is nothing but zero bytes, otherwise + `-`. An all-zero "body" is not a function: it is a zero-filled region the + walk ran through, which happens when a data word decodes as a `jal` whose + target lands in the zero band. Those groups are recorded, not hidden, so the + finding stays visible and the worklist can exclude them. + +The content hash is computed and **never written**: grouping is the finding, and +addresses, sizes and grades are the only ROM-derived quantities this project +tracks (the same rule `config/function_inventory.tsv` and +`config/function_extents.tsv` follow). No instruction bytes are read into the +output, and the executable is read only from the caller-supplied path. + +The census is a measurement, not a match claim: two identical bodies are +evidence of shared code, and nothing more. A shared *tail* is not a duplicate +function, so a group member still has to be verified as a real function start by +the ordinary workflow. + +Exit codes: 0 success, 2 usage or environment error. +""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path +import struct +import sys +from typing import Sequence + + +EXE_MAGIC = b"PS-X EXE" +HEADER_SIZE = 0x800 +PAYLOAD_LMA = 0x800 + +EXTENT_GRADES = frozenset({"exact", "fallthrough", "indirect", "escape"}) + +HEADER_LINES = ( + "# Syphon Filter 3 (USA) duplicate-body census.", + "# Columns: groupsizecountaddressesgradesflags.", + "# Multi-address groups only; addresses, sizes and grades only; no bytes.", + "# A group is evidence of a shared body, not a claim that every member is", + "# a real function start: verify a member before registering it.", + "# flags=zero means the body is all zero bytes -- a zero-filled region the", + "# walk ran through, not a function. Exclude those from matching.", + "# Regenerate: ./tools/sf3_dupes census --exe '' \\", + "# --extents config/function_extents.tsv --out config/duplicate_bodies.tsv --force", +) + + +class ToolError(Exception): + """A usage or environment problem; maps to exit code 2.""" + + +def parse_hex(text: str, label: str) -> int: + try: + return int(text, 16) + except ValueError as exc: + raise ToolError(f"{label}: not a hex address: {text!r}") from exc + + +def require_file(path: Path, label: str) -> Path: + if not path.is_file(): + raise ToolError(f"{label} is not a regular file: {path}") + return path + + +def resolve_output(path: Path, force: bool) -> Path: + if path.exists() or path.is_symlink(): + if not force: + raise ToolError(f"output already exists (use --force to overwrite): {path}") + if not path.is_file() or path.is_symlink(): + raise ToolError(f"output is not a regular file: {path}") + return path + + +def parse_psx_exe(header: bytes) -> tuple[int, int, int]: + if len(header) < HEADER_SIZE: + raise ToolError("executable is smaller than a PS-X EXE header") + if header[:8] != EXE_MAGIC: + raise ToolError("executable does not carry the PS-X EXE magic") + entry, _gp, text_address, text_size = struct.unpack_from(" list[tuple[int, int, str]]: + """Read a generated extents table; keep rows that carry an extent.""" + rows: list[tuple[int, int, str]] = [] + for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + fields = line.split("\t") + if len(fields) != 7: + raise ToolError(f"extents line {number}: expected seven fields") + address = parse_hex(fields[0], f"extents line {number}") + grade = fields[5] + if fields[1] == "-": + continue + end = parse_hex(fields[1], f"extents line {number}") + if end <= address: + raise ToolError(f"extents line {number}: end is not after address") + if grade not in EXTENT_GRADES: + raise ToolError(f"extents line {number}: {grade!r} carries no extent") + rows.append((address, end, grade)) + if not rows: + raise ToolError("extents table contains no rows with an extent") + return rows + + +class Group: + """One set of addresses whose bodies are byte-identical.""" + + __slots__ = ("size", "members", "all_zero") + + def __init__(self, size: int, all_zero: bool) -> None: + self.size = size + self.members: list[tuple[int, str]] = [] + self.all_zero = all_zero + + +def census(payload: bytes, text_address: int, rows: Sequence[tuple[int, int, str]], + min_size: int, exact_only: bool) -> tuple[list[Group], int]: + """Group rows by (size, body bytes). Returns the groups and the singleton count.""" + buckets: dict[tuple[int, bytes], Group] = {} + for address, end, grade in rows: + if exact_only and grade != "exact": + continue + size = end - address + if size < min_size: + continue + start = address - text_address + body = payload[start:start + size] + if len(body) != size: + raise ToolError(f"extent 0x{address:08X}..0x{end:08X} is outside the payload") + key = (size, hashlib.sha1(body).digest()) + group = buckets.get(key) + if group is None: + group = buckets[key] = Group(size, not any(body)) + group.members.append((address, grade)) + + groups = [group for group in buckets.values() if len(group.members) > 1] + singletons = sum(1 for group in buckets.values() if len(group.members) == 1) + for group in groups: + group.members.sort() + groups.sort(key=lambda group: (group.size, group.members[0][0])) + return groups, singletons + + +def format_census(groups: Sequence[Group]) -> str: + lines = list(HEADER_LINES) + for index, group in enumerate(groups, 1): + addresses = ",".join(f"0x{address:08X}" for address, _grade in group.members) + grades = ",".join(grade for _address, grade in group.members) + flags = "zero" if group.all_zero else "-" + lines.append("\t".join(( + f"g{index:04d}", str(group.size), str(len(group.members)), addresses, grades, flags, + ))) + return "\n".join(lines) + "\n" + + +def command_census(args: argparse.Namespace) -> int: + exe_path = require_file(args.exe, "executable") + extents_path = require_file(args.extents, "function extents") + out = resolve_output(args.out, args.force) + if args.min_size < 0: + raise ToolError("--min-size cannot be negative") + + data = exe_path.read_bytes() + _entry, text_address, text_size = parse_psx_exe(data) + payload = data[PAYLOAD_LMA:PAYLOAD_LMA + text_size] + rows = load_extents(extents_path) + groups, singletons = census(payload, text_address, rows, args.min_size, args.exact_only) + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(format_census(groups), encoding="ascii") + + addresses = sum(len(group.members) for group in groups) + duplicated = sum(group.size * (len(group.members) - 1) for group in groups) + zero_groups = sum(1 for group in groups if group.all_zero) + print(f"extents={len(rows)}") + print(f"groups={len(groups)}") + print(f"groups_zero_body={zero_groups}") + print(f"groups_with_code={len(groups) - zero_groups}") + print(f"addresses_in_groups={addresses}") + print(f"singletons={singletons}") + print(f"duplicated_bytes={duplicated}") + if groups: + print(f"largest_group={max(len(group.members) for group in groups)}") + print(f"output={out}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + census_parser = subparsers.add_parser("census", help="group byte-identical bodies") + census_parser.add_argument("--exe", required=True, type=Path) + census_parser.add_argument("--extents", required=True, type=Path, + help="derived function extents (tools/sf3_extents output)") + census_parser.add_argument("--out", required=True, type=Path) + census_parser.add_argument("--force", action="store_true", + help="overwrite an existing output file") + census_parser.add_argument("--min-size", type=int, default=1, dest="min_size", + help="ignore bodies shorter than N bytes (default 1)") + census_parser.add_argument("--exact-only", action="store_true", + help="census only extents graded exact") + census_parser.set_defaults(handler=command_census) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return args.handler(args) + except ToolError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except OSError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_sf3_dupes.py b/tools/tests/test_sf3_dupes.py new file mode 100644 index 0000000..68e19dc --- /dev/null +++ b/tools/tests/test_sf3_dupes.py @@ -0,0 +1,262 @@ +"""Synthetic-only tests for the duplicate-body census tool. + +Fixtures are self-authored bytes in temporary directories. They never read the +local disc or the extracted game executable. +""" + +from __future__ import annotations + +import contextlib +import importlib.machinery +import importlib.util +import io +from pathlib import Path +import struct +import sys +import tempfile +import unittest + + +TOOL_PATH = Path(__file__).resolve().parents[1] / "sf3_dupes" + + +def _load_tool() -> object: + loader = importlib.machinery.SourceFileLoader("sf3_dupes_under_test", str(TOOL_PATH)) + spec = importlib.util.spec_from_loader(loader.name, loader) + if spec is None: + raise RuntimeError("could not create an import specification") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + loader.exec_module(module) + return module + + +sf3_dupes = _load_tool() + +PAYLOAD = 0x80010000 + +# Payload-relative layout of the fixture. +SHARED_A = 0x00 # 8 bytes of code, duplicated at SHARED_B +SHARED_B = 0x10 +UNIQUE = 0x20 # 12 bytes, no twin +ZERO_A = 0x30 # 16 zero bytes, duplicated at ZERO_B +ZERO_B = 0x40 + + +def _payload() -> bytes: + words = [0] * (0x50 // 4) + words[SHARED_A // 4] = 0x03E00008 # jr ra + words[SHARED_B // 4] = 0x03E00008 # jr ra + words[UNIQUE // 4] = 0x24020001 + words[UNIQUE // 4 + 1] = 0x24030002 + words[UNIQUE // 4 + 2] = 0x03E00008 + return b"".join(struct.pack(" bytes: + header = bytearray(0x800) + header[:8] = b"PS-X EXE" + struct.pack_into(" str: + lines = [ + "# synthetic extents", + "# Columns: addressendsizenextgapgradeevidence.", + ] + + def row(offset: int, size: int, grade: str, evidence: str) -> str: + start = PAYLOAD + offset + end = start + size + return f"0x{start:08X}\t0x{end:08X}\t{size}\t-\t-\t{grade}\t{evidence}" + + lines.append(row(SHARED_A, 8, "exact", "term=jr_ra")) + lines.append(row(SHARED_B, 8, "exact", "term=jr_ra")) + lines.append(row(UNIQUE, 12, "exact", "term=jr_ra")) + lines.append(row(ZERO_A, 16, "fallthrough", "hit=0x80010040")) + lines.append(row(ZERO_B, 16, "fallthrough", "hit=0x80010050")) + lines.append(f"0x{PAYLOAD + 0x60:08X}\t-\t-\t-\t-\tcontained\tinside=0x80010000") + return "\n".join(lines) + "\n" + + +def _rows() -> list[tuple[int, int, str]]: + return [ + (PAYLOAD + SHARED_A, PAYLOAD + SHARED_A + 8, "exact"), + (PAYLOAD + SHARED_B, PAYLOAD + SHARED_B + 8, "exact"), + (PAYLOAD + UNIQUE, PAYLOAD + UNIQUE + 12, "exact"), + (PAYLOAD + ZERO_A, PAYLOAD + ZERO_A + 16, "fallthrough"), + (PAYLOAD + ZERO_B, PAYLOAD + ZERO_B + 16, "fallthrough"), + ] + + +class ParseTests(unittest.TestCase): + def test_rejects_missing_magic(self) -> None: + with self.assertRaises(sf3_dupes.ToolError): + sf3_dupes.parse_psx_exe(bytes(0x800)) + + def test_rejects_empty_payload(self) -> None: + header = bytearray(0x800) + header[:8] = b"PS-X EXE" + with self.assertRaises(sf3_dupes.ToolError): + sf3_dupes.parse_psx_exe(bytes(header)) + + +class ExtentLoadingTests(unittest.TestCase): + def test_skips_rows_without_an_extent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "extents.tsv" + path.write_text(_extents_text(), encoding="ascii") + rows = sf3_dupes.load_extents(path) + self.assertEqual(len(rows), 5) + + def test_rejects_a_row_whose_grade_carries_no_extent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "extents.tsv" + path.write_text( + f"0x{PAYLOAD:08X}\t0x{PAYLOAD + 8:08X}\t8\t-\t-\tcontained\tinside=0x1\n", + encoding="ascii") + with self.assertRaises(sf3_dupes.ToolError): + sf3_dupes.load_extents(path) + + def test_rejects_an_empty_extents_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "extents.tsv" + path.write_text("# nothing\n", encoding="ascii") + with self.assertRaises(sf3_dupes.ToolError): + sf3_dupes.load_extents(path) + + +class CensusTests(unittest.TestCase): + def setUp(self) -> None: + self.payload = _payload() + + def _groups(self, **kwargs: object) -> list[object]: + options = {"min_size": 1, "exact_only": False} + options.update(kwargs) + groups, _singletons = sf3_dupes.census( + self.payload, PAYLOAD, _rows(), options["min_size"], options["exact_only"]) + return groups + + def test_identical_bodies_are_grouped(self) -> None: + groups = self._groups() + addresses = [[address for address, _grade in group.members] for group in groups] + self.assertIn([PAYLOAD + SHARED_A, PAYLOAD + SHARED_B], addresses) + self.assertIn([PAYLOAD + ZERO_A, PAYLOAD + ZERO_B], addresses) + + def test_unique_bodies_are_not_grouped(self) -> None: + groups = self._groups() + for group in groups: + self.assertNotIn(PAYLOAD + UNIQUE, [address for address, _g in group.members]) + + def test_a_zero_body_is_flagged(self) -> None: + groups = {group.members[0][0]: group for group in self._groups()} + self.assertTrue(groups[PAYLOAD + ZERO_A].all_zero) + self.assertFalse(groups[PAYLOAD + SHARED_A].all_zero) + + def test_min_size_drops_short_bodies(self) -> None: + groups = self._groups(min_size=10) + sizes = {group.size for group in groups} + self.assertEqual(sizes, {16}) + + def test_exact_only_drops_other_grades(self) -> None: + groups = self._groups(exact_only=True) + self.assertEqual([group.size for group in groups], [8]) + for group in groups: + for _address, grade in group.members: + self.assertEqual(grade, "exact") + + def test_singletons_are_counted_not_listed(self) -> None: + groups, singletons = sf3_dupes.census(self.payload, PAYLOAD, _rows(), 1, False) + self.assertEqual(singletons, 1) + self.assertEqual(len(groups), 2) + + def test_groups_are_ordered_by_size_then_address(self) -> None: + groups = self._groups() + keys = [(group.size, group.members[0][0]) for group in groups] + self.assertEqual(keys, sorted(keys)) + + def test_an_extent_outside_the_payload_is_rejected(self) -> None: + rows = [(PAYLOAD + 0x1000, PAYLOAD + 0x1008, "exact")] + with self.assertRaises(sf3_dupes.ToolError): + sf3_dupes.census(self.payload, PAYLOAD, rows, 1, False) + + +class FormatTests(unittest.TestCase): + def test_header_and_column_count(self) -> None: + groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False) + text = sf3_dupes.format_census(groups) + body = [line for line in text.splitlines() if not line.startswith("#")] + self.assertTrue(body) + for line in body: + self.assertEqual(len(line.split("\t")), 6) + + def test_group_labels_are_stable_and_sorted(self) -> None: + groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False) + labels = [line.split("\t")[0] for line in sf3_dupes.format_census(groups).splitlines() + if not line.startswith("#")] + self.assertEqual(labels, ["g0001", "g0002"]) + + def test_no_body_bytes_are_written(self) -> None: + groups, _ = sf3_dupes.census(_payload(), PAYLOAD, _rows(), 1, False) + text = sf3_dupes.format_census(groups) + self.assertNotIn("03e00008", text.lower()) + self.assertNotIn("24020001", text.lower()) + + +class MainTests(unittest.TestCase): + def _write_inputs(self, root: Path) -> tuple[Path, Path]: + exe = root / "synthetic.exe" + exe.write_bytes(_synthetic_exe(_payload(), PAYLOAD)) + extents = root / "extents.tsv" + extents.write_text(_extents_text(), encoding="ascii") + return exe, extents + + def test_census_writes_the_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + exe, extents = self._write_inputs(root) + out = root / "dupes.tsv" + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents), + "--out", str(out)]) + self.assertEqual(rc, 0) + text = out.read_text(encoding="ascii") + self.assertIn(f"0x{PAYLOAD + SHARED_A:08X},0x{PAYLOAD + SHARED_B:08X}", text) + self.assertIn("groups=2", stdout.getvalue()) + self.assertIn("groups_zero_body=1", stdout.getvalue()) + + def test_census_refuses_an_existing_output(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + exe, extents = self._write_inputs(root) + out = root / "dupes.tsv" + out.write_text("", encoding="ascii") + rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents), + "--out", str(out)]) + self.assertEqual(rc, 2) + + def test_census_force_overwrites(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + exe, extents = self._write_inputs(root) + out = root / "dupes.tsv" + out.write_text("stale\n", encoding="ascii") + with contextlib.redirect_stdout(io.StringIO()): + rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents), + "--out", str(out), "--force"]) + self.assertEqual(rc, 0) + self.assertNotIn("stale", out.read_text(encoding="ascii")) + + def test_rejects_a_negative_min_size(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + exe, extents = self._write_inputs(root) + rc = sf3_dupes.main(["census", "--exe", str(exe), "--extents", str(extents), + "--out", str(root / "dupes.tsv"), "--min-size", "-1"]) + self.assertEqual(rc, 2) + + +if __name__ == "__main__": + unittest.main()