diff --git a/.run/P35/census/coverage_notes.txt b/.run/P35/census/coverage_notes.txt new file mode 100644 index 000000000..ea88d8f14 --- /dev/null +++ b/.run/P35/census/coverage_notes.txt @@ -0,0 +1,3 @@ +# unaccounted sig instances (alias:addr:name:nins) +# instances with >=2 non-stub forms +# unresolved (non-func_, not in the binary's symbol stack) definition/site names diff --git a/.run/P35/census/share_census.json b/.run/P35/census/share_census.json new file mode 100644 index 000000000..66ceffff2 --- /dev/null +++ b/.run/P35/census/share_census.json @@ -0,0 +1,101 @@ +{ + "binaries": 218, + "classes_ge2": 10180, + "classified": 362389, + "controls": [ + { + "expected": "B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it)", + "got": "B/141/{'include': 141}/missing 7", + "name": "func_80144B9C" + }, + { + "expected": "A+E/2", + "got": "A+E/2", + "name": "clearTbl40" + }, + { + "expected": "B/282(E),B/145(E),B/141", + "got": "B/282(E),B/145(E),B/141", + "name": "ov_setters x3" + }, + { + "expected": "equal (each = the twin's unregistered classes)", + "got": "653 / 653", + "name": "twin symmetry SC01_005 / SC01_006" + } + ], + "cross_address_deferred": { + "classes": 3801, + "copies": 12938, + "instances": 30347 + }, + "duplicate_text_classes": 6845, + "duplicate_text_sites": 23269, + "elapsed_s": 41.1, + "flags": { + "ALIAS": 44, + "E": 3826, + "F": 54, + "PINS": 687, + "TWIN-PENDING": 3748, + "TYPEDEF": 96 + }, + "ins_by_verdict": { + "A": 6048041, + "B": 1175147, + "C": 663203, + "D": 165250, + "M": 28564 + }, + "instances_by_verdict": { + "A": 214478, + "B": 45226, + "C": 14839, + "D": 5796, + "M": 462 + }, + "macro_sites": 255947, + "multi_form": 0, + "same_vram_unregistered": { + "classes": 4667, + "collapsible": 31309, + "copies": 11767, + "ins": 1710469, + "instances": 35976, + "twin_pending": 3568, + "twin_pending_instances": 7136 + }, + "sig_instances": 362389, + "twin_sets": [ + [ + "ov_SC01_005", + "ov_SC01_006" + ], + [ + "ov_SC02_000", + "ov_SC02_003" + ], + [ + "ov_SC03_014", + "ov_SC03_015" + ], + [ + "ov_SC03_118", + "ov_SC03_119" + ], + [ + "ov_SC04_018", + "ov_SC04_019" + ] + ], + "unaccounted": 0, + "unresolved_names": 0, + "verdicts": { + "A": 1712, + "B": 282, + "C": 6107, + "D": 1861, + "M": 218 + }, + "violations": 4667 +} diff --git a/.run/P35/census/share_census.txt b/.run/P35/census/share_census.txt new file mode 100644 index 000000000..0ed4c5752 --- /dev/null +++ b/.run/P35/census/share_census.txt @@ -0,0 +1,39 @@ +share_census: 218 binaries · 362,389 sig instances · 362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes with >=2 instances + verdicts (classes): A 1,712 · B 282 · C 6,107 · D 1,861 · M 218 + instances by verdict: A 214,478 · B 45,226 · C 14,839 · D 5,796 · M 462 + flags: ALIAS 44 · E 3,826 · F 54 · PINS 687 · TWIN-PENDING 3,748 · TYPEDEF 96 + SAME-VRAM UNREGISTERED (the phase's backlog): 4,667 classes · 35,976 instances · 11,767 private copies · 31,309 collapsible · 1,710,469 ins; of which twin-pending 3,568 classes / 7,136 instances + CROSS-ADDRESS / CROSS-SPACE (deferred to the names phase): 3,801 classes · 30,347 instances · 12,938 private copies + S1 violations (scope as run): 4,667 · multi-form instances 0 · unresolved names 0 + controls (R39): + func_80144B9C got B/141/{'include': 141}/missing 7 expected B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it) + clearTbl40 got A+E/2 expected A+E/2 + ov_setters x3 got B/282(E),B/145(E),B/141 expected B/282(E),B/145(E),B/141 + twin symmetry SC01_005 / SC01_006 got 653 / 653 expected equal (each = the twin's unregistered classes) + top classes by instances x nins (verdict flags instances nins addrs names): + B - 141 x 770 0x80144B9C func_80144b9c + A - 141 x 476 0x80141CA4 func_80141ca4 + A - 141 x 400 0x80132784 func_80132784 + A E 1128 x 41 0x80151FB4,0x801522CC,0x801525F4 func_80151fb4,func_801522cc + A - 141 x 284 0x8012D714 func_8012d714 + A - 141 x 254 0x8015126C func_8015126c + A - 141 x 248 0x8014EE14 func_8014ee14 + A - 141 x 214 0x80156B74 func_80156b74 + A - 141 x 213 0x80148094 func_80148094 + A - 141 x 207 0x801372B0 func_801372b0 + B PINS 141 x 204 0x8013A530 func_8013a530 + A - 141 x 198 0x801412A8 func_801412a8 + A - 141 x 198 0x801571C4 func_801571c4 + A - 141 x 195 0x80153E00 func_80153e00 + A - 141 x 191 0x80129CF8 func_80129cf8 + A - 141 x 189 0x8012D098 func_8012d098 + A E 282 x 93 0x8014FFDC,0x801502EC func_8014ffdc,func_801502ec + A - 141 x 185 0x8013AF20 func_8013af20 + A - 141 x 183 0x8014EA4C func_8014ea4c + A - 141 x 182 0x801392FC func_801392fc + A - 141 x 178 0x8012EC04 func_8012ec04 + A - 141 x 174 0x8014F74C func_8014f74c + B E,F 11852 x 2 0x80014554,0x8001455C,0x8001513C func_80014554,func_8001455c + A - 141 x 167 0x80163C2C func_80163c2c + A - 141 x 165 0x80178004 func_80178004 + macro sites 255,947 · duplicate-text classes 6,845 (23,269 sites) · elapsed 41.1 s diff --git a/config/dedup_exceptions.tsv b/config/dedup_exceptions.tsv new file mode 100644 index 000000000..5816a4b53 --- /dev/null +++ b/config/dedup_exceptions.tsv @@ -0,0 +1,10 @@ +# config/dedup_exceptions.tsv — the S1 exception ledger (Phase 35): a byte-identical class with >=2 instances that does NOT share +# one source, with the MEASURED reason. Read by tools/share_census.py --check; one row per h_exact class. Reason codes: +# JTBL-CARVE a member's body owns a jump-table carve that the shared form cannot carry +# O0-LOCAL a member lives in a per-binary -O0 split translation unit +# TU-CONFLICT the chosen text is refused by a member's translation unit (a conflicting declaration) +# GATE-REJECT the chosen text compiled but the member's binary did not stay byte-identical (the compiler message class in note) +# PINNED every candidate text carries register pins and sharing would spread them (Phase 36's work) +# CROSS-SPACE instances in different address spaces (deferred with the cross-address classes) +# A row is evidence, not a shrug: note names the tool run and the message. Empty at the phase's open (2026-09-08). +h_exact reason nins instances note diff --git a/config/tool_dictionary.tsv b/config/tool_dictionary.tsv index 1d8054f47..8bf1823a5 100644 --- a/config/tool_dictionary.tsv +++ b/config/tool_dictionary.tsv @@ -327,3 +327,4 @@ tools/worklist.py P5 ADAPT join the target pool and near-miss ledger into one by tools/xsig/tests/make_fixtures.sh P6 ADAPT regenerate the signature tool's fixtures with the pinned toolchain Regenerates the signature-tool fixtures with the pinned toolchain compiler triple, repo tool paths LIVE tools/xsig/tests/test_xsig.py P6 PORTABLE property-test the signature tool on committed fixtures, needing no compiler Property tests for the signature tool on committed fixtures, needing no compiler fixture paths LIVE tools/xsig/xsig.py P6 PORTABLE sign functions with relocations masked for cross-project code identification Relocation-masked per-function signatures for cross-project code identification MIPS/relocation model only LIVE +tools/share_census.py P35 ADAPT measure duplicate function bodies across the fleet and assert one source per unique function The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test repo paths, the registry and signature schemas LIVE diff --git a/docs/SETUP.md b/docs/SETUP.md index 7e7f52a93..1888d539c 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -776,6 +776,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/gitignore_template_check.py` | **(P33.5 task 7)** The ```` ```gitignore ```` fence of `docs/wiki/The-ROM-firewall.md` must equal `decomp-architect/templates/gitignore.decomp` byte for byte (one source, two copies; R75-shaped). rc 1 on drift, rc 2 when the template does not exist yet ("nothing to compare" — never a pass, R43); refuses a page with ≠ 1 fence. In `tools-health` behind an existence test that skips loudly until the kit lands (task 11). | | | `tools/kit_lint.py [--selftest] [--paths …]` | **(P33.5 task 11)** The day-one decomp kit (`decomp-architect/`) stays de-specialised: (1) LEAK — no line outside a ```` ```calibration ```` fence and off a `provenance:` line matches `SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\.17\.|\bR[0-9]{1,2}\b|§[0-9]+` (fence-aware: `grep -v calibration` would drop only lines containing the word); (2) PLACEHOLDERS — the `{{NAME}}` set used under the package equals the backticked set in `templates/PLACEHOLDERS.md` (the contract); (3) SYNTAX — `bash -n` / `py_compile` (no bytecode written) / JSON+YAML parse; (4) the gitignore template diff (delegated); (5) `TODO(platform)` / `TODO(phase-N)` counts; (6) coverage — zero files is a failure. rc 1 findings, rc 2 package absent (R43). `--selftest` = the R39 control (a planted leak line + a planted unlisted placeholder must be caught, fenced and provenance lines must not). In `tools-health` (selftest, then the real run). | | | `tools/tool_census.py [--check | --manifest | --corpus | --all | --consumers FILE]` | **(P33.5 task 13.5)** The tools audit as a derived instrument: two independent enumerations of every tool file under `tools/` (`find` vs `git ls-files`, submodules/vendored/downloaded excluded — they must agree, R34); per tool the docstring line, its SETUP row, its CONSUMERS (Makefile/`.mk` targets, CI, the wave playbook, other tools by import or by name) and hence its class (LIVE · REFERENCED · ORPHAN); the AUTHORED facts live in `config/tool_dictionary.tsv` (phase · portability · the NEED the tool answers · what · what it hard-codes · the retirement verdict with its successor or product) with coverage asserted BOTH ways (R32 — a new tool without a row fails `--check`). Generates `docs/tool-index.md` (the need-keyed dictionary; KEEP-GEN), the kit's `tools/MANIFEST.md` (`--manifest`) and the three verbatim corpora `decomp-architect/corpus/tools//` + `corpus/cookbook/` + `corpus/record/` (the how-to, the decision log, the accelerators, the retrospective, the story, the playbook, the effort map, the gen3 docs, the digest, every PhaseEnd — task 14.5) (`--corpus`; superseded tools as pointer files; sha1-equal to their sources). `--check` in `tools-health`; `make kit-corpus` = `--all`. `--consumers FILE` is the referrer census before any `git mv` of a tool. | +| | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. | | | `tools/kit_coverage.py` | **(P33.5 task 14.5)** The kit's DISTILLATION coverage: derives the rule population (every `- **R` of DIGEST §3, asserted contiguous) and the hindsight population (every `## ` heading of `docs/accelerators.md` at numbered-item granularity — 58 at S92) and asserts each is cited by a `provenance:` line of the registry seed / the kernels OR dispositioned in `config/kit_coverage_map.tsv` (`G` / `DK-` / `FOLDED:G` / `ENV` / `PA` / `SEED:` / `KIT:` / `RECORD` / `COOKBOOK` / `NOT-PORTABLE`; unknown ids refused, R43); counts with denominators (R41); rc 1 on any gap. In `tools-health` after `tool_census --check`. Its first run found 26 uncited rules and 21 uncited entries → three new kernels (DK-66–DK-68) and 41 authored dispositions. | | | `decomp-architect/` (the day-one decomp kit) | **(P33.5 tasks 9–14)** The package a new matching-decomp project installs as Phase 0.5 on ProjectArchitect 2.0: `README.md` (the three steps), `intake.decomp.md` (ProjectArchitect's twelve items pre-answered + the phase ladder + the six readability inversions), `SETUP.md` (the installer, Step 0 contract … Step 10 verify + hard stop; `answers: ` for unattended runs), `decomp-architect.md` (the methodology), `templates/` (the firewall pack — `gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, the planted fixture, `no-rom.template.yml` —, the READMEs, `pa-overlays.md`, `registry-E.decomp.md` G1–G67, the skeletons, `PLACEHOLDERS.md`, `layout-contract.md`), `corpus/decomp-kernels.md` (DK-1 … DK-80), the three dictionaries `corpus/tools//` + `corpus/cookbook/` + `corpus/record/` (generated by `make kit-corpus`), `memory-seed/` (18), `tools/MANIFEST.md` (generated). **How it is checked:** `tools/kit_lint.py` (de-specialisation, placeholders, syntax, the gitignore-template diff) + `tools/tool_census.py --check` (the corpora equal their sources) + `tools/gitignore_template_check.py`, all in `tools-health`; the dry-run harness under `.run/P33.5/kit-dryrun/` (`answers.md`, `expected-manifest.txt`, `judge.py`, the install logs and verdicts of runs 1–5 — a kit change is re-verified by a resume on the last throwaway `repo/`, a fresh full run only when SETUP's steps change; the judge compares the real tree's dirty PATH SETS before/after). Wiki page: `docs/wiki/Start-a-new-decomp-project.md`. Split into its own repository after the flip. | | **Verification** | `tools/verify_contract.sh` | **(P33 A5/C8) THE recorded contract run**: 00 tree · 01 check-env · 02 family_hseq · 03 `make clean && extract-all && check-all` · 04 sdk-dual (or a recorded SKIP) · 05 tools-health (zero `[warn]`) · 06 audit-frontier · 07 audit-disc · 08 report; one log per step ending `EXIT=`, abort on the first red (R53), every step asserted by its contract line (R49), `SUMMARY.md` generated → `.run/P33/verify/` (tracked evidence, quoted by `docs/verification.md` §2); step 00 ignores its own output dir. ≈14 min on 32 CPUs. | @@ -1509,6 +1510,21 @@ fills fast). Nothing is leaking — but the host does not get the memory back on or a quoted include that resolves to no file / outside the repo is an offender. R39 controls: 34 offenders in 19 files before the fix (all ABSOLUTE), 0 after; 4,299 sources scanned (the coverage line, R32). +### P35 T1 (S94, 2026-09-08) — the share census and the S1 invariant ("one source per unique function") +- **`tools/share_census.py`** — the census of byte-identical function classes (keyed by `h_exact`, never by name — R48) across all + 218 binaries, and the S1 checker. One masked pass per translation unit yields every instance's source form (`macro` / `macro-param` / + `include` / `param-include` / `def` / `stub` / `asm-verbatim`); a sig instance with zero or two forms fails the run (R32 — the + first run found 2,360 K&R definitions the scanner could not see; the last found 0 of 362,389). Verdicts A/B/C/D/M and flags + E/F/TWIN-COVERED/TWIN-PENDING/TYPEDEF/PINS/ALIAS as in the docstring. The fleet, the source dirs, the contracts (`_CHECK_SHA`), + the address spaces (`_VRAM_BASE`) and the twin sets (equal contracts) are derived from the Makefile and `config/` (R33); + groups only through `dedup_integrate.group_members`. `--check` (exit code) = S1 with `config/dedup_exceptions.tsv` as the ledger + (`h_exact reason nins instances note`; reason codes in the file header); `--strict-macros` / `--strict-text` are the + post-conversion halves (T7 wires them into `tools-health`); `--scope all` counts the deferred cross-address classes as violations. + `--selftest` is the R39 fixture (7 verdicts + the scanner on a synthetic TU). Outputs under `.run/P35/census/`: `share_census.json` + (summary + controls), `share_census.txt` (the table), `coverage_notes.txt` (tracked); `classes.jsonl` and `cache/` (ignored). + **Measured S94 on the pre-conversion tree:** 10,180 classes; A 1,712 · B 282 · C 6,107 · D 1,861 · M 218; same-vram unregistered + 4,667 classes / 11,767 private copies; cross-address deferred 3,801 classes; 43 s (16 workers). + ### P33 B1 (S86, 2026-09-06) — `make disc-extract`: the rom→decoder step, promoted into the build - **`make disc-extract`** (`DISC_DIR ?= disks`): the repository ships no ROM bytes (H1 in force). The target (1) probes `extracted/retail/` against the committed oracle (`extract.py --verify`, **0.7 s** when up to date → no-op), else (2) diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index ef554d737..165f51f33 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -34,7 +34,8 @@ the probe on ov_SC06_033: **34/34 objects byte-identical** in the include form, the binary `BYTE-IDENTICAL` both ways, build 1.12 s → 0.52 s wall (CPU 12.7 s → 5.7 s), warnings 801 → 663 (all 137 macro-redefinition warnings gone), `.d` 11.6 KB → 182 KB (`.run/P35/probe/probe_ledger.txt`). -- ☐ **T1** — `tools/share_census.py` + `config/dedup_exceptions.tsv` (the census + the S1 checker; `--selftest` 7/7; the four negative controls). +- ☑ **T1** (S94) — `tools/share_census.py` + `config/dedup_exceptions.tsv`: `selftest: 7/7 verdicts correct`; `362,389 sig instances · + 362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes`; the four controls as measured (see the log); SETUP + dictionary rows. - ☐ **T2** — the health chain learns the header + twin forms while the old tree is still green (the source-dir oracle; corpus/cdecl/ overlay_src_split/progress/dedup_integrate/audit_binaries/lint_symbol_refs/shared_lock/fix_arity_callers/family_remap/blocker_probe/ demacroize/export_pairs/harvest_verify); negative control: `make tools-health` OK on the UNCHANGED tree. @@ -89,6 +90,27 @@ - **S94 — T0 tools-health, run 3 GREEN:** `tools-health: OK — sigs fresh; corpus(+resident) + cdecl + binaries + report(lint+dedup) + cookbook-index all green.` `exit=0`, `wall=474.23 s` (`.run/P35/baseline/tools_health_open3.log`). `docs/story-timeline.md/.svg` regenerated by the chain (74 rows; the v2.0.0 tag row). **T0 ☑.** +- **S94 — T1 `tools/share_census.py` + `config/dedup_exceptions.tsv`.** Built as designed (one pass per TU with comments, dead + `#if 0`/`NON_MATCHING` halves and macro-continuation text masked; forms macro / macro-param / include / param-include / def / stub / + asm-verbatim; addresses from each binary's symbol stack, never names; the fleet, the source dirs, the contracts (`_CHECK_SHA`, + main = `check.us.sha`), the address spaces (`_VRAM_BASE`) and the twin sets derived from the Makefile and the contracts; classes + through `dedup_integrate.group_members`). Five scanner defects found by its own self-test and coverage line, each fixed at the cause: + string contents were blanked before include paths and asm labels were read; K&R definitions (a blank tail after the last `;`, + and the one-line `void f(a, b) void *a; s16 b; {` form); a second definition head on the same line after an `extern …;`; the + alias regex spanning a previous macro site's parentheses; `asm(` as well as `__asm__(`; implicit-return-type heads with no type + line; `static inline` helpers (no address) excluded from the unresolved list. **Result: `--selftest` → `selftest: 7/7 verdicts + correct`; the census → `218 binaries · 362,389 sig instances · 362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes with >=2 + instances`; verdicts A 1,712 · B 282 · C 6,107 · D 1,861 · M 218 (instances 214,478 / 45,226 / 14,839 / 5,796 / 462); flags ALIAS 44 · + E 3,826 · F 54 · PINS 687 · TWIN-PENDING 3,748 · TYPEDEF 96; SAME-VRAM UNREGISTERED 4,667 classes · 35,976 instances · 11,767 private + copies · 31,309 collapsible · 1,710,469 ins (twin-pending 3,568 classes / 7,136 instances); CROSS-ADDRESS/SPACE deferred 3,801 classes · + 30,347 instances · 12,938 private copies; macro sites 255,947; duplicate-text classes 6,845 (23,269 sites); 43 s uncached, 41 s cached + (the cache is not the cost — profile at T7); exit 0.** Controls (R39): `func_80144B9C` = B/141/{'include': 141}/missing 7 (the registry + lists 134 — never extended to the 7 later overlays; T5 bucket 0), `clearTbl40` = A+E/2, the three setters = B/282(E), B/145(E), B/141 + (the same 2-instruction bodies also sit at other vrams), twin symmetry SC01_005/006 = 653/653. **Instrument findings for the record:** + the B class is 282 registered-but-incomplete groups (204 never extended, 74 with a private-copy site — demacroized escapes —, 2 with a + member the sigs do not show); a 2-instruction empty-body class (`jr ra; nop`) has 11,852 instances across every space (E,F → deferred + with the cross-address set; the names phase decides how an empty function is written). Outputs `.run/P35/census/{share_census.json, + share_census.txt, coverage_notes.txt}` (tracked), `classes.jsonl` + `cache/` (ignored). SETUP + dictionary rows (R21/R87). **T1 ☑.** ## Approved plan (verbatim, gate 1 — 2026-09-08) @@ -382,7 +404,7 @@ same commit — history keeps it. Candidate for the PhaseEnd: "a shared body has exactly one source; a duplicate copy is a defect the health chain asserts, and a count of them is published with its rule" (this phase's invariant). -## 🛑 SESSION CHECKPOINT — S94 (2026-09-08): Phase 35 OPEN at gate 1; T0 ☑ (baseline 218/218 in 157 s; tools-health OK; probe 34/34); NEXT = T1 `tools/share_census.py` +## 🛑 SESSION CHECKPOINT — S94 (2026-09-08): Phase 35 OPEN at gate 1; T0 ☑, T1 ☑ (the census: 362,389/362,389 covered, 10,180 classes, self-test 7/7); NEXT = T2 (the health chain learns the header + twin forms) ### 0. How to use this block A fresh session reads CLAUDE.md's load order, replays THIS block verbatim, and resumes at the first unchecked task in §Tasks above. @@ -409,11 +431,12 @@ P5's conditions. Commit per task; banks the moment they are green (R42); Drew pu - **T0 — DONE.** The L0 recipe that worked (reuse in T4/T5): copy the A objects from `build/src//*.o`, apply the edit, `make check BINARY= -j16`, copy the B objects, `cmp` per object. The probe converter's parser (`macro_index` last-wins, the one site shape, the prelude = `engine_types.h` + the `ENGINE_SHB` line copied from `engine_core.h:20`) is the seed of `tools/macro_to_header.py`. -- **T1:** design from the plan's "The three tools" §share_census; sigs enumerated from `dup_report.BINARIES` (218; never a glob — `.run/` - holds 223 sig files); parse the registry ONLY through `dedup_integrate.group_members`; the source-form classifier must mirror - `dedup_propagate.find_site` (a prototype is not a definition) and resolve the asm-label alias form (`family_remap._alias_decl_for`); - negative controls: `func_80144B9C` A/141, clearTbl40 A+E, the 3 ov_setters A, `ov_SC01_005` 657 unregistered (S75 said 557 at 174 - binaries). Output under `.run/P35/census/` (allowlisted: json/txt/md). +- **T1 — DONE.** `tools/share_census.py` (run it plain for the table; `--check` for S1; `--selftest` for the fixture; `--no-cache` after a + scanner change — the cache is keyed by file mtime/size, not by the scanner's version). The numbers every later task measures against + are in the log entry and `.run/P35/census/share_census.json`. Known shapes the scanner handles (each was a finding): K&R definitions + (multi-line and the one-line `void f(a, b) void *a; s16 b; {`), implicit-return-type heads, a second head after `extern …;` on one + line, the asm-label alias in both `__asm__(` and `asm(` spellings, `static inline` helpers (no address). The census's per-class output + `classes.jsonl` is the input for T3's twin accounting and T5's plan (`share_body.py --plan` reads it). - **T2–T9:** see the task rows and the plan; the Plan agent's consumer table (which tools text-parse the macro form, which ask cpp and need nothing) is reproduced in the plan's §Consumers and §T6. diff --git a/tools/share_census.py b/tools/share_census.py new file mode 100644 index 000000000..0b7663cda --- /dev/null +++ b/tools/share_census.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python3 +"""share_census.py — the census of duplicate function bodies across the fleet, and the S1 invariant checker: +"ONE SOURCE PER UNIQUE FUNCTION" (Phase 35 T1, 2026-09-08). + + tools/share_census.py # the census: summary + controls to stdout; .run/P35/census/{share_census.json,share_census.txt} + tools/share_census.py --check # S1: every same-address byte-identical class with >=2 instances shares ONE source, + # is twin-covered, or is ledgered in config/dedup_exceptions.tsv -> exit 1 otherwise + tools/share_census.py --check --strict-macros --strict-text # the post-conversion form of S1 (T7 wires it into tools-health) + tools/share_census.py --selftest # the R39 fixture: every verdict/flag exercised in memory, no tree needed + tools/share_census.py --scope all # count the deferred cross-address / cross-space classes as violations too + +WHAT IT MEASURES (keyed by h_exact = SHA1 of the raw instruction bytes, tools/sig_image.py; relocated immediates INCLUDED, so two +instances with one h_exact reference identical addresses and their derived names coincide). A CLASS is one h_exact with >=2 instances +fleet-wide (218 binaries). Every instance gets exactly ONE source form from the binary's own translation units: + macro DEFINE_func_X() (the Phase-15 shared-body form; retired at T4) + macro-param SETTER(func_X,..)/RETCONST/CLEAR_TBL40 (the three legacy name-parameterized headers) + include #include "../shared/<..>.h" that DEFINES the function (the Phase-35 form; sotn's shape) + param-include #define SHARED_FN func_X / #include ../shared/<..>.h / #undef (the cross-address form; clearTbl40 after T4) + def an inline C definition (a private copy) + stub INCLUDE_ASM(...) (none left in game code; kept so the census can never mistake one for a copy) + asm-verbatim the five PERMANENT hand-asm bodies of config/verbatim_manifest.json (the authoritative list, never a regex) +and a sig instance with ZERO or >=2 forms is a COVERAGE DEFECT that fails the run (R32) — a silent skip is exactly the defect class +this project has paid for most often. + +VERDICT per class A registered (config/dedup.us.yaml) and every instance is a member whose site is a shared form + B registered but an instance is missing from the group, or a member's site is still a private copy + C unregistered; every copy is a private definition with the SAME normalized text (name-blind, comment-free) + D unregistered; the copies' texts DIFFER (matched independently) — one text must be chosen and re-gated + M unregistered, MIXED forms (a shared-form site without a group — e.g. a macro instantiated in one binary) +FLAGS (orthogonal) E cross-address (the member's own name differs per site -> the name-parameterized form; DEFERRED by decision) + F cross-space (instances in different address spaces: the overlay slot, a module slot base, main, resident) + TWIN-COVERED every instance's source dir is one directory (a twin binary built from its primary, T3) + TWIN-PENDING every instance lives in binaries of ONE identical-payload twin set (T3 will cover it) + TYPEDEF / PINS / ALIAS the copy carries a local type definition / register pins / an asm-label alias +S1 (--check): violations = classes with verdict != A that are neither TWIN-COVERED nor (E or F, unless --scope all) nor ledgered in +config/dedup_exceptions.tsv (h_exact reason nins instances note). --strict-macros adds "no DEFINE_func_ token under src/"; +--strict-text adds the SECOND, sig-blind oracle (R34): no name-blind normalized definition text appears in >1 translation unit +outside src/shared/. Both are informational until T7. + +ORACLES (derived, never hand-listed — R33): the fleet and each binary's source dir from the Makefile (`compile_only.src_dirs`, asserted +equal to the set of config/check.*.sha contracts, R32); address spaces from `_VRAM_BASE`; twin sets from equal contracts; +names from each binary's own symbol stack (`corpus.symbols`) else func_; groups through the ONE registry parser +`dedup_integrate.group_members`. Outputs: the small summary JSON + the human table are tracked evidence; classes.jsonl (every class) +and the per-TU scan cache stay ignored scratch. R41: every number in the summary names its denominator. +""" +import argparse +import collections +import hashlib +import json +import os +import pathlib +import re +import sys +import time +from concurrent.futures import ProcessPoolExecutor + +REPO = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "tools")) + +SHARED_FORMS = {"macro", "macro-param", "include", "param-include"} +KEYWORDS = {"if", "while", "for", "switch", "return", "else", "goto", "case", "sizeof", "do", "typedef", + "struct", "union", "enum", "defined", "break", "continue", "default"} +FUNC_RE = re.compile(r"func_([0-9A-Fa-f]{8})$") +OUT_DIR_DEFAULT = ".run/P35/census" + +# ---------------------------------------------------------------------------------------------------------------------- +# text masking: comments, dead preprocessor halves, macro-continuation blocks -> spaces (newlines kept, so line numbers hold) +# ---------------------------------------------------------------------------------------------------------------------- +_TOK = re.compile(r'/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.S) + + +def _blank_keep_newlines(s): + return re.sub(r"[^\n]", " ", s) + + +def mask_text(text): + """Comments become spaces (string/char literals are KEPT — include paths and asm labels live in them; the brace walk skips them); + dead `#if 0` blocks and the `#ifdef NON_MATCHING` half are blanked; `#define ... \\` continuation blocks are blanked (a + definition inside a macro body is macro text, not a definition).""" + def repl(m): + s = m.group(0) + if s.startswith("/*") or s.startswith("//"): + return _blank_keep_newlines(s) + return s + t = _TOK.sub(repl, text) + lines = t.split("\n") + out, i, n = [], 0, len(lines) + depth_stack = [] # entries: 'dead' (blank until the matching #endif) / 'live' (an #if we do not evaluate) + in_macro = False + for ln in lines: + s = ln.strip() + if in_macro: + out.append("") + in_macro = ln.rstrip().endswith("\\") + continue + if s.startswith("#"): + d = s[1:].strip() + if re.match(r"if\s+0\b", d) or d.startswith("ifdef NON_MATCHING"): + depth_stack.append("dead") + out.append("") + continue + if d.startswith(("if", "ifdef", "ifndef")): + depth_stack.append("live") + out.append("") + continue + if d.startswith("else") or d.startswith("elif"): + if depth_stack and depth_stack[-1] == "dead": + depth_stack[-1] = "live-else" # the #else half of a dead block is live + elif depth_stack and depth_stack[-1] == "live-else": + depth_stack[-1] = "dead" + out.append("") + continue + if d.startswith("endif"): + if depth_stack: + depth_stack.pop() + out.append("") + continue + if d.startswith("define") and ln.rstrip().endswith("\\"): + in_macro = True + out.append("") + continue + if any(x == "dead" for x in depth_stack): + out.append("") + continue + out.append(ln) + return "\n".join(out) + + +# ---------------------------------------------------------------------------------------------------------------------- +# the per-TU scanner +# ---------------------------------------------------------------------------------------------------------------------- +# a head may follow a line start OR a ';' on the same line (a one-line `extern …; int f(…) { … }` is a real shape in the tree) +HEAD = re.compile(r"(?:^|(?<=;))[ \t]*([A-Za-z_][\w \t\*]*?)\b([A-Za-z_]\w*)[ \t]*\(", re.M) +# a name at line start with NO type prefix (implicit int / the type on the line above); the body-extent check decides +BARE_HEAD = re.compile(r"^[ \t]*([A-Za-z_]\w*)[ \t]*\(", re.M) +MACRO_SITE = re.compile(r"^[ \t]*DEFINE_(func_[0-9A-Fa-f]{8})\(\)", re.M) +MACRO_PARAM_SITE = re.compile(r"^[ \t]*(SETTER|RETCONST|CLEAR_TBL40)\(\s*([A-Za-z_]\w*)", re.M) +INCLUDE_LINE = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"([^"]+)"', re.M) +STUB = re.compile(r"INCLUDE_ASM\([^)]*,\s*([A-Za-z_]\w*)\s*\)") +# the parameter list may not contain parentheses (a function-pointer parameter would surface as an unaccounted instance, loudly) +ALIAS_DECL = re.compile(r"\b([A-Za-z_]\w*)\s*\([^;{}()]*\)\s*(?:__asm__|asm)\s*\(\s*\"(func_[0-9A-Fa-f]{8})\"\s*\)") +SHARED_FN_DEF = re.compile(r"^[ \t]*#[ \t]*define[ \t]+SHARED_FN[ \t]+([A-Za-z_]\w*)", re.M) +SHARED_FN_UNDEF = re.compile(r"^[ \t]*#[ \t]*undef[ \t]+SHARED_FN\b", re.M) +PIN_RE = re.compile(r'register [^;]*__asm__\("\$?[a-z0-9]+"\)') +TYPEDEF_RE = re.compile(r"\b(typedef\b|(struct|union|enum)\s+\w*\s*\{)") +_BRACE = re.compile(r'"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'|[{}]') # literals are matched and ignored + + +def _body_extent(masked, head_end): + """From the '(' at head_end-1: walk the parameter parens, then decide prototype / definition; return (kind, open_idx, close_idx). + kind: 'proto' or 'def'. For a def, open_idx/close_idx index the body's braces in `masked`.""" + i, depth, n = head_end - 1, 0, len(masked) + while i < n: + c = masked[i] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + break + i += 1 + if i >= n: + return ("proto", None, None) + j = i + 1 + # after the parameter list: whitespace, then ';' (prototype), '{' (definition) or K&R declarations / an asm label + while j < n and masked[j] in " \t\r\n": + j += 1 + if j >= n: + return ("proto", None, None) + if masked[j] == ";": + return ("proto", None, None) + if masked[j] != "{": + # Something stands between the parameter list and the next '{'. It is a DEFINITION only if that something is K&R + # parameter declarations (`type name;` parts, nothing after the last ';'); an asm label, a directive, a call statement, + # or the head of the NEXT function all make this a declaration/statement, not a body. + seg_end_brace = masked.find("{", j) + if seg_end_brace == -1: + return ("proto", None, None) + seg = masked[j:seg_end_brace] + if "__asm__" in seg or "asm(" in seg or "#" in seg: + return ("proto", None, None) + parts = seg.split(";") + if len(parts) < 2 or parts[-1].strip() or not all(re.fullmatch(r"\s*[A-Za-z_][\w \t\*\[\],]*\s*", p) for p in parts[:-1]): + return ("proto", None, None) + j = seg_end_brace + # brace walk from j + depth = 0 + for m in _BRACE.finditer(masked, j): + tok = m.group(0) + if tok == "{": + depth += 1 + elif tok == "}": + depth -= 1 + if depth == 0: + return ("def", j, m.start()) + return ("proto", None, None) + + +def _norm(body, names): + t = body + for nm in sorted(set(names), key=len, reverse=True): + t = re.sub(r"\b%s\b" % re.escape(nm), "@FN@", t) + t = re.sub(r"\s+", " ", t).strip() + return hashlib.sha1(t.encode("utf-8", "surrogateescape")).hexdigest() + + +def scan_text(text, rel, shared_defs=None): + """All site records of one file. Records: dict(form, name, line, ...). shared_defs: header path -> defined names (for includes).""" + masked = mask_text(text) + recs = [] + line_of = lambda idx: masked.count("\n", 0, idx) + 1 + aliases = {a: s for a, s in ALIAS_DECL.findall(masked)} # alias ident -> real symbol + for m in MACRO_SITE.finditer(masked): + recs.append(dict(form="macro", name=m.group(1), line=line_of(m.start()))) + for m in MACRO_PARAM_SITE.finditer(masked): + recs.append(dict(form="macro-param", name=m.group(2), line=line_of(m.start()), via=m.group(1))) + for m in STUB.finditer(masked): + recs.append(dict(form="stub", name=m.group(1), line=line_of(m.start()))) + # includes (resolved by the caller through shared_defs), with the SHARED_FN state + if shared_defs is not None: + events = [(m.start(), "def", m.group(1)) for m in SHARED_FN_DEF.finditer(masked)] + events += [(m.start(), "undef", None) for m in SHARED_FN_UNDEF.finditer(masked)] + events += [(m.start(), "inc", m.group(1)) for m in INCLUDE_LINE.finditer(masked)] + cur = None + for pos, kind, val in sorted(events): + if kind == "def": + cur = val + elif kind == "undef": + cur = None + else: + key = os.path.normpath(os.path.join(os.path.dirname(rel), val)) + defs = shared_defs.get(key) + if not defs: + continue + for nm in defs: + if nm == "SHARED_FN": + if cur: + recs.append(dict(form="param-include", name=cur, line=line_of(pos), header=key)) + else: + recs.append(dict(form="include", name=nm, line=line_of(pos), header=key)) + # definitions + seen_heads = set() + heads = [(m.start(), m.end(), m.group(2), m.group(1)) for m in HEAD.finditer(masked)] + heads += [(m.start(), m.end(), m.group(1), "") for m in BARE_HEAD.finditer(masked)] + for start, end, name, prefix in heads: + if name in KEYWORDS or end in seen_heads: + continue + seen_heads.add(end) + if "inline" in prefix.split(): + continue # a `static inline` helper is expanded into its callers: it owns no address and is no sig instance + kind, o, c = _body_extent(masked, end) + if kind != "def": + continue + body = masked[start:c + 1] + real = aliases.get(name, name) + recs.append(dict(form="def", name=real, line=line_of(start), end=line_of(c), alias=(real != name), + text_hash=_norm(body, [name, real]), pins=bool(PIN_RE.search(body)), + typedef=bool(TYPEDEF_RE.search(body)), nlines=body.count("\n") + 1)) + return recs + + +def header_defined_names(path): + """The function names a shared header DEFINES (masked: macro bodies never count).""" + text = path.read_text(errors="surrogateescape") + recs = scan_text(text, path.relative_to(REPO).as_posix(), shared_defs=None) + return sorted({r["name"] for r in recs if r["form"] == "def"}) + + +def _scan_worker(args): + rel, mtime, size, shared_defs = args + p = REPO / rel + text = p.read_text(errors="surrogateescape") + return rel, scan_text(text, rel, shared_defs) + + +# ---------------------------------------------------------------------------------------------------------------------- +# oracles +# ---------------------------------------------------------------------------------------------------------------------- +def contracts(): + """alias -> its config/check..sha path, from the Makefile's `_CHECK_SHA :=` (main's contract is check.us.sha).""" + out = {} + texts = [(REPO / "Makefile").read_text()] + [p.read_text() for p in sorted((REPO / "config").glob("*.mk"))] + for t in texts: + for m in re.finditer(r"^(\w+)_CHECK_SHA\s*:?=\s*(\S+)", t, re.M): + out[m.group(1)] = m.group(2) + return out + + +def fleet_and_dirs(): + import compile_only + dirs = compile_only.src_dirs() + con = contracts() + on_disk = {p.relative_to(REPO).as_posix() for p in (REPO / "config").glob("check.*.sha")} + if set(dirs) != set(con) or set(con.values()) != on_disk: + sys.exit("share_census: the Makefile's binaries, its _CHECK_SHA contracts and config/check.*.sha DISAGREE (R32): " + f"dirs-only {sorted(set(dirs) - set(con))[:8]}, contracts-only {sorted(set(con) - set(dirs))[:8]}, " + f"files-not-declared {sorted(on_disk - set(con.values()))[:8]}, declared-not-on-disk {sorted(set(con.values()) - on_disk)[:8]}") + return sorted(dirs), dirs + + +def vram_bases(): + out = {} + texts = [(REPO / "Makefile").read_text()] + [p.read_text() for p in sorted((REPO / "config").glob("*.mk"))] + for t in texts: + for m in re.finditer(r"^(\w+)_VRAM_BASE\s*:?=\s*(0x[0-9A-Fa-f]+)", t, re.M): + out[m.group(1)] = int(m.group(2), 16) + return out + + +def twin_sets(aliases): + by_sha = collections.defaultdict(list) + con = contracts() + for a in aliases: + sha = (REPO / con[a]).read_text().split()[0] + by_sha[sha].append(a) + sets = {} + for sha, members in by_sha.items(): + if len(members) > 1: + for a in members: + sets[a] = tuple(sorted(members)) + return sets + + +def twin_of_map(): + out = {} + for p in sorted((REPO / "config").glob("*.mk")): + for m in re.finditer(r"^(\w+)_TWIN_OF\s*:?=\s*(\w+)", p.read_text(), re.M): + out[m.group(1)] = m.group(2) + return out + + +def sig_rows(alias): + p = REPO / ".run" / f"sig.{alias}.jsonl" + if not p.exists(): + sys.exit(f"share_census: {p.relative_to(REPO)} is missing — run `make tools-health` (or the sig-* targets) first; the census " + f"refuses to measure a fleet it cannot see (R43)") + rows = [] + for ln in p.read_text().splitlines(): + if not ln.strip(): + continue + r = json.loads(ln) + rows.append((int(r["addr"], 16), int(r.get("nins", 0)), r["h_exact"], r.get("name"))) + return rows + + +def curated_names(alias): + import corpus + try: + syms = corpus.symbols(alias) + except Exception: + syms = {} + return {addr: nm for nm, addr in syms.items()} + + +def load_groups(): + import yaml + d = yaml.safe_load((REPO / "config" / "dedup.us.yaml").read_text()) + return d["groups"] if isinstance(d, dict) else d + + +def verbatim_instances(): + d = json.loads((REPO / "config" / "verbatim_manifest.json").read_text()) + return {(r["binary"], int(r["addr"], 16)) for r in d["rows"]} + + +def load_exceptions(path): + out = {} + p = REPO / path + if not p.exists(): + return out + for ln in p.read_text().splitlines(): + if not ln.strip() or ln.startswith("#") or ln.startswith("h_exact\t"): + continue + parts = ln.split("\t") + out[parts[0]] = dict(reason=parts[1] if len(parts) > 1 else "?", note=parts[-1]) + return out + + +# ---------------------------------------------------------------------------------------------------------------------- +# the census +# ---------------------------------------------------------------------------------------------------------------------- +def build_forms(aliases, dirs, jobs, use_cache=True, out_dir=None): + """(alias -> {addr: rec}), coverage notes. One scan per TU, cached by (mtime, size).""" + import compile_only + # shared headers: which function names each defines (masked, so macro-only headers define nothing) + shared_defs = {} + for p in sorted((REPO / "src" / "shared").rglob("*.h")): + shared_defs[p.relative_to(REPO).as_posix()] = header_defined_names(p) + cache_p = (REPO / (out_dir or OUT_DIR_DEFAULT) / "cache" / "scan_cache.json") + cache = {} + if use_cache and cache_p.exists(): + try: + cache = json.loads(cache_p.read_text()) + except Exception: + cache = {} + work, tu_of = [], {} + for a in aliases: + for p in compile_only.tus_of(a, dirs): + rel = p.relative_to(REPO).as_posix() + st = p.stat() + tu_of.setdefault(a, []).append(rel) + key = f"{rel}|{int(st.st_mtime)}|{st.st_size}" + if key not in cache: + work.append((rel, int(st.st_mtime), st.st_size, shared_defs)) + scanned = {} + if work: + with ProcessPoolExecutor(max_workers=jobs) as ex: + for rel, recs in ex.map(_scan_worker, work, chunksize=8): + scanned[rel] = recs + new_cache = {} + forms, notes = {}, dict(unresolved_names=[], multi_form=[], stubs_unsigned=0) + for a in aliases: + names = curated_names(a) + rev = {nm: addr for addr, nm in names.items()} + per = {} + for rel in tu_of.get(a, []): + st = (REPO / rel).stat() + key = f"{rel}|{int(st.st_mtime)}|{st.st_size}" + recs = scanned.get(rel) if rel in scanned else cache.get(key) + if recs is None: + recs = scan_text((REPO / rel).read_text(errors="surrogateescape"), rel, shared_defs) + new_cache[key] = recs + for r in recs: + nm = r["name"] + m = FUNC_RE.match(nm) + addr = int(m.group(1), 16) if m else rev.get(nm) + if addr is None: + if r["form"] != "stub": + notes["unresolved_names"].append(f"{a}:{rel}:{r['line']}:{nm}:{r['form']}") + continue + rec = dict(r, tu=rel) + if addr in per and per[addr]["form"] != "stub" and rec["form"] != "stub": + notes["multi_form"].append(f"{a}:0x{addr:08X}:{per[addr]['form']}@{per[addr]['tu']}:{per[addr]['line']} vs " + f"{rec['form']}@{rel}:{rec['line']}") + continue + if addr in per and rec["form"] == "stub": + continue + per[addr] = rec + forms[a] = per + if use_cache: + cache_p.parent.mkdir(parents=True, exist_ok=True) + cache_p.write_text(json.dumps(new_cache)) + return forms, notes + + +def classify(sigs, forms, groups, spaces, dirs, twins, twin_of, verbatim, exceptions, scope="same-vram"): + """Pure: sigs {alias: [(addr,nins,h,name)]}, forms {alias: {addr: rec}}, groups (registry list), spaces {alias: base}, + dirs {alias: src dir}, twins {alias: twin-set tuple}, twin_of {alias: primary}, verbatim {(alias,addr)}, exceptions {h: ...}.""" + from dedup_integrate import group_members + # registry index + reg = collections.defaultdict(list) + for g in groups: + reg[g.get("hash")].append(g) + # instances + classes = collections.defaultdict(list) + unaccounted = [] + n_inst = 0 + for a, rows in sigs.items(): + per = forms.get(a, {}) + for addr, nins, h, name in rows: + n_inst += 1 + rec = per.get(addr) + if rec is None and (a, addr) in verbatim: + rec = dict(form="asm-verbatim", name=name, tu=None, line=None) + if rec is None: + unaccounted.append(f"{a}:0x{addr:08X}:{name}:nins={nins}") + continue + classes[h].append(dict(alias=a, addr=addr, nins=nins, name=name, form=rec["form"], tu=rec.get("tu"), + text_hash=rec.get("text_hash"), pins=rec.get("pins", False), typedef=rec.get("typedef", False), + alias_form=rec.get("alias", False))) + out = [] + for h, insts in classes.items(): + if len(insts) < 2: + continue + gs = reg.get(h, []) + members = set() + for g in gs: + for b, addr, _ in group_members(g): + members.add((b, addr)) + inst_set = {(i["alias"], i["addr"]) for i in insts} + forms_seen = collections.Counter(i["form"] for i in insts) + addrs = sorted({i["addr"] for i in insts}) + aliases = sorted({i["alias"] for i in insts}) + space_set = {spaces.get(i["alias"], -1) for i in insts} + flags = [] + if len(addrs) > 1: + flags.append("E") + if len(space_set) > 1: + flags.append("F") + src_dirs = {dirs.get(i["alias"]) for i in insts} + if len(aliases) > 1 and len(src_dirs) == 1: + flags.append("TWIN-COVERED") + elif len(aliases) > 1 and all(a in twins for a in aliases) and len({twins[a] for a in aliases}) == 1: + flags.append("TWIN-PENDING") + if any(i["typedef"] for i in insts): + flags.append("TYPEDEF") + if any(i["pins"] for i in insts): + flags.append("PINS") + if any(i["alias_form"] for i in insts): + flags.append("ALIAS") + if any(i["form"] == "stub" for i in insts): + flags.append("STUB") + if gs: + missing = sorted(inst_set - members) + extra = sorted(members - inst_set) + bad_site = [i for i in insts if (i["alias"], i["addr"]) in members and i["form"] not in SHARED_FORMS] + verdict = "A" if not missing and not bad_site and not extra else "B" + detail = dict(groups=[g.get("id") for g in gs], missing=len(missing), extra=len(extra), bad_site=len(bad_site)) + else: + defs = [i for i in insts if i["form"] == "def"] + if len(defs) == len(insts): + texts = {i["text_hash"] for i in insts} + verdict = "C" if len(texts) == 1 else "D" + detail = dict(texts=len(texts)) + else: + verdict = "M" + detail = dict(forms=dict(forms_seen)) + nins = max(i["nins"] for i in insts) + band = "1-7" if nins < 8 else "8-15" if nins < 16 else "16-31" if nins < 32 else "32+" + excepted = exceptions.get(h) + deferred = ("E" in flags or "F" in flags) and scope != "all" + violation = (verdict != "A") and ("TWIN-COVERED" not in flags) and not deferred and not excepted + out.append(dict(h=h, verdict=verdict, flags=flags, nins=nins, band=band, instances=len(insts), copies=forms_seen.get("def", 0), + aliases=aliases, addrs=[f"0x{x:08X}" for x in addrs], forms=dict(forms_seen), detail=detail, + excepted=(excepted or {}).get("reason"), deferred=deferred, violation=violation, + names=sorted({i["name"] for i in insts})[:4])) + out.sort(key=lambda c: (-c["instances"] * c["nins"], c["h"])) + return out, dict(instances=n_inst, classified=n_inst - len(unaccounted), unaccounted=unaccounted) + + +def summarize(classes, cov, notes, aliases): + by_v = collections.Counter(c["verdict"] for c in classes) + by_f = collections.Counter(f for c in classes for f in c["flags"]) + inst_by_v = collections.Counter() + ins_by_v = collections.Counter() + for c in classes: + inst_by_v[c["verdict"]] += c["instances"] + ins_by_v[c["verdict"]] += c["instances"] * c["nins"] + same_vram_unreg = [c for c in classes if c["verdict"] in ("B", "C", "D", "M") and "E" not in c["flags"] and "F" not in c["flags"]] + cross = [c for c in classes if ("E" in c["flags"] or "F" in c["flags"]) and c["verdict"] != "A"] + return dict( + binaries=len(aliases), sig_instances=cov["instances"], classified=cov["classified"], unaccounted=len(cov["unaccounted"]), + classes_ge2=len(classes), verdicts=dict(by_v), instances_by_verdict=dict(inst_by_v), ins_by_verdict=dict(ins_by_v), + flags=dict(by_f), + same_vram_unregistered=dict(classes=len(same_vram_unreg), instances=sum(c["instances"] for c in same_vram_unreg), + copies=sum(c["copies"] for c in same_vram_unreg), + collapsible=sum(c["instances"] - 1 for c in same_vram_unreg), + ins=sum(c["instances"] * c["nins"] for c in same_vram_unreg), + twin_pending=sum(1 for c in same_vram_unreg if "TWIN-PENDING" in c["flags"]), + twin_pending_instances=sum(c["instances"] for c in same_vram_unreg if "TWIN-PENDING" in c["flags"])), + cross_address_deferred=dict(classes=len(cross), instances=sum(c["instances"] for c in cross), + copies=sum(c["copies"] for c in cross)), + violations=sum(1 for c in classes if c["violation"]), + multi_form=len(notes.get("multi_form", [])), unresolved_names=len(notes.get("unresolved_names", [])), + ) + + +def controls(classes, sigs, forms): + """The known-true cases (R39). Each returns (name, got, expected).""" + out = [] + # func_80144B9C: the -O0 whale — 141 include sites, registered for 134 (never extended to the 7 later-onboarded overlays) + w = [c for c in classes if "0x80144B9C" in c["addrs"] and "ov_SC01_077" in c["aliases"]] + out.append(("func_80144B9C", f"{w[0]['verdict']}/{w[0]['instances']}/{w[0]['forms']}/missing {w[0]['detail'].get('missing')}" if w else "absent", + "B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it)")) + # clearTbl40: main 0x80037004 + 0x80037334, one class, registered, cross-vram + ct = [c for c in classes if "0x80037004" in c["addrs"] and "main" in c["aliases"]] + out.append(("clearTbl40", f"{ct[0]['verdict']}+{'E' if 'E' in ct[0]['flags'] else '-'}/{ct[0]['instances']}" if ct else "absent", "A+E/2")) + # ov_setters: the three SC01_005/006 groups — registered for 2 of the 141 instances each (the same trivial body sits at that + # vram in every overlay); B by construction until T5 extends them + # Measured S94: the 2-instruction bodies of two of them also sit at OTHER vrams (282 and 145 instances, cross-address), the + # third only at its own (141) — so the known-true answer is three B classes of 282(E) / 145(E) / 141 instances. + st = sorted([c for c in classes if any(x in c["addrs"] for x in ("0x8012AD64", "0x8012BF4C", "0x8012E27C")) and "ov_SC01_005" in c["aliases"]], + key=lambda c: -c["instances"]) + out.append(("ov_setters x3", ",".join(f"{c['verdict']}/{c['instances']}{'(E)' if 'E' in c['flags'] else ''}" for c in st) or "absent", + "B/282(E),B/145(E),B/141")) + # twin symmetry: ov_SC01_005 and ov_SC01_006 are one payload, so their unregistered-class counts must be EQUAL (a known-true + # invariant rather than a remembered number: S75 counted 557 at 174 binaries, the S94 Explore 657 at 218, this census's rule differs) + def unreg(alias): + return sum(1 for c in classes if c["verdict"] in ("C", "D", "M") and alias in c["aliases"]) + a5, a6 = unreg("ov_SC01_005"), unreg("ov_SC01_006") + out.append(("twin symmetry SC01_005 / SC01_006", f"{a5} / {a6}", "equal (each = the twin's unregistered classes)")) + return out + + +def render_table(classes, summary, cov_notes, ctrl): + L = [] + s = summary + L.append(f"share_census: {s['binaries']} binaries · {s['sig_instances']:,} sig instances · {s['classified']:,} classified · " + f"{s['unaccounted']} UNACCOUNTED · {s['classes_ge2']:,} h_exact classes with >=2 instances") + L.append(f" verdicts (classes): " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['verdicts'].items()))) + L.append(f" instances by verdict: " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['instances_by_verdict'].items()))) + L.append(f" flags: " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['flags'].items()))) + u = s["same_vram_unregistered"] + L.append(f" SAME-VRAM UNREGISTERED (the phase's backlog): {u['classes']:,} classes · {u['instances']:,} instances · {u['copies']:,} private " + f"copies · {u['collapsible']:,} collapsible · {u['ins']:,} ins; of which twin-pending {u['twin_pending']:,} classes / " + f"{u['twin_pending_instances']:,} instances") + d = s["cross_address_deferred"] + L.append(f" CROSS-ADDRESS / CROSS-SPACE (deferred to the names phase): {d['classes']:,} classes · {d['instances']:,} instances · " + f"{d['copies']:,} private copies") + L.append(f" S1 violations (scope as run): {s['violations']:,} · multi-form instances {s['multi_form']} · unresolved names {s['unresolved_names']}") + L.append(" controls (R39):") + for name, got, exp in ctrl: + L.append(f" {name:34s} got {got:40s} expected {exp}") + L.append(" top classes by instances x nins (verdict flags instances nins addrs names):") + for c in classes[:25]: + L.append(f" {c['verdict']} {','.join(c['flags']) or '-':22s} {c['instances']:4d} x {c['nins']:5d} {','.join(c['addrs'][:3])} {','.join(c['names'][:2])}") + if cov_notes.get("unaccounted"): + L.append(" UNACCOUNTED (first 20): " + "; ".join(cov_notes["unaccounted"][:20])) + return "\n".join(L) + + +# ---------------------------------------------------------------------------------------------------------------------- +# the second oracle: duplicate definition TEXT across translation units (sig-blind) +# ---------------------------------------------------------------------------------------------------------------------- +def text_duplicates(forms): + by_text = collections.defaultdict(set) + for a, per in forms.items(): + for addr, rec in per.items(): + if rec["form"] == "def" and rec.get("text_hash") and rec.get("nlines", 0) >= 2: + by_text[rec["text_hash"]].add((a, rec["tu"], addr)) + return {h: sorted(v) for h, v in by_text.items() if len({(a, tu) for a, tu, _ in v}) > 1} + + +# ---------------------------------------------------------------------------------------------------------------------- +# self-test fixture (R39): every verdict and flag, in memory +# ---------------------------------------------------------------------------------------------------------------------- +def selftest(): + A, B, C = "fx_A", "fx_B", "fx_C" + spaces = {A: 0x80128158, B: 0x80128158, C: 0x800CAE08} + dirs = {A: "src/fx_A", B: "src/fx_B", C: "src/fx_C"} + twins = {} + sigs = {A: [], B: [], C: []} + forms = {A: {}, B: {}, C: {}} + groups = [] + + def put(alias, addr, h, form, text="t", nins=12, **kw): + sigs[alias].append((addr, nins, h, f"func_{addr:08X}")) + forms[alias][addr] = dict(form=form, name=f"func_{addr:08X}", tu=f"{dirs[alias]}/x.c", line=1, text_hash=text, **kw) + + # 1 A: registered, both macro sites + put(A, 0x80130000, "h1", "macro"); put(B, 0x80130000, "h1", "macro") + groups.append(dict(id="g1", hash="h1", vram=0x80130000, binaries=[A, B])) + # 2 B: registered but B's instance is missing from the group + put(A, 0x80130010, "h2", "macro"); put(B, 0x80130010, "h2", "def") + groups.append(dict(id="g2", hash="h2", vram=0x80130010, binaries=[A])) + # 3 C: unregistered, identical text + put(A, 0x80130020, "h3", "def", text="same"); put(B, 0x80130020, "h3", "def", text="same") + # 4 D: unregistered, differing text + put(A, 0x80130030, "h4", "def", text="x"); put(B, 0x80130030, "h4", "def", text="y", pins=True) + # 5 C+E: cross-vram identical + put(A, 0x80130040, "h5", "def", text="same"); put(B, 0x80130050, "h5", "def", text="same") + # 6 C+F: cross-space + put(A, 0x80130060, "h6", "def", text="same"); put(C, 0x800CB000, "h6", "def", text="same") + # 7 M: unregistered mixed (a macro site in one binary, a private copy in the other) + put(A, 0x80130070, "h7", "macro"); put(B, 0x80130070, "h7", "def", typedef=True) + # a singleton (must not be a class) and an unaccounted instance (must be reported) + put(A, 0x80130080, "h8", "def") + sigs[B].append((0x80130090, 4, "h9", "func_80130090")) + classes, cov = classify(sigs, forms, groups, spaces, dirs, twins, {}, set(), {}, scope="same-vram") + got = {c["h"]: (c["verdict"], tuple(c["flags"])) for c in classes} + exp = {"h1": ("A", ()), "h2": ("B", ()), "h3": ("C", ()), "h4": ("D", ("PINS",)), "h5": ("C", ("E",)), "h6": ("C", ("E", "F")), + "h7": ("M", ("TYPEDEF",))} + ok = 0 + for h, e in exp.items(): + g = got.get(h) + if g == e: + ok += 1 + else: + print(f"selftest: {h}: got {g} expected {e}") + if "h8" in got: + print("selftest: a singleton became a class"); ok -= 1 + if cov["unaccounted"] != [f"{B}:0x80130090:func_80130090:nins=4"]: + print(f"selftest: unaccounted reporting wrong: {cov['unaccounted']}"); ok -= 1 + viol = sorted(c["h"] for c in classes if c["violation"]) + if viol != ["h2", "h3", "h4", "h7"]: + print(f"selftest: violations wrong: {viol} (E/F deferred by default)"); ok -= 1 + classes_all, _ = classify(sigs, forms, groups, spaces, dirs, twins, {}, set(), {"h3": dict(reason="TEST")}, scope="all") + viol_all = sorted(c["h"] for c in classes_all if c["violation"]) + if viol_all != ["h2", "h4", "h5", "h6", "h7"]: + print(f"selftest: --scope all / exceptions wrong: {viol_all}"); ok -= 1 + # the masker + scanner on a synthetic TU + tu = ('#include "common.h"\n#include "../shared/engine_core.h"\n\nDEFINE_func_80128158() /* dedup: shared */\n' + '/* void func_DEAD0001(void) { } */\n#if 0\nvoid func_DEAD0002(void) { }\n#endif\n' + 'extern s32 D_1;\ns32 func_80128218(void) {\n return D_1; /* } */\n}\n' + 'int aF80128300(int a) __asm__("func_80128300");\nint aF80128300(int a) {\n register int r __asm__("$16");\n return a;\n}\n' + 'INCLUDE_ASM("asm/x", func_80128400);\nSETTER(func_8012AD64, 0x34, s16)\n' + '#define SHARED_FN func_80037334\n#include "../shared/main/func_80037004__a0744d60.h"\n#undef SHARED_FN\n' + 'void\nfunc_80128500(void)\n{\n}\n') + sd = {"src/shared/main/func_80037004__a0744d60.h": ["SHARED_FN"], "src/shared/engine_core.h": []} + recs = scan_text(tu, "src/fx_A/x.c", shared_defs=sd) + want = {("macro", "func_80128158"), ("def", "func_80128218"), ("def", "func_80128300"), ("stub", "func_80128400"), + ("macro-param", "func_8012AD64"), ("param-include", "func_80037334"), ("def", "func_80128500")} + gotset = {(r["form"], r["name"]) for r in recs} + if gotset != want: + print(f"selftest: scanner: got {sorted(gotset)}\n want {sorted(want)}"); ok -= 1 + else: + pins = [r for r in recs if r["form"] == "def" and r["name"] == "func_80128300"][0] + if not (pins["pins"] and pins["alias"]): + print("selftest: the alias-form def lost its PINS/ALIAS flags"); ok -= 1 + total = len(exp) + print(f"selftest: {ok}/{total} verdicts correct" + ("" if ok == total else " — FAIL")) + return ok == total + + +# ---------------------------------------------------------------------------------------------------------------------- +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--check", action="store_true", help="the S1 invariant; exit 1 on any violation") + ap.add_argument("--strict-macros", action="store_true", help="with --check: any DEFINE_func_ token under src/ is a violation") + ap.add_argument("--strict-text", action="store_true", help="with --check: any definition text duplicated across TUs is a violation") + ap.add_argument("--scope", choices=["same-vram", "all"], default="same-vram") + ap.add_argument("--exceptions", default="config/dedup_exceptions.tsv") + ap.add_argument("--out-dir", default=OUT_DIR_DEFAULT) + ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 4) + ap.add_argument("--no-cache", action="store_true") + ap.add_argument("--selftest", action="store_true") + ap.add_argument("--quiet", action="store_true") + a = ap.parse_args() + if a.selftest: + sys.exit(0 if selftest() else 1) + t0 = time.time() + aliases, dirs = fleet_and_dirs() + spaces = vram_bases() + missing_space = [x for x in aliases if x not in spaces] + if missing_space: + sys.exit(f"share_census: no _VRAM_BASE for {missing_space[:5]} (R32)") + twins = twin_sets(aliases) + twin_of = twin_of_map() + sigs = {x: sig_rows(x) for x in aliases} + forms, notes = build_forms(aliases, dirs, a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir) + groups = load_groups() + exceptions = load_exceptions(a.exceptions) + verb = verbatim_instances() + classes, cov = classify(sigs, forms, groups, spaces, dirs, twins, twin_of, verb, exceptions, scope=a.scope) + summary = summarize(classes, cov, notes, aliases) + ctrl = controls(classes, sigs, forms) + macro_tokens = 0 + for x in aliases: + for per in [forms[x]]: + macro_tokens += sum(1 for r in per.values() if r["form"] == "macro") + dup_text = text_duplicates(forms) + summary["macro_sites"] = macro_tokens + summary["duplicate_text_classes"] = len(dup_text) + summary["duplicate_text_sites"] = sum(len(v) for v in dup_text.values()) + summary["twin_sets"] = sorted({v for v in twins.values()}) + summary["elapsed_s"] = round(time.time() - t0, 1) + summary["controls"] = [dict(name=n, got=g, expected=e) for n, g, e in ctrl] + out = REPO / a.out_dir + out.mkdir(parents=True, exist_ok=True) + (out / "share_census.json").write_text(json.dumps(summary, indent=1, sort_keys=True) + "\n") + with open(out / "classes.jsonl", "w") as f: + for c in classes: + f.write(json.dumps(c) + "\n") + (out / "coverage_notes.txt").write_text("\n".join( + ["# unaccounted sig instances (alias:addr:name:nins)"] + cov["unaccounted"] + + ["# instances with >=2 non-stub forms"] + notes["multi_form"] + + ["# unresolved (non-func_, not in the binary's symbol stack) definition/site names"] + notes["unresolved_names"]) + "\n") + table = render_table(classes, summary, cov, ctrl) + table += f"\n macro sites {macro_tokens:,} · duplicate-text classes {len(dup_text):,} ({summary['duplicate_text_sites']:,} sites) · elapsed {summary['elapsed_s']} s" + (out / "share_census.txt").write_text(table + "\n") + if not a.quiet: + print(table) + rc = 0 + if cov["unaccounted"]: + print(f"share_census: COVERAGE DEFECT — {len(cov['unaccounted'])} sig instance(s) have no source form (R32); see {a.out_dir}/coverage_notes.txt") + rc = 1 + if notes["multi_form"]: + print(f"share_census: COVERAGE DEFECT — {len(notes['multi_form'])} instance(s) carry two non-stub forms; see coverage_notes.txt") + rc = 1 + if a.check: + v = [c for c in classes if c["violation"]] + bad = len(v) + if a.strict_macros and macro_tokens: + print(f"share_census: S1 — {macro_tokens} DEFINE_func_ site(s) remain under src/ (--strict-macros)"); bad += 1 + if a.strict_text and dup_text: + print(f"share_census: S1 — {len(dup_text)} definition text(s) duplicated across TUs (--strict-text)"); bad += 1 + exc = sum(1 for c in classes if c["excepted"]) + twinc = sum(1 for c in classes if "TWIN-COVERED" in c["flags"] and c["verdict"] != "A") + print(f"S1: one source per unique function — {len(classes):,} classes, {len(classes) - len(v):,} satisfied " + f"({twinc:,} twin-covered, {exc:,} excepted, {summary['cross_address_deferred']['classes']:,} deferred cross-address), " + f"{len(v):,} VIOLATION(S)" + (" — OK" if bad == 0 else " — FAIL")) + for c in v[:15]: + print(f" {c['verdict']} {','.join(c['flags']) or '-'} h={c['h'][:10]} {c['instances']} inst x {c['nins']} ins {c['addrs'][:2]} {c['aliases'][:3]}") + if bad: + rc = 1 + sys.exit(rc) + + +if __name__ == "__main__": + main()