mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-28 14:59:48 -04:00
tools(phase-35): T1 — share_census.py, the fleet census of byte-identical function classes + the S1 'one source per unique function' checker with its exception ledger; 362,389/362,389 sig instances covered, 10,180 classes (A 1,712 · B 282 · C 6,107 · D 1,861 · M 218), self-test 7/7, four controls as measured; SETUP + dictionary rows, kit corpus regenerated
This commit is contained in:
Drew T
@@ -0,0 +1,3 @@
|
||||
# unaccounted sig instances (alias:addr:name:nins)
|
||||
# instances with >=2 non-stub forms
|
||||
# unresolved (non-func_, not in the binary's symbol stack) definition/site names
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"binaries": 218,
|
||||
"classes_ge2": 10180,
|
||||
"classified": 362389,
|
||||
"controls": [
|
||||
{
|
||||
"expected": "B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it)",
|
||||
"got": "B/141/{'include': 141}/missing 7",
|
||||
"name": "func_80144B9C"
|
||||
},
|
||||
{
|
||||
"expected": "A+E/2",
|
||||
"got": "A+E/2",
|
||||
"name": "clearTbl40"
|
||||
},
|
||||
{
|
||||
"expected": "B/282(E),B/145(E),B/141",
|
||||
"got": "B/282(E),B/145(E),B/141",
|
||||
"name": "ov_setters x3"
|
||||
},
|
||||
{
|
||||
"expected": "equal (each = the twin's unregistered classes)",
|
||||
"got": "653 / 653",
|
||||
"name": "twin symmetry SC01_005 / SC01_006"
|
||||
}
|
||||
],
|
||||
"cross_address_deferred": {
|
||||
"classes": 3801,
|
||||
"copies": 12938,
|
||||
"instances": 30347
|
||||
},
|
||||
"duplicate_text_classes": 6845,
|
||||
"duplicate_text_sites": 23269,
|
||||
"elapsed_s": 41.1,
|
||||
"flags": {
|
||||
"ALIAS": 44,
|
||||
"E": 3826,
|
||||
"F": 54,
|
||||
"PINS": 687,
|
||||
"TWIN-PENDING": 3748,
|
||||
"TYPEDEF": 96
|
||||
},
|
||||
"ins_by_verdict": {
|
||||
"A": 6048041,
|
||||
"B": 1175147,
|
||||
"C": 663203,
|
||||
"D": 165250,
|
||||
"M": 28564
|
||||
},
|
||||
"instances_by_verdict": {
|
||||
"A": 214478,
|
||||
"B": 45226,
|
||||
"C": 14839,
|
||||
"D": 5796,
|
||||
"M": 462
|
||||
},
|
||||
"macro_sites": 255947,
|
||||
"multi_form": 0,
|
||||
"same_vram_unregistered": {
|
||||
"classes": 4667,
|
||||
"collapsible": 31309,
|
||||
"copies": 11767,
|
||||
"ins": 1710469,
|
||||
"instances": 35976,
|
||||
"twin_pending": 3568,
|
||||
"twin_pending_instances": 7136
|
||||
},
|
||||
"sig_instances": 362389,
|
||||
"twin_sets": [
|
||||
[
|
||||
"ov_SC01_005",
|
||||
"ov_SC01_006"
|
||||
],
|
||||
[
|
||||
"ov_SC02_000",
|
||||
"ov_SC02_003"
|
||||
],
|
||||
[
|
||||
"ov_SC03_014",
|
||||
"ov_SC03_015"
|
||||
],
|
||||
[
|
||||
"ov_SC03_118",
|
||||
"ov_SC03_119"
|
||||
],
|
||||
[
|
||||
"ov_SC04_018",
|
||||
"ov_SC04_019"
|
||||
]
|
||||
],
|
||||
"unaccounted": 0,
|
||||
"unresolved_names": 0,
|
||||
"verdicts": {
|
||||
"A": 1712,
|
||||
"B": 282,
|
||||
"C": 6107,
|
||||
"D": 1861,
|
||||
"M": 218
|
||||
},
|
||||
"violations": 4667
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
share_census: 218 binaries · 362,389 sig instances · 362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes with >=2 instances
|
||||
verdicts (classes): A 1,712 · B 282 · C 6,107 · D 1,861 · M 218
|
||||
instances by verdict: A 214,478 · B 45,226 · C 14,839 · D 5,796 · M 462
|
||||
flags: ALIAS 44 · E 3,826 · F 54 · PINS 687 · TWIN-PENDING 3,748 · TYPEDEF 96
|
||||
SAME-VRAM UNREGISTERED (the phase's backlog): 4,667 classes · 35,976 instances · 11,767 private copies · 31,309 collapsible · 1,710,469 ins; of which twin-pending 3,568 classes / 7,136 instances
|
||||
CROSS-ADDRESS / CROSS-SPACE (deferred to the names phase): 3,801 classes · 30,347 instances · 12,938 private copies
|
||||
S1 violations (scope as run): 4,667 · multi-form instances 0 · unresolved names 0
|
||||
controls (R39):
|
||||
func_80144B9C got B/141/{'include': 141}/missing 7 expected B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it)
|
||||
clearTbl40 got A+E/2 expected A+E/2
|
||||
ov_setters x3 got B/282(E),B/145(E),B/141 expected B/282(E),B/145(E),B/141
|
||||
twin symmetry SC01_005 / SC01_006 got 653 / 653 expected equal (each = the twin's unregistered classes)
|
||||
top classes by instances x nins (verdict flags instances nins addrs names):
|
||||
B - 141 x 770 0x80144B9C func_80144b9c
|
||||
A - 141 x 476 0x80141CA4 func_80141ca4
|
||||
A - 141 x 400 0x80132784 func_80132784
|
||||
A E 1128 x 41 0x80151FB4,0x801522CC,0x801525F4 func_80151fb4,func_801522cc
|
||||
A - 141 x 284 0x8012D714 func_8012d714
|
||||
A - 141 x 254 0x8015126C func_8015126c
|
||||
A - 141 x 248 0x8014EE14 func_8014ee14
|
||||
A - 141 x 214 0x80156B74 func_80156b74
|
||||
A - 141 x 213 0x80148094 func_80148094
|
||||
A - 141 x 207 0x801372B0 func_801372b0
|
||||
B PINS 141 x 204 0x8013A530 func_8013a530
|
||||
A - 141 x 198 0x801412A8 func_801412a8
|
||||
A - 141 x 198 0x801571C4 func_801571c4
|
||||
A - 141 x 195 0x80153E00 func_80153e00
|
||||
A - 141 x 191 0x80129CF8 func_80129cf8
|
||||
A - 141 x 189 0x8012D098 func_8012d098
|
||||
A E 282 x 93 0x8014FFDC,0x801502EC func_8014ffdc,func_801502ec
|
||||
A - 141 x 185 0x8013AF20 func_8013af20
|
||||
A - 141 x 183 0x8014EA4C func_8014ea4c
|
||||
A - 141 x 182 0x801392FC func_801392fc
|
||||
A - 141 x 178 0x8012EC04 func_8012ec04
|
||||
A - 141 x 174 0x8014F74C func_8014f74c
|
||||
B E,F 11852 x 2 0x80014554,0x8001455C,0x8001513C func_80014554,func_8001455c
|
||||
A - 141 x 167 0x80163C2C func_80163c2c
|
||||
A - 141 x 165 0x80178004 func_80178004
|
||||
macro sites 255,947 · duplicate-text classes 6,845 (23,269 sites) · elapsed 41.1 s
|
||||
@@ -0,0 +1,10 @@
|
||||
# config/dedup_exceptions.tsv — the S1 exception ledger (Phase 35): a byte-identical class with >=2 instances that does NOT share
|
||||
# one source, with the MEASURED reason. Read by tools/share_census.py --check; one row per h_exact class. Reason codes:
|
||||
# JTBL-CARVE a member's body owns a jump-table carve that the shared form cannot carry
|
||||
# O0-LOCAL a member lives in a per-binary -O0 split translation unit
|
||||
# TU-CONFLICT the chosen text is refused by a member's translation unit (a conflicting declaration)
|
||||
# GATE-REJECT the chosen text compiled but the member's binary did not stay byte-identical (the compiler message class in note)
|
||||
# PINNED every candidate text carries register pins and sharing would spread them (Phase 36's work)
|
||||
# CROSS-SPACE instances in different address spaces (deferred with the cross-address classes)
|
||||
# A row is evidence, not a shrug: note names the tool run and the message. Empty at the phase's open (2026-09-08).
|
||||
h_exact reason nins instances note
|
||||
|
@@ -327,3 +327,4 @@ tools/worklist.py P5 ADAPT join the target pool and near-miss ledger into one by
|
||||
tools/xsig/tests/make_fixtures.sh P6 ADAPT regenerate the signature tool's fixtures with the pinned toolchain Regenerates the signature-tool fixtures with the pinned toolchain compiler triple, repo tool paths LIVE
|
||||
tools/xsig/tests/test_xsig.py P6 PORTABLE property-test the signature tool on committed fixtures, needing no compiler Property tests for the signature tool on committed fixtures, needing no compiler fixture paths LIVE
|
||||
tools/xsig/xsig.py P6 PORTABLE sign functions with relocations masked for cross-project code identification Relocation-masked per-function signatures for cross-project code identification MIPS/relocation model only LIVE
|
||||
tools/share_census.py P35 ADAPT measure duplicate function bodies across the fleet and assert one source per unique function The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test repo paths, the registry and signature schemas LIVE
|
||||
|
||||
|
@@ -776,6 +776,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo
|
||||
| | `tools/gitignore_template_check.py` | **(P33.5 task 7)** The ```` ```gitignore ```` fence of `docs/wiki/The-ROM-firewall.md` must equal `decomp-architect/templates/gitignore.decomp` byte for byte (one source, two copies; R75-shaped). rc 1 on drift, rc 2 when the template does not exist yet ("nothing to compare" — never a pass, R43); refuses a page with ≠ 1 fence. In `tools-health` behind an existence test that skips loudly until the kit lands (task 11). |
|
||||
| | `tools/kit_lint.py [--selftest] [--paths …]` | **(P33.5 task 11)** The day-one decomp kit (`decomp-architect/`) stays de-specialised: (1) LEAK — no line outside a ```` ```calibration ```` fence and off a `provenance:` line matches `SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\.17\.|\bR[0-9]{1,2}\b|§[0-9]+` (fence-aware: `grep -v calibration` would drop only lines containing the word); (2) PLACEHOLDERS — the `{{NAME}}` set used under the package equals the backticked set in `templates/PLACEHOLDERS.md` (the contract); (3) SYNTAX — `bash -n` / `py_compile` (no bytecode written) / JSON+YAML parse; (4) the gitignore template diff (delegated); (5) `TODO(platform)` / `TODO(phase-N)` counts; (6) coverage — zero files is a failure. rc 1 findings, rc 2 package absent (R43). `--selftest` = the R39 control (a planted leak line + a planted unlisted placeholder must be caught, fenced and provenance lines must not). In `tools-health` (selftest, then the real run). |
|
||||
| | `tools/tool_census.py [--check | --manifest | --corpus | --all | --consumers FILE]` | **(P33.5 task 13.5)** The tools audit as a derived instrument: two independent enumerations of every tool file under `tools/` (`find` vs `git ls-files`, submodules/vendored/downloaded excluded — they must agree, R34); per tool the docstring line, its SETUP row, its CONSUMERS (Makefile/`.mk` targets, CI, the wave playbook, other tools by import or by name) and hence its class (LIVE · REFERENCED · ORPHAN); the AUTHORED facts live in `config/tool_dictionary.tsv` (phase · portability · the NEED the tool answers · what · what it hard-codes · the retirement verdict with its successor or product) with coverage asserted BOTH ways (R32 — a new tool without a row fails `--check`). Generates `docs/tool-index.md` (the need-keyed dictionary; KEEP-GEN), the kit's `tools/MANIFEST.md` (`--manifest`) and the three verbatim corpora `decomp-architect/corpus/tools/<phase>/` + `corpus/cookbook/` + `corpus/record/` (the how-to, the decision log, the accelerators, the retrospective, the story, the playbook, the effort map, the gen3 docs, the digest, every PhaseEnd — task 14.5) (`--corpus`; superseded tools as pointer files; sha1-equal to their sources). `--check` in `tools-health`; `make kit-corpus` = `--all`. `--consumers FILE` is the referrer census before any `git mv` of a tool. |
|
||||
| | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. |
|
||||
| | `tools/kit_coverage.py` | **(P33.5 task 14.5)** The kit's DISTILLATION coverage: derives the rule population (every `- **R<n>` of DIGEST §3, asserted contiguous) and the hindsight population (every `## ` heading of `docs/accelerators.md` at numbered-item granularity — 58 at S92) and asserts each is cited by a `provenance:` line of the registry seed / the kernels OR dispositioned in `config/kit_coverage_map.tsv` (`G<id>` / `DK-<n>` / `FOLDED:G<id>` / `ENV` / `PA` / `SEED:<file>` / `KIT:<path>` / `RECORD` / `COOKBOOK` / `NOT-PORTABLE`; unknown ids refused, R43); counts with denominators (R41); rc 1 on any gap. In `tools-health` after `tool_census --check`. Its first run found 26 uncited rules and 21 uncited entries → three new kernels (DK-66–DK-68) and 41 authored dispositions. |
|
||||
| | `decomp-architect/` (the day-one decomp kit) | **(P33.5 tasks 9–14)** The package a new matching-decomp project installs as Phase 0.5 on ProjectArchitect 2.0: `README.md` (the three steps), `intake.decomp.md` (ProjectArchitect's twelve items pre-answered + the phase ladder + the six readability inversions), `SETUP.md` (the installer, Step 0 contract … Step 10 verify + hard stop; `answers: <path>` for unattended runs), `decomp-architect.md` (the methodology), `templates/` (the firewall pack — `gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, the planted fixture, `no-rom.template.yml` —, the READMEs, `pa-overlays.md`, `registry-E.decomp.md` G1–G67, the skeletons, `PLACEHOLDERS.md`, `layout-contract.md`), `corpus/decomp-kernels.md` (DK-1 … DK-80), the three dictionaries `corpus/tools/<phase>/` + `corpus/cookbook/` + `corpus/record/` (generated by `make kit-corpus`), `memory-seed/` (18), `tools/MANIFEST.md` (generated). **How it is checked:** `tools/kit_lint.py` (de-specialisation, placeholders, syntax, the gitignore-template diff) + `tools/tool_census.py --check` (the corpora equal their sources) + `tools/gitignore_template_check.py`, all in `tools-health`; the dry-run harness under `.run/P33.5/kit-dryrun/` (`answers.md`, `expected-manifest.txt`, `judge.py`, the install logs and verdicts of runs 1–5 — a kit change is re-verified by a resume on the last throwaway `repo/`, a fresh full run only when SETUP's steps change; the judge compares the real tree's dirty PATH SETS before/after). Wiki page: `docs/wiki/Start-a-new-decomp-project.md`. Split into its own repository after the flip. |
|
||||
| **Verification** | `tools/verify_contract.sh` | **(P33 A5/C8) THE recorded contract run**: 00 tree · 01 check-env · 02 family_hseq · 03 `make clean && extract-all && check-all` · 04 sdk-dual (or a recorded SKIP) · 05 tools-health (zero `[warn]`) · 06 audit-frontier · 07 audit-disc · 08 report; one log per step ending `EXIT=<rc>`, abort on the first red (R53), every step asserted by its contract line (R49), `SUMMARY.md` generated → `.run/P33/verify/` (tracked evidence, quoted by `docs/verification.md` §2); step 00 ignores its own output dir. ≈14 min on 32 CPUs. |
|
||||
@@ -1509,6 +1510,21 @@ fills fast). Nothing is leaking — but the host does not get the memory back on
|
||||
or a quoted include that resolves to no file / outside the repo is an offender. R39 controls: 34 offenders in 19 files
|
||||
before the fix (all ABSOLUTE), 0 after; 4,299 sources scanned (the coverage line, R32).
|
||||
|
||||
### P35 T1 (S94, 2026-09-08) — the share census and the S1 invariant ("one source per unique function")
|
||||
- **`tools/share_census.py`** — the census of byte-identical function classes (keyed by `h_exact`, never by name — R48) across all
|
||||
218 binaries, and the S1 checker. One masked pass per translation unit yields every instance's source form (`macro` / `macro-param` /
|
||||
`include` / `param-include` / `def` / `stub` / `asm-verbatim`); a sig instance with zero or two forms fails the run (R32 — the
|
||||
first run found 2,360 K&R definitions the scanner could not see; the last found 0 of 362,389). Verdicts A/B/C/D/M and flags
|
||||
E/F/TWIN-COVERED/TWIN-PENDING/TYPEDEF/PINS/ALIAS as in the docstring. The fleet, the source dirs, the contracts (`<alias>_CHECK_SHA`),
|
||||
the address spaces (`<alias>_VRAM_BASE`) and the twin sets (equal contracts) are derived from the Makefile and `config/` (R33);
|
||||
groups only through `dedup_integrate.group_members`. `--check` (exit code) = S1 with `config/dedup_exceptions.tsv` as the ledger
|
||||
(`h_exact reason nins instances note`; reason codes in the file header); `--strict-macros` / `--strict-text` are the
|
||||
post-conversion halves (T7 wires them into `tools-health`); `--scope all` counts the deferred cross-address classes as violations.
|
||||
`--selftest` is the R39 fixture (7 verdicts + the scanner on a synthetic TU). Outputs under `.run/P35/census/`: `share_census.json`
|
||||
(summary + controls), `share_census.txt` (the table), `coverage_notes.txt` (tracked); `classes.jsonl` and `cache/` (ignored).
|
||||
**Measured S94 on the pre-conversion tree:** 10,180 classes; A 1,712 · B 282 · C 6,107 · D 1,861 · M 218; same-vram unregistered
|
||||
4,667 classes / 11,767 private copies; cross-address deferred 3,801 classes; 43 s (16 workers).
|
||||
|
||||
### P33 B1 (S86, 2026-09-06) — `make disc-extract`: the rom→decoder step, promoted into the build
|
||||
- **`make disc-extract`** (`DISC_DIR ?= disks`): the repository ships no ROM bytes (H1 in force). The target (1) probes
|
||||
`extracted/retail/` against the committed oracle (`extract.py --verify`, **0.7 s** when up to date → no-op), else (2)
|
||||
|
||||
@@ -34,7 +34,8 @@
|
||||
the probe on ov_SC06_033: **34/34 objects byte-identical** in the include form, the binary `BYTE-IDENTICAL` both ways, build 1.12 s →
|
||||
0.52 s wall (CPU 12.7 s → 5.7 s), warnings 801 → 663 (all 137 macro-redefinition warnings gone), `.d` 11.6 KB → 182 KB
|
||||
(`.run/P35/probe/probe_ledger.txt`).
|
||||
- ☐ **T1** — `tools/share_census.py` + `config/dedup_exceptions.tsv` (the census + the S1 checker; `--selftest` 7/7; the four negative controls).
|
||||
- ☑ **T1** (S94) — `tools/share_census.py` + `config/dedup_exceptions.tsv`: `selftest: 7/7 verdicts correct`; `362,389 sig instances ·
|
||||
362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes`; the four controls as measured (see the log); SETUP + dictionary rows.
|
||||
- ☐ **T2** — the health chain learns the header + twin forms while the old tree is still green (the source-dir oracle; corpus/cdecl/
|
||||
overlay_src_split/progress/dedup_integrate/audit_binaries/lint_symbol_refs/shared_lock/fix_arity_callers/family_remap/blocker_probe/
|
||||
demacroize/export_pairs/harvest_verify); negative control: `make tools-health` OK on the UNCHANGED tree.
|
||||
@@ -89,6 +90,27 @@
|
||||
- **S94 — T0 tools-health, run 3 GREEN:** `tools-health: OK — sigs fresh; corpus(+resident) + cdecl + binaries + report(lint+dedup) +
|
||||
cookbook-index all green.` `exit=0`, `wall=474.23 s` (`.run/P35/baseline/tools_health_open3.log`). `docs/story-timeline.md/.svg`
|
||||
regenerated by the chain (74 rows; the v2.0.0 tag row). **T0 ☑.**
|
||||
- **S94 — T1 `tools/share_census.py` + `config/dedup_exceptions.tsv`.** Built as designed (one pass per TU with comments, dead
|
||||
`#if 0`/`NON_MATCHING` halves and macro-continuation text masked; forms macro / macro-param / include / param-include / def / stub /
|
||||
asm-verbatim; addresses from each binary's symbol stack, never names; the fleet, the source dirs, the contracts (`<alias>_CHECK_SHA`,
|
||||
main = `check.us.sha`), the address spaces (`_VRAM_BASE`) and the twin sets derived from the Makefile and the contracts; classes
|
||||
through `dedup_integrate.group_members`). Five scanner defects found by its own self-test and coverage line, each fixed at the cause:
|
||||
string contents were blanked before include paths and asm labels were read; K&R definitions (a blank tail after the last `;`,
|
||||
and the one-line `void f(a, b) void *a; s16 b; {` form); a second definition head on the same line after an `extern …;`; the
|
||||
alias regex spanning a previous macro site's parentheses; `asm(` as well as `__asm__(`; implicit-return-type heads with no type
|
||||
line; `static inline` helpers (no address) excluded from the unresolved list. **Result: `--selftest` → `selftest: 7/7 verdicts
|
||||
correct`; the census → `218 binaries · 362,389 sig instances · 362,389 classified · 0 UNACCOUNTED · 10,180 h_exact classes with >=2
|
||||
instances`; verdicts A 1,712 · B 282 · C 6,107 · D 1,861 · M 218 (instances 214,478 / 45,226 / 14,839 / 5,796 / 462); flags ALIAS 44 ·
|
||||
E 3,826 · F 54 · PINS 687 · TWIN-PENDING 3,748 · TYPEDEF 96; SAME-VRAM UNREGISTERED 4,667 classes · 35,976 instances · 11,767 private
|
||||
copies · 31,309 collapsible · 1,710,469 ins (twin-pending 3,568 classes / 7,136 instances); CROSS-ADDRESS/SPACE deferred 3,801 classes ·
|
||||
30,347 instances · 12,938 private copies; macro sites 255,947; duplicate-text classes 6,845 (23,269 sites); 43 s uncached, 41 s cached
|
||||
(the cache is not the cost — profile at T7); exit 0.** Controls (R39): `func_80144B9C` = B/141/{'include': 141}/missing 7 (the registry
|
||||
lists 134 — never extended to the 7 later overlays; T5 bucket 0), `clearTbl40` = A+E/2, the three setters = B/282(E), B/145(E), B/141
|
||||
(the same 2-instruction bodies also sit at other vrams), twin symmetry SC01_005/006 = 653/653. **Instrument findings for the record:**
|
||||
the B class is 282 registered-but-incomplete groups (204 never extended, 74 with a private-copy site — demacroized escapes —, 2 with a
|
||||
member the sigs do not show); a 2-instruction empty-body class (`jr ra; nop`) has 11,852 instances across every space (E,F → deferred
|
||||
with the cross-address set; the names phase decides how an empty function is written). Outputs `.run/P35/census/{share_census.json,
|
||||
share_census.txt, coverage_notes.txt}` (tracked), `classes.jsonl` + `cache/` (ignored). SETUP + dictionary rows (R21/R87). **T1 ☑.**
|
||||
|
||||
## Approved plan (verbatim, gate 1 — 2026-09-08)
|
||||
|
||||
@@ -382,7 +404,7 @@ same commit — history keeps it.
|
||||
Candidate for the PhaseEnd: "a shared body has exactly one source; a duplicate copy is a defect the health chain asserts, and a
|
||||
count of them is published with its rule" (this phase's invariant).
|
||||
|
||||
## 🛑 SESSION CHECKPOINT — S94 (2026-09-08): Phase 35 OPEN at gate 1; T0 ☑ (baseline 218/218 in 157 s; tools-health OK; probe 34/34); NEXT = T1 `tools/share_census.py`
|
||||
## 🛑 SESSION CHECKPOINT — S94 (2026-09-08): Phase 35 OPEN at gate 1; T0 ☑, T1 ☑ (the census: 362,389/362,389 covered, 10,180 classes, self-test 7/7); NEXT = T2 (the health chain learns the header + twin forms)
|
||||
|
||||
### 0. How to use this block
|
||||
A fresh session reads CLAUDE.md's load order, replays THIS block verbatim, and resumes at the first unchecked task in §Tasks above.
|
||||
@@ -409,11 +431,12 @@ P5's conditions. Commit per task; banks the moment they are green (R42); Drew pu
|
||||
- **T0 — DONE.** The L0 recipe that worked (reuse in T4/T5): copy the A objects from `build/src/<bin>/*.o`, apply the edit, `make check
|
||||
BINARY=<bin> -j16`, copy the B objects, `cmp` per object. The probe converter's parser (`macro_index` last-wins, the one site shape,
|
||||
the prelude = `engine_types.h` + the `ENGINE_SHB` line copied from `engine_core.h:20`) is the seed of `tools/macro_to_header.py`.
|
||||
- **T1:** design from the plan's "The three tools" §share_census; sigs enumerated from `dup_report.BINARIES` (218; never a glob — `.run/`
|
||||
holds 223 sig files); parse the registry ONLY through `dedup_integrate.group_members`; the source-form classifier must mirror
|
||||
`dedup_propagate.find_site` (a prototype is not a definition) and resolve the asm-label alias form (`family_remap._alias_decl_for`);
|
||||
negative controls: `func_80144B9C` A/141, clearTbl40 A+E, the 3 ov_setters A, `ov_SC01_005` 657 unregistered (S75 said 557 at 174
|
||||
binaries). Output under `.run/P35/census/` (allowlisted: json/txt/md).
|
||||
- **T1 — DONE.** `tools/share_census.py` (run it plain for the table; `--check` for S1; `--selftest` for the fixture; `--no-cache` after a
|
||||
scanner change — the cache is keyed by file mtime/size, not by the scanner's version). The numbers every later task measures against
|
||||
are in the log entry and `.run/P35/census/share_census.json`. Known shapes the scanner handles (each was a finding): K&R definitions
|
||||
(multi-line and the one-line `void f(a, b) void *a; s16 b; {`), implicit-return-type heads, a second head after `extern …;` on one
|
||||
line, the asm-label alias in both `__asm__(` and `asm(` spellings, `static inline` helpers (no address). The census's per-class output
|
||||
`classes.jsonl` is the input for T3's twin accounting and T5's plan (`share_body.py --plan` reads it).
|
||||
- **T2–T9:** see the task rows and the plan; the Plan agent's consumer table (which tools text-parse the macro form, which ask cpp and
|
||||
need nothing) is reproduced in the plan's §Consumers and §T6.
|
||||
|
||||
|
||||
@@ -0,0 +1,790 @@
|
||||
#!/usr/bin/env python3
|
||||
"""share_census.py — the census of duplicate function bodies across the fleet, and the S1 invariant checker:
|
||||
"ONE SOURCE PER UNIQUE FUNCTION" (Phase 35 T1, 2026-09-08).
|
||||
|
||||
tools/share_census.py # the census: summary + controls to stdout; .run/P35/census/{share_census.json,share_census.txt}
|
||||
tools/share_census.py --check # S1: every same-address byte-identical class with >=2 instances shares ONE source,
|
||||
# is twin-covered, or is ledgered in config/dedup_exceptions.tsv -> exit 1 otherwise
|
||||
tools/share_census.py --check --strict-macros --strict-text # the post-conversion form of S1 (T7 wires it into tools-health)
|
||||
tools/share_census.py --selftest # the R39 fixture: every verdict/flag exercised in memory, no tree needed
|
||||
tools/share_census.py --scope all # count the deferred cross-address / cross-space classes as violations too
|
||||
|
||||
WHAT IT MEASURES (keyed by h_exact = SHA1 of the raw instruction bytes, tools/sig_image.py; relocated immediates INCLUDED, so two
|
||||
instances with one h_exact reference identical addresses and their derived names coincide). A CLASS is one h_exact with >=2 instances
|
||||
fleet-wide (218 binaries). Every instance gets exactly ONE source form from the binary's own translation units:
|
||||
macro DEFINE_func_X() (the Phase-15 shared-body form; retired at T4)
|
||||
macro-param SETTER(func_X,..)/RETCONST/CLEAR_TBL40 (the three legacy name-parameterized headers)
|
||||
include #include "../shared/<..>.h" that DEFINES the function (the Phase-35 form; sotn's shape)
|
||||
param-include #define SHARED_FN func_X / #include ../shared/<..>.h / #undef (the cross-address form; clearTbl40 after T4)
|
||||
def an inline C definition (a private copy)
|
||||
stub INCLUDE_ASM(...) (none left in game code; kept so the census can never mistake one for a copy)
|
||||
asm-verbatim the five PERMANENT hand-asm bodies of config/verbatim_manifest.json (the authoritative list, never a regex)
|
||||
and a sig instance with ZERO or >=2 forms is a COVERAGE DEFECT that fails the run (R32) — a silent skip is exactly the defect class
|
||||
this project has paid for most often.
|
||||
|
||||
VERDICT per class A registered (config/dedup.us.yaml) and every instance is a member whose site is a shared form
|
||||
B registered but an instance is missing from the group, or a member's site is still a private copy
|
||||
C unregistered; every copy is a private definition with the SAME normalized text (name-blind, comment-free)
|
||||
D unregistered; the copies' texts DIFFER (matched independently) — one text must be chosen and re-gated
|
||||
M unregistered, MIXED forms (a shared-form site without a group — e.g. a macro instantiated in one binary)
|
||||
FLAGS (orthogonal) E cross-address (the member's own name differs per site -> the name-parameterized form; DEFERRED by decision)
|
||||
F cross-space (instances in different address spaces: the overlay slot, a module slot base, main, resident)
|
||||
TWIN-COVERED every instance's source dir is one directory (a twin binary built from its primary, T3)
|
||||
TWIN-PENDING every instance lives in binaries of ONE identical-payload twin set (T3 will cover it)
|
||||
TYPEDEF / PINS / ALIAS the copy carries a local type definition / register pins / an asm-label alias
|
||||
S1 (--check): violations = classes with verdict != A that are neither TWIN-COVERED nor (E or F, unless --scope all) nor ledgered in
|
||||
config/dedup_exceptions.tsv (h_exact reason nins instances note). --strict-macros adds "no DEFINE_func_ token under src/";
|
||||
--strict-text adds the SECOND, sig-blind oracle (R34): no name-blind normalized definition text appears in >1 translation unit
|
||||
outside src/shared/. Both are informational until T7.
|
||||
|
||||
ORACLES (derived, never hand-listed — R33): the fleet and each binary's source dir from the Makefile (`compile_only.src_dirs`, asserted
|
||||
equal to the set of config/check.*.sha contracts, R32); address spaces from `<alias>_VRAM_BASE`; twin sets from equal contracts;
|
||||
names from each binary's own symbol stack (`corpus.symbols`) else func_<ADDR>; groups through the ONE registry parser
|
||||
`dedup_integrate.group_members`. Outputs: the small summary JSON + the human table are tracked evidence; classes.jsonl (every class)
|
||||
and the per-TU scan cache stay ignored scratch. R41: every number in the summary names its denominator.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
|
||||
REPO = pathlib.Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO / "tools"))
|
||||
|
||||
SHARED_FORMS = {"macro", "macro-param", "include", "param-include"}
|
||||
KEYWORDS = {"if", "while", "for", "switch", "return", "else", "goto", "case", "sizeof", "do", "typedef",
|
||||
"struct", "union", "enum", "defined", "break", "continue", "default"}
|
||||
FUNC_RE = re.compile(r"func_([0-9A-Fa-f]{8})$")
|
||||
OUT_DIR_DEFAULT = ".run/P35/census"
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# text masking: comments, dead preprocessor halves, macro-continuation blocks -> spaces (newlines kept, so line numbers hold)
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
_TOK = re.compile(r'/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.S)
|
||||
|
||||
|
||||
def _blank_keep_newlines(s):
|
||||
return re.sub(r"[^\n]", " ", s)
|
||||
|
||||
|
||||
def mask_text(text):
|
||||
"""Comments become spaces (string/char literals are KEPT — include paths and asm labels live in them; the brace walk skips them);
|
||||
dead `#if 0` blocks and the `#ifdef NON_MATCHING` half are blanked; `#define ... \\` continuation blocks are blanked (a
|
||||
definition inside a macro body is macro text, not a definition)."""
|
||||
def repl(m):
|
||||
s = m.group(0)
|
||||
if s.startswith("/*") or s.startswith("//"):
|
||||
return _blank_keep_newlines(s)
|
||||
return s
|
||||
t = _TOK.sub(repl, text)
|
||||
lines = t.split("\n")
|
||||
out, i, n = [], 0, len(lines)
|
||||
depth_stack = [] # entries: 'dead' (blank until the matching #endif) / 'live' (an #if we do not evaluate)
|
||||
in_macro = False
|
||||
for ln in lines:
|
||||
s = ln.strip()
|
||||
if in_macro:
|
||||
out.append("")
|
||||
in_macro = ln.rstrip().endswith("\\")
|
||||
continue
|
||||
if s.startswith("#"):
|
||||
d = s[1:].strip()
|
||||
if re.match(r"if\s+0\b", d) or d.startswith("ifdef NON_MATCHING"):
|
||||
depth_stack.append("dead")
|
||||
out.append("")
|
||||
continue
|
||||
if d.startswith(("if", "ifdef", "ifndef")):
|
||||
depth_stack.append("live")
|
||||
out.append("")
|
||||
continue
|
||||
if d.startswith("else") or d.startswith("elif"):
|
||||
if depth_stack and depth_stack[-1] == "dead":
|
||||
depth_stack[-1] = "live-else" # the #else half of a dead block is live
|
||||
elif depth_stack and depth_stack[-1] == "live-else":
|
||||
depth_stack[-1] = "dead"
|
||||
out.append("")
|
||||
continue
|
||||
if d.startswith("endif"):
|
||||
if depth_stack:
|
||||
depth_stack.pop()
|
||||
out.append("")
|
||||
continue
|
||||
if d.startswith("define") and ln.rstrip().endswith("\\"):
|
||||
in_macro = True
|
||||
out.append("")
|
||||
continue
|
||||
if any(x == "dead" for x in depth_stack):
|
||||
out.append("")
|
||||
continue
|
||||
out.append(ln)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# the per-TU scanner
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# a head may follow a line start OR a ';' on the same line (a one-line `extern …; int f(…) { … }` is a real shape in the tree)
|
||||
HEAD = re.compile(r"(?:^|(?<=;))[ \t]*([A-Za-z_][\w \t\*]*?)\b([A-Za-z_]\w*)[ \t]*\(", re.M)
|
||||
# a name at line start with NO type prefix (implicit int / the type on the line above); the body-extent check decides
|
||||
BARE_HEAD = re.compile(r"^[ \t]*([A-Za-z_]\w*)[ \t]*\(", re.M)
|
||||
MACRO_SITE = re.compile(r"^[ \t]*DEFINE_(func_[0-9A-Fa-f]{8})\(\)", re.M)
|
||||
MACRO_PARAM_SITE = re.compile(r"^[ \t]*(SETTER|RETCONST|CLEAR_TBL40)\(\s*([A-Za-z_]\w*)", re.M)
|
||||
INCLUDE_LINE = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"([^"]+)"', re.M)
|
||||
STUB = re.compile(r"INCLUDE_ASM\([^)]*,\s*([A-Za-z_]\w*)\s*\)")
|
||||
# the parameter list may not contain parentheses (a function-pointer parameter would surface as an unaccounted instance, loudly)
|
||||
ALIAS_DECL = re.compile(r"\b([A-Za-z_]\w*)\s*\([^;{}()]*\)\s*(?:__asm__|asm)\s*\(\s*\"(func_[0-9A-Fa-f]{8})\"\s*\)")
|
||||
SHARED_FN_DEF = re.compile(r"^[ \t]*#[ \t]*define[ \t]+SHARED_FN[ \t]+([A-Za-z_]\w*)", re.M)
|
||||
SHARED_FN_UNDEF = re.compile(r"^[ \t]*#[ \t]*undef[ \t]+SHARED_FN\b", re.M)
|
||||
PIN_RE = re.compile(r'register [^;]*__asm__\("\$?[a-z0-9]+"\)')
|
||||
TYPEDEF_RE = re.compile(r"\b(typedef\b|(struct|union|enum)\s+\w*\s*\{)")
|
||||
_BRACE = re.compile(r'"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'|[{}]') # literals are matched and ignored
|
||||
|
||||
|
||||
def _body_extent(masked, head_end):
|
||||
"""From the '(' at head_end-1: walk the parameter parens, then decide prototype / definition; return (kind, open_idx, close_idx).
|
||||
kind: 'proto' or 'def'. For a def, open_idx/close_idx index the body's braces in `masked`."""
|
||||
i, depth, n = head_end - 1, 0, len(masked)
|
||||
while i < n:
|
||||
c = masked[i]
|
||||
if c == "(":
|
||||
depth += 1
|
||||
elif c == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
i += 1
|
||||
if i >= n:
|
||||
return ("proto", None, None)
|
||||
j = i + 1
|
||||
# after the parameter list: whitespace, then ';' (prototype), '{' (definition) or K&R declarations / an asm label
|
||||
while j < n and masked[j] in " \t\r\n":
|
||||
j += 1
|
||||
if j >= n:
|
||||
return ("proto", None, None)
|
||||
if masked[j] == ";":
|
||||
return ("proto", None, None)
|
||||
if masked[j] != "{":
|
||||
# Something stands between the parameter list and the next '{'. It is a DEFINITION only if that something is K&R
|
||||
# parameter declarations (`type name;` parts, nothing after the last ';'); an asm label, a directive, a call statement,
|
||||
# or the head of the NEXT function all make this a declaration/statement, not a body.
|
||||
seg_end_brace = masked.find("{", j)
|
||||
if seg_end_brace == -1:
|
||||
return ("proto", None, None)
|
||||
seg = masked[j:seg_end_brace]
|
||||
if "__asm__" in seg or "asm(" in seg or "#" in seg:
|
||||
return ("proto", None, None)
|
||||
parts = seg.split(";")
|
||||
if len(parts) < 2 or parts[-1].strip() or not all(re.fullmatch(r"\s*[A-Za-z_][\w \t\*\[\],]*\s*", p) for p in parts[:-1]):
|
||||
return ("proto", None, None)
|
||||
j = seg_end_brace
|
||||
# brace walk from j
|
||||
depth = 0
|
||||
for m in _BRACE.finditer(masked, j):
|
||||
tok = m.group(0)
|
||||
if tok == "{":
|
||||
depth += 1
|
||||
elif tok == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return ("def", j, m.start())
|
||||
return ("proto", None, None)
|
||||
|
||||
|
||||
def _norm(body, names):
|
||||
t = body
|
||||
for nm in sorted(set(names), key=len, reverse=True):
|
||||
t = re.sub(r"\b%s\b" % re.escape(nm), "@FN@", t)
|
||||
t = re.sub(r"\s+", " ", t).strip()
|
||||
return hashlib.sha1(t.encode("utf-8", "surrogateescape")).hexdigest()
|
||||
|
||||
|
||||
def scan_text(text, rel, shared_defs=None):
|
||||
"""All site records of one file. Records: dict(form, name, line, ...). shared_defs: header path -> defined names (for includes)."""
|
||||
masked = mask_text(text)
|
||||
recs = []
|
||||
line_of = lambda idx: masked.count("\n", 0, idx) + 1
|
||||
aliases = {a: s for a, s in ALIAS_DECL.findall(masked)} # alias ident -> real symbol
|
||||
for m in MACRO_SITE.finditer(masked):
|
||||
recs.append(dict(form="macro", name=m.group(1), line=line_of(m.start())))
|
||||
for m in MACRO_PARAM_SITE.finditer(masked):
|
||||
recs.append(dict(form="macro-param", name=m.group(2), line=line_of(m.start()), via=m.group(1)))
|
||||
for m in STUB.finditer(masked):
|
||||
recs.append(dict(form="stub", name=m.group(1), line=line_of(m.start())))
|
||||
# includes (resolved by the caller through shared_defs), with the SHARED_FN state
|
||||
if shared_defs is not None:
|
||||
events = [(m.start(), "def", m.group(1)) for m in SHARED_FN_DEF.finditer(masked)]
|
||||
events += [(m.start(), "undef", None) for m in SHARED_FN_UNDEF.finditer(masked)]
|
||||
events += [(m.start(), "inc", m.group(1)) for m in INCLUDE_LINE.finditer(masked)]
|
||||
cur = None
|
||||
for pos, kind, val in sorted(events):
|
||||
if kind == "def":
|
||||
cur = val
|
||||
elif kind == "undef":
|
||||
cur = None
|
||||
else:
|
||||
key = os.path.normpath(os.path.join(os.path.dirname(rel), val))
|
||||
defs = shared_defs.get(key)
|
||||
if not defs:
|
||||
continue
|
||||
for nm in defs:
|
||||
if nm == "SHARED_FN":
|
||||
if cur:
|
||||
recs.append(dict(form="param-include", name=cur, line=line_of(pos), header=key))
|
||||
else:
|
||||
recs.append(dict(form="include", name=nm, line=line_of(pos), header=key))
|
||||
# definitions
|
||||
seen_heads = set()
|
||||
heads = [(m.start(), m.end(), m.group(2), m.group(1)) for m in HEAD.finditer(masked)]
|
||||
heads += [(m.start(), m.end(), m.group(1), "") for m in BARE_HEAD.finditer(masked)]
|
||||
for start, end, name, prefix in heads:
|
||||
if name in KEYWORDS or end in seen_heads:
|
||||
continue
|
||||
seen_heads.add(end)
|
||||
if "inline" in prefix.split():
|
||||
continue # a `static inline` helper is expanded into its callers: it owns no address and is no sig instance
|
||||
kind, o, c = _body_extent(masked, end)
|
||||
if kind != "def":
|
||||
continue
|
||||
body = masked[start:c + 1]
|
||||
real = aliases.get(name, name)
|
||||
recs.append(dict(form="def", name=real, line=line_of(start), end=line_of(c), alias=(real != name),
|
||||
text_hash=_norm(body, [name, real]), pins=bool(PIN_RE.search(body)),
|
||||
typedef=bool(TYPEDEF_RE.search(body)), nlines=body.count("\n") + 1))
|
||||
return recs
|
||||
|
||||
|
||||
def header_defined_names(path):
|
||||
"""The function names a shared header DEFINES (masked: macro bodies never count)."""
|
||||
text = path.read_text(errors="surrogateescape")
|
||||
recs = scan_text(text, path.relative_to(REPO).as_posix(), shared_defs=None)
|
||||
return sorted({r["name"] for r in recs if r["form"] == "def"})
|
||||
|
||||
|
||||
def _scan_worker(args):
|
||||
rel, mtime, size, shared_defs = args
|
||||
p = REPO / rel
|
||||
text = p.read_text(errors="surrogateescape")
|
||||
return rel, scan_text(text, rel, shared_defs)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# oracles
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def contracts():
|
||||
"""alias -> its config/check.<x>.sha path, from the Makefile's `<alias>_CHECK_SHA :=` (main's contract is check.us.sha)."""
|
||||
out = {}
|
||||
texts = [(REPO / "Makefile").read_text()] + [p.read_text() for p in sorted((REPO / "config").glob("*.mk"))]
|
||||
for t in texts:
|
||||
for m in re.finditer(r"^(\w+)_CHECK_SHA\s*:?=\s*(\S+)", t, re.M):
|
||||
out[m.group(1)] = m.group(2)
|
||||
return out
|
||||
|
||||
|
||||
def fleet_and_dirs():
|
||||
import compile_only
|
||||
dirs = compile_only.src_dirs()
|
||||
con = contracts()
|
||||
on_disk = {p.relative_to(REPO).as_posix() for p in (REPO / "config").glob("check.*.sha")}
|
||||
if set(dirs) != set(con) or set(con.values()) != on_disk:
|
||||
sys.exit("share_census: the Makefile's binaries, its _CHECK_SHA contracts and config/check.*.sha DISAGREE (R32): "
|
||||
f"dirs-only {sorted(set(dirs) - set(con))[:8]}, contracts-only {sorted(set(con) - set(dirs))[:8]}, "
|
||||
f"files-not-declared {sorted(on_disk - set(con.values()))[:8]}, declared-not-on-disk {sorted(set(con.values()) - on_disk)[:8]}")
|
||||
return sorted(dirs), dirs
|
||||
|
||||
|
||||
def vram_bases():
|
||||
out = {}
|
||||
texts = [(REPO / "Makefile").read_text()] + [p.read_text() for p in sorted((REPO / "config").glob("*.mk"))]
|
||||
for t in texts:
|
||||
for m in re.finditer(r"^(\w+)_VRAM_BASE\s*:?=\s*(0x[0-9A-Fa-f]+)", t, re.M):
|
||||
out[m.group(1)] = int(m.group(2), 16)
|
||||
return out
|
||||
|
||||
|
||||
def twin_sets(aliases):
|
||||
by_sha = collections.defaultdict(list)
|
||||
con = contracts()
|
||||
for a in aliases:
|
||||
sha = (REPO / con[a]).read_text().split()[0]
|
||||
by_sha[sha].append(a)
|
||||
sets = {}
|
||||
for sha, members in by_sha.items():
|
||||
if len(members) > 1:
|
||||
for a in members:
|
||||
sets[a] = tuple(sorted(members))
|
||||
return sets
|
||||
|
||||
|
||||
def twin_of_map():
|
||||
out = {}
|
||||
for p in sorted((REPO / "config").glob("*.mk")):
|
||||
for m in re.finditer(r"^(\w+)_TWIN_OF\s*:?=\s*(\w+)", p.read_text(), re.M):
|
||||
out[m.group(1)] = m.group(2)
|
||||
return out
|
||||
|
||||
|
||||
def sig_rows(alias):
|
||||
p = REPO / ".run" / f"sig.{alias}.jsonl"
|
||||
if not p.exists():
|
||||
sys.exit(f"share_census: {p.relative_to(REPO)} is missing — run `make tools-health` (or the sig-* targets) first; the census "
|
||||
f"refuses to measure a fleet it cannot see (R43)")
|
||||
rows = []
|
||||
for ln in p.read_text().splitlines():
|
||||
if not ln.strip():
|
||||
continue
|
||||
r = json.loads(ln)
|
||||
rows.append((int(r["addr"], 16), int(r.get("nins", 0)), r["h_exact"], r.get("name")))
|
||||
return rows
|
||||
|
||||
|
||||
def curated_names(alias):
|
||||
import corpus
|
||||
try:
|
||||
syms = corpus.symbols(alias)
|
||||
except Exception:
|
||||
syms = {}
|
||||
return {addr: nm for nm, addr in syms.items()}
|
||||
|
||||
|
||||
def load_groups():
|
||||
import yaml
|
||||
d = yaml.safe_load((REPO / "config" / "dedup.us.yaml").read_text())
|
||||
return d["groups"] if isinstance(d, dict) else d
|
||||
|
||||
|
||||
def verbatim_instances():
|
||||
d = json.loads((REPO / "config" / "verbatim_manifest.json").read_text())
|
||||
return {(r["binary"], int(r["addr"], 16)) for r in d["rows"]}
|
||||
|
||||
|
||||
def load_exceptions(path):
|
||||
out = {}
|
||||
p = REPO / path
|
||||
if not p.exists():
|
||||
return out
|
||||
for ln in p.read_text().splitlines():
|
||||
if not ln.strip() or ln.startswith("#") or ln.startswith("h_exact\t"):
|
||||
continue
|
||||
parts = ln.split("\t")
|
||||
out[parts[0]] = dict(reason=parts[1] if len(parts) > 1 else "?", note=parts[-1])
|
||||
return out
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# the census
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def build_forms(aliases, dirs, jobs, use_cache=True, out_dir=None):
|
||||
"""(alias -> {addr: rec}), coverage notes. One scan per TU, cached by (mtime, size)."""
|
||||
import compile_only
|
||||
# shared headers: which function names each defines (masked, so macro-only headers define nothing)
|
||||
shared_defs = {}
|
||||
for p in sorted((REPO / "src" / "shared").rglob("*.h")):
|
||||
shared_defs[p.relative_to(REPO).as_posix()] = header_defined_names(p)
|
||||
cache_p = (REPO / (out_dir or OUT_DIR_DEFAULT) / "cache" / "scan_cache.json")
|
||||
cache = {}
|
||||
if use_cache and cache_p.exists():
|
||||
try:
|
||||
cache = json.loads(cache_p.read_text())
|
||||
except Exception:
|
||||
cache = {}
|
||||
work, tu_of = [], {}
|
||||
for a in aliases:
|
||||
for p in compile_only.tus_of(a, dirs):
|
||||
rel = p.relative_to(REPO).as_posix()
|
||||
st = p.stat()
|
||||
tu_of.setdefault(a, []).append(rel)
|
||||
key = f"{rel}|{int(st.st_mtime)}|{st.st_size}"
|
||||
if key not in cache:
|
||||
work.append((rel, int(st.st_mtime), st.st_size, shared_defs))
|
||||
scanned = {}
|
||||
if work:
|
||||
with ProcessPoolExecutor(max_workers=jobs) as ex:
|
||||
for rel, recs in ex.map(_scan_worker, work, chunksize=8):
|
||||
scanned[rel] = recs
|
||||
new_cache = {}
|
||||
forms, notes = {}, dict(unresolved_names=[], multi_form=[], stubs_unsigned=0)
|
||||
for a in aliases:
|
||||
names = curated_names(a)
|
||||
rev = {nm: addr for addr, nm in names.items()}
|
||||
per = {}
|
||||
for rel in tu_of.get(a, []):
|
||||
st = (REPO / rel).stat()
|
||||
key = f"{rel}|{int(st.st_mtime)}|{st.st_size}"
|
||||
recs = scanned.get(rel) if rel in scanned else cache.get(key)
|
||||
if recs is None:
|
||||
recs = scan_text((REPO / rel).read_text(errors="surrogateescape"), rel, shared_defs)
|
||||
new_cache[key] = recs
|
||||
for r in recs:
|
||||
nm = r["name"]
|
||||
m = FUNC_RE.match(nm)
|
||||
addr = int(m.group(1), 16) if m else rev.get(nm)
|
||||
if addr is None:
|
||||
if r["form"] != "stub":
|
||||
notes["unresolved_names"].append(f"{a}:{rel}:{r['line']}:{nm}:{r['form']}")
|
||||
continue
|
||||
rec = dict(r, tu=rel)
|
||||
if addr in per and per[addr]["form"] != "stub" and rec["form"] != "stub":
|
||||
notes["multi_form"].append(f"{a}:0x{addr:08X}:{per[addr]['form']}@{per[addr]['tu']}:{per[addr]['line']} vs "
|
||||
f"{rec['form']}@{rel}:{rec['line']}")
|
||||
continue
|
||||
if addr in per and rec["form"] == "stub":
|
||||
continue
|
||||
per[addr] = rec
|
||||
forms[a] = per
|
||||
if use_cache:
|
||||
cache_p.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_p.write_text(json.dumps(new_cache))
|
||||
return forms, notes
|
||||
|
||||
|
||||
def classify(sigs, forms, groups, spaces, dirs, twins, twin_of, verbatim, exceptions, scope="same-vram"):
|
||||
"""Pure: sigs {alias: [(addr,nins,h,name)]}, forms {alias: {addr: rec}}, groups (registry list), spaces {alias: base},
|
||||
dirs {alias: src dir}, twins {alias: twin-set tuple}, twin_of {alias: primary}, verbatim {(alias,addr)}, exceptions {h: ...}."""
|
||||
from dedup_integrate import group_members
|
||||
# registry index
|
||||
reg = collections.defaultdict(list)
|
||||
for g in groups:
|
||||
reg[g.get("hash")].append(g)
|
||||
# instances
|
||||
classes = collections.defaultdict(list)
|
||||
unaccounted = []
|
||||
n_inst = 0
|
||||
for a, rows in sigs.items():
|
||||
per = forms.get(a, {})
|
||||
for addr, nins, h, name in rows:
|
||||
n_inst += 1
|
||||
rec = per.get(addr)
|
||||
if rec is None and (a, addr) in verbatim:
|
||||
rec = dict(form="asm-verbatim", name=name, tu=None, line=None)
|
||||
if rec is None:
|
||||
unaccounted.append(f"{a}:0x{addr:08X}:{name}:nins={nins}")
|
||||
continue
|
||||
classes[h].append(dict(alias=a, addr=addr, nins=nins, name=name, form=rec["form"], tu=rec.get("tu"),
|
||||
text_hash=rec.get("text_hash"), pins=rec.get("pins", False), typedef=rec.get("typedef", False),
|
||||
alias_form=rec.get("alias", False)))
|
||||
out = []
|
||||
for h, insts in classes.items():
|
||||
if len(insts) < 2:
|
||||
continue
|
||||
gs = reg.get(h, [])
|
||||
members = set()
|
||||
for g in gs:
|
||||
for b, addr, _ in group_members(g):
|
||||
members.add((b, addr))
|
||||
inst_set = {(i["alias"], i["addr"]) for i in insts}
|
||||
forms_seen = collections.Counter(i["form"] for i in insts)
|
||||
addrs = sorted({i["addr"] for i in insts})
|
||||
aliases = sorted({i["alias"] for i in insts})
|
||||
space_set = {spaces.get(i["alias"], -1) for i in insts}
|
||||
flags = []
|
||||
if len(addrs) > 1:
|
||||
flags.append("E")
|
||||
if len(space_set) > 1:
|
||||
flags.append("F")
|
||||
src_dirs = {dirs.get(i["alias"]) for i in insts}
|
||||
if len(aliases) > 1 and len(src_dirs) == 1:
|
||||
flags.append("TWIN-COVERED")
|
||||
elif len(aliases) > 1 and all(a in twins for a in aliases) and len({twins[a] for a in aliases}) == 1:
|
||||
flags.append("TWIN-PENDING")
|
||||
if any(i["typedef"] for i in insts):
|
||||
flags.append("TYPEDEF")
|
||||
if any(i["pins"] for i in insts):
|
||||
flags.append("PINS")
|
||||
if any(i["alias_form"] for i in insts):
|
||||
flags.append("ALIAS")
|
||||
if any(i["form"] == "stub" for i in insts):
|
||||
flags.append("STUB")
|
||||
if gs:
|
||||
missing = sorted(inst_set - members)
|
||||
extra = sorted(members - inst_set)
|
||||
bad_site = [i for i in insts if (i["alias"], i["addr"]) in members and i["form"] not in SHARED_FORMS]
|
||||
verdict = "A" if not missing and not bad_site and not extra else "B"
|
||||
detail = dict(groups=[g.get("id") for g in gs], missing=len(missing), extra=len(extra), bad_site=len(bad_site))
|
||||
else:
|
||||
defs = [i for i in insts if i["form"] == "def"]
|
||||
if len(defs) == len(insts):
|
||||
texts = {i["text_hash"] for i in insts}
|
||||
verdict = "C" if len(texts) == 1 else "D"
|
||||
detail = dict(texts=len(texts))
|
||||
else:
|
||||
verdict = "M"
|
||||
detail = dict(forms=dict(forms_seen))
|
||||
nins = max(i["nins"] for i in insts)
|
||||
band = "1-7" if nins < 8 else "8-15" if nins < 16 else "16-31" if nins < 32 else "32+"
|
||||
excepted = exceptions.get(h)
|
||||
deferred = ("E" in flags or "F" in flags) and scope != "all"
|
||||
violation = (verdict != "A") and ("TWIN-COVERED" not in flags) and not deferred and not excepted
|
||||
out.append(dict(h=h, verdict=verdict, flags=flags, nins=nins, band=band, instances=len(insts), copies=forms_seen.get("def", 0),
|
||||
aliases=aliases, addrs=[f"0x{x:08X}" for x in addrs], forms=dict(forms_seen), detail=detail,
|
||||
excepted=(excepted or {}).get("reason"), deferred=deferred, violation=violation,
|
||||
names=sorted({i["name"] for i in insts})[:4]))
|
||||
out.sort(key=lambda c: (-c["instances"] * c["nins"], c["h"]))
|
||||
return out, dict(instances=n_inst, classified=n_inst - len(unaccounted), unaccounted=unaccounted)
|
||||
|
||||
|
||||
def summarize(classes, cov, notes, aliases):
|
||||
by_v = collections.Counter(c["verdict"] for c in classes)
|
||||
by_f = collections.Counter(f for c in classes for f in c["flags"])
|
||||
inst_by_v = collections.Counter()
|
||||
ins_by_v = collections.Counter()
|
||||
for c in classes:
|
||||
inst_by_v[c["verdict"]] += c["instances"]
|
||||
ins_by_v[c["verdict"]] += c["instances"] * c["nins"]
|
||||
same_vram_unreg = [c for c in classes if c["verdict"] in ("B", "C", "D", "M") and "E" not in c["flags"] and "F" not in c["flags"]]
|
||||
cross = [c for c in classes if ("E" in c["flags"] or "F" in c["flags"]) and c["verdict"] != "A"]
|
||||
return dict(
|
||||
binaries=len(aliases), sig_instances=cov["instances"], classified=cov["classified"], unaccounted=len(cov["unaccounted"]),
|
||||
classes_ge2=len(classes), verdicts=dict(by_v), instances_by_verdict=dict(inst_by_v), ins_by_verdict=dict(ins_by_v),
|
||||
flags=dict(by_f),
|
||||
same_vram_unregistered=dict(classes=len(same_vram_unreg), instances=sum(c["instances"] for c in same_vram_unreg),
|
||||
copies=sum(c["copies"] for c in same_vram_unreg),
|
||||
collapsible=sum(c["instances"] - 1 for c in same_vram_unreg),
|
||||
ins=sum(c["instances"] * c["nins"] for c in same_vram_unreg),
|
||||
twin_pending=sum(1 for c in same_vram_unreg if "TWIN-PENDING" in c["flags"]),
|
||||
twin_pending_instances=sum(c["instances"] for c in same_vram_unreg if "TWIN-PENDING" in c["flags"])),
|
||||
cross_address_deferred=dict(classes=len(cross), instances=sum(c["instances"] for c in cross),
|
||||
copies=sum(c["copies"] for c in cross)),
|
||||
violations=sum(1 for c in classes if c["violation"]),
|
||||
multi_form=len(notes.get("multi_form", [])), unresolved_names=len(notes.get("unresolved_names", [])),
|
||||
)
|
||||
|
||||
|
||||
def controls(classes, sigs, forms):
|
||||
"""The known-true cases (R39). Each returns (name, got, expected)."""
|
||||
out = []
|
||||
# func_80144B9C: the -O0 whale — 141 include sites, registered for 134 (never extended to the 7 later-onboarded overlays)
|
||||
w = [c for c in classes if "0x80144B9C" in c["addrs"] and "ov_SC01_077" in c["aliases"]]
|
||||
out.append(("func_80144B9C", f"{w[0]['verdict']}/{w[0]['instances']}/{w[0]['forms']}/missing {w[0]['detail'].get('missing')}" if w else "absent",
|
||||
"B/141/{'include': 141}/missing 7 (the registry lists 134; T5 bucket 0 extends it)"))
|
||||
# clearTbl40: main 0x80037004 + 0x80037334, one class, registered, cross-vram
|
||||
ct = [c for c in classes if "0x80037004" in c["addrs"] and "main" in c["aliases"]]
|
||||
out.append(("clearTbl40", f"{ct[0]['verdict']}+{'E' if 'E' in ct[0]['flags'] else '-'}/{ct[0]['instances']}" if ct else "absent", "A+E/2"))
|
||||
# ov_setters: the three SC01_005/006 groups — registered for 2 of the 141 instances each (the same trivial body sits at that
|
||||
# vram in every overlay); B by construction until T5 extends them
|
||||
# Measured S94: the 2-instruction bodies of two of them also sit at OTHER vrams (282 and 145 instances, cross-address), the
|
||||
# third only at its own (141) — so the known-true answer is three B classes of 282(E) / 145(E) / 141 instances.
|
||||
st = sorted([c for c in classes if any(x in c["addrs"] for x in ("0x8012AD64", "0x8012BF4C", "0x8012E27C")) and "ov_SC01_005" in c["aliases"]],
|
||||
key=lambda c: -c["instances"])
|
||||
out.append(("ov_setters x3", ",".join(f"{c['verdict']}/{c['instances']}{'(E)' if 'E' in c['flags'] else ''}" for c in st) or "absent",
|
||||
"B/282(E),B/145(E),B/141"))
|
||||
# twin symmetry: ov_SC01_005 and ov_SC01_006 are one payload, so their unregistered-class counts must be EQUAL (a known-true
|
||||
# invariant rather than a remembered number: S75 counted 557 at 174 binaries, the S94 Explore 657 at 218, this census's rule differs)
|
||||
def unreg(alias):
|
||||
return sum(1 for c in classes if c["verdict"] in ("C", "D", "M") and alias in c["aliases"])
|
||||
a5, a6 = unreg("ov_SC01_005"), unreg("ov_SC01_006")
|
||||
out.append(("twin symmetry SC01_005 / SC01_006", f"{a5} / {a6}", "equal (each = the twin's unregistered classes)"))
|
||||
return out
|
||||
|
||||
|
||||
def render_table(classes, summary, cov_notes, ctrl):
|
||||
L = []
|
||||
s = summary
|
||||
L.append(f"share_census: {s['binaries']} binaries · {s['sig_instances']:,} sig instances · {s['classified']:,} classified · "
|
||||
f"{s['unaccounted']} UNACCOUNTED · {s['classes_ge2']:,} h_exact classes with >=2 instances")
|
||||
L.append(f" verdicts (classes): " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['verdicts'].items())))
|
||||
L.append(f" instances by verdict: " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['instances_by_verdict'].items())))
|
||||
L.append(f" flags: " + " · ".join(f"{k} {v:,}" for k, v in sorted(s['flags'].items())))
|
||||
u = s["same_vram_unregistered"]
|
||||
L.append(f" SAME-VRAM UNREGISTERED (the phase's backlog): {u['classes']:,} classes · {u['instances']:,} instances · {u['copies']:,} private "
|
||||
f"copies · {u['collapsible']:,} collapsible · {u['ins']:,} ins; of which twin-pending {u['twin_pending']:,} classes / "
|
||||
f"{u['twin_pending_instances']:,} instances")
|
||||
d = s["cross_address_deferred"]
|
||||
L.append(f" CROSS-ADDRESS / CROSS-SPACE (deferred to the names phase): {d['classes']:,} classes · {d['instances']:,} instances · "
|
||||
f"{d['copies']:,} private copies")
|
||||
L.append(f" S1 violations (scope as run): {s['violations']:,} · multi-form instances {s['multi_form']} · unresolved names {s['unresolved_names']}")
|
||||
L.append(" controls (R39):")
|
||||
for name, got, exp in ctrl:
|
||||
L.append(f" {name:34s} got {got:40s} expected {exp}")
|
||||
L.append(" top classes by instances x nins (verdict flags instances nins addrs names):")
|
||||
for c in classes[:25]:
|
||||
L.append(f" {c['verdict']} {','.join(c['flags']) or '-':22s} {c['instances']:4d} x {c['nins']:5d} {','.join(c['addrs'][:3])} {','.join(c['names'][:2])}")
|
||||
if cov_notes.get("unaccounted"):
|
||||
L.append(" UNACCOUNTED (first 20): " + "; ".join(cov_notes["unaccounted"][:20]))
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# the second oracle: duplicate definition TEXT across translation units (sig-blind)
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def text_duplicates(forms):
|
||||
by_text = collections.defaultdict(set)
|
||||
for a, per in forms.items():
|
||||
for addr, rec in per.items():
|
||||
if rec["form"] == "def" and rec.get("text_hash") and rec.get("nlines", 0) >= 2:
|
||||
by_text[rec["text_hash"]].add((a, rec["tu"], addr))
|
||||
return {h: sorted(v) for h, v in by_text.items() if len({(a, tu) for a, tu, _ in v}) > 1}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
# self-test fixture (R39): every verdict and flag, in memory
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def selftest():
|
||||
A, B, C = "fx_A", "fx_B", "fx_C"
|
||||
spaces = {A: 0x80128158, B: 0x80128158, C: 0x800CAE08}
|
||||
dirs = {A: "src/fx_A", B: "src/fx_B", C: "src/fx_C"}
|
||||
twins = {}
|
||||
sigs = {A: [], B: [], C: []}
|
||||
forms = {A: {}, B: {}, C: {}}
|
||||
groups = []
|
||||
|
||||
def put(alias, addr, h, form, text="t", nins=12, **kw):
|
||||
sigs[alias].append((addr, nins, h, f"func_{addr:08X}"))
|
||||
forms[alias][addr] = dict(form=form, name=f"func_{addr:08X}", tu=f"{dirs[alias]}/x.c", line=1, text_hash=text, **kw)
|
||||
|
||||
# 1 A: registered, both macro sites
|
||||
put(A, 0x80130000, "h1", "macro"); put(B, 0x80130000, "h1", "macro")
|
||||
groups.append(dict(id="g1", hash="h1", vram=0x80130000, binaries=[A, B]))
|
||||
# 2 B: registered but B's instance is missing from the group
|
||||
put(A, 0x80130010, "h2", "macro"); put(B, 0x80130010, "h2", "def")
|
||||
groups.append(dict(id="g2", hash="h2", vram=0x80130010, binaries=[A]))
|
||||
# 3 C: unregistered, identical text
|
||||
put(A, 0x80130020, "h3", "def", text="same"); put(B, 0x80130020, "h3", "def", text="same")
|
||||
# 4 D: unregistered, differing text
|
||||
put(A, 0x80130030, "h4", "def", text="x"); put(B, 0x80130030, "h4", "def", text="y", pins=True)
|
||||
# 5 C+E: cross-vram identical
|
||||
put(A, 0x80130040, "h5", "def", text="same"); put(B, 0x80130050, "h5", "def", text="same")
|
||||
# 6 C+F: cross-space
|
||||
put(A, 0x80130060, "h6", "def", text="same"); put(C, 0x800CB000, "h6", "def", text="same")
|
||||
# 7 M: unregistered mixed (a macro site in one binary, a private copy in the other)
|
||||
put(A, 0x80130070, "h7", "macro"); put(B, 0x80130070, "h7", "def", typedef=True)
|
||||
# a singleton (must not be a class) and an unaccounted instance (must be reported)
|
||||
put(A, 0x80130080, "h8", "def")
|
||||
sigs[B].append((0x80130090, 4, "h9", "func_80130090"))
|
||||
classes, cov = classify(sigs, forms, groups, spaces, dirs, twins, {}, set(), {}, scope="same-vram")
|
||||
got = {c["h"]: (c["verdict"], tuple(c["flags"])) for c in classes}
|
||||
exp = {"h1": ("A", ()), "h2": ("B", ()), "h3": ("C", ()), "h4": ("D", ("PINS",)), "h5": ("C", ("E",)), "h6": ("C", ("E", "F")),
|
||||
"h7": ("M", ("TYPEDEF",))}
|
||||
ok = 0
|
||||
for h, e in exp.items():
|
||||
g = got.get(h)
|
||||
if g == e:
|
||||
ok += 1
|
||||
else:
|
||||
print(f"selftest: {h}: got {g} expected {e}")
|
||||
if "h8" in got:
|
||||
print("selftest: a singleton became a class"); ok -= 1
|
||||
if cov["unaccounted"] != [f"{B}:0x80130090:func_80130090:nins=4"]:
|
||||
print(f"selftest: unaccounted reporting wrong: {cov['unaccounted']}"); ok -= 1
|
||||
viol = sorted(c["h"] for c in classes if c["violation"])
|
||||
if viol != ["h2", "h3", "h4", "h7"]:
|
||||
print(f"selftest: violations wrong: {viol} (E/F deferred by default)"); ok -= 1
|
||||
classes_all, _ = classify(sigs, forms, groups, spaces, dirs, twins, {}, set(), {"h3": dict(reason="TEST")}, scope="all")
|
||||
viol_all = sorted(c["h"] for c in classes_all if c["violation"])
|
||||
if viol_all != ["h2", "h4", "h5", "h6", "h7"]:
|
||||
print(f"selftest: --scope all / exceptions wrong: {viol_all}"); ok -= 1
|
||||
# the masker + scanner on a synthetic TU
|
||||
tu = ('#include "common.h"\n#include "../shared/engine_core.h"\n\nDEFINE_func_80128158() /* dedup: shared */\n'
|
||||
'/* void func_DEAD0001(void) { } */\n#if 0\nvoid func_DEAD0002(void) { }\n#endif\n'
|
||||
'extern s32 D_1;\ns32 func_80128218(void) {\n return D_1; /* } */\n}\n'
|
||||
'int aF80128300(int a) __asm__("func_80128300");\nint aF80128300(int a) {\n register int r __asm__("$16");\n return a;\n}\n'
|
||||
'INCLUDE_ASM("asm/x", func_80128400);\nSETTER(func_8012AD64, 0x34, s16)\n'
|
||||
'#define SHARED_FN func_80037334\n#include "../shared/main/func_80037004__a0744d60.h"\n#undef SHARED_FN\n'
|
||||
'void\nfunc_80128500(void)\n{\n}\n')
|
||||
sd = {"src/shared/main/func_80037004__a0744d60.h": ["SHARED_FN"], "src/shared/engine_core.h": []}
|
||||
recs = scan_text(tu, "src/fx_A/x.c", shared_defs=sd)
|
||||
want = {("macro", "func_80128158"), ("def", "func_80128218"), ("def", "func_80128300"), ("stub", "func_80128400"),
|
||||
("macro-param", "func_8012AD64"), ("param-include", "func_80037334"), ("def", "func_80128500")}
|
||||
gotset = {(r["form"], r["name"]) for r in recs}
|
||||
if gotset != want:
|
||||
print(f"selftest: scanner: got {sorted(gotset)}\n want {sorted(want)}"); ok -= 1
|
||||
else:
|
||||
pins = [r for r in recs if r["form"] == "def" and r["name"] == "func_80128300"][0]
|
||||
if not (pins["pins"] and pins["alias"]):
|
||||
print("selftest: the alias-form def lost its PINS/ALIAS flags"); ok -= 1
|
||||
total = len(exp)
|
||||
print(f"selftest: {ok}/{total} verdicts correct" + ("" if ok == total else " — FAIL"))
|
||||
return ok == total
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--check", action="store_true", help="the S1 invariant; exit 1 on any violation")
|
||||
ap.add_argument("--strict-macros", action="store_true", help="with --check: any DEFINE_func_ token under src/ is a violation")
|
||||
ap.add_argument("--strict-text", action="store_true", help="with --check: any definition text duplicated across TUs is a violation")
|
||||
ap.add_argument("--scope", choices=["same-vram", "all"], default="same-vram")
|
||||
ap.add_argument("--exceptions", default="config/dedup_exceptions.tsv")
|
||||
ap.add_argument("--out-dir", default=OUT_DIR_DEFAULT)
|
||||
ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument("--no-cache", action="store_true")
|
||||
ap.add_argument("--selftest", action="store_true")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
a = ap.parse_args()
|
||||
if a.selftest:
|
||||
sys.exit(0 if selftest() else 1)
|
||||
t0 = time.time()
|
||||
aliases, dirs = fleet_and_dirs()
|
||||
spaces = vram_bases()
|
||||
missing_space = [x for x in aliases if x not in spaces]
|
||||
if missing_space:
|
||||
sys.exit(f"share_census: no _VRAM_BASE for {missing_space[:5]} (R32)")
|
||||
twins = twin_sets(aliases)
|
||||
twin_of = twin_of_map()
|
||||
sigs = {x: sig_rows(x) for x in aliases}
|
||||
forms, notes = build_forms(aliases, dirs, a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir)
|
||||
groups = load_groups()
|
||||
exceptions = load_exceptions(a.exceptions)
|
||||
verb = verbatim_instances()
|
||||
classes, cov = classify(sigs, forms, groups, spaces, dirs, twins, twin_of, verb, exceptions, scope=a.scope)
|
||||
summary = summarize(classes, cov, notes, aliases)
|
||||
ctrl = controls(classes, sigs, forms)
|
||||
macro_tokens = 0
|
||||
for x in aliases:
|
||||
for per in [forms[x]]:
|
||||
macro_tokens += sum(1 for r in per.values() if r["form"] == "macro")
|
||||
dup_text = text_duplicates(forms)
|
||||
summary["macro_sites"] = macro_tokens
|
||||
summary["duplicate_text_classes"] = len(dup_text)
|
||||
summary["duplicate_text_sites"] = sum(len(v) for v in dup_text.values())
|
||||
summary["twin_sets"] = sorted({v for v in twins.values()})
|
||||
summary["elapsed_s"] = round(time.time() - t0, 1)
|
||||
summary["controls"] = [dict(name=n, got=g, expected=e) for n, g, e in ctrl]
|
||||
out = REPO / a.out_dir
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / "share_census.json").write_text(json.dumps(summary, indent=1, sort_keys=True) + "\n")
|
||||
with open(out / "classes.jsonl", "w") as f:
|
||||
for c in classes:
|
||||
f.write(json.dumps(c) + "\n")
|
||||
(out / "coverage_notes.txt").write_text("\n".join(
|
||||
["# unaccounted sig instances (alias:addr:name:nins)"] + cov["unaccounted"] +
|
||||
["# instances with >=2 non-stub forms"] + notes["multi_form"] +
|
||||
["# unresolved (non-func_, not in the binary's symbol stack) definition/site names"] + notes["unresolved_names"]) + "\n")
|
||||
table = render_table(classes, summary, cov, ctrl)
|
||||
table += f"\n macro sites {macro_tokens:,} · duplicate-text classes {len(dup_text):,} ({summary['duplicate_text_sites']:,} sites) · elapsed {summary['elapsed_s']} s"
|
||||
(out / "share_census.txt").write_text(table + "\n")
|
||||
if not a.quiet:
|
||||
print(table)
|
||||
rc = 0
|
||||
if cov["unaccounted"]:
|
||||
print(f"share_census: COVERAGE DEFECT — {len(cov['unaccounted'])} sig instance(s) have no source form (R32); see {a.out_dir}/coverage_notes.txt")
|
||||
rc = 1
|
||||
if notes["multi_form"]:
|
||||
print(f"share_census: COVERAGE DEFECT — {len(notes['multi_form'])} instance(s) carry two non-stub forms; see coverage_notes.txt")
|
||||
rc = 1
|
||||
if a.check:
|
||||
v = [c for c in classes if c["violation"]]
|
||||
bad = len(v)
|
||||
if a.strict_macros and macro_tokens:
|
||||
print(f"share_census: S1 — {macro_tokens} DEFINE_func_ site(s) remain under src/ (--strict-macros)"); bad += 1
|
||||
if a.strict_text and dup_text:
|
||||
print(f"share_census: S1 — {len(dup_text)} definition text(s) duplicated across TUs (--strict-text)"); bad += 1
|
||||
exc = sum(1 for c in classes if c["excepted"])
|
||||
twinc = sum(1 for c in classes if "TWIN-COVERED" in c["flags"] and c["verdict"] != "A")
|
||||
print(f"S1: one source per unique function — {len(classes):,} classes, {len(classes) - len(v):,} satisfied "
|
||||
f"({twinc:,} twin-covered, {exc:,} excepted, {summary['cross_address_deferred']['classes']:,} deferred cross-address), "
|
||||
f"{len(v):,} VIOLATION(S)" + (" — OK" if bad == 0 else " — FAIL"))
|
||||
for c in v[:15]:
|
||||
print(f" {c['verdict']} {','.join(c['flags']) or '-'} h={c['h'][:10]} {c['instances']} inst x {c['nins']} ins {c['addrs'][:2]} {c['aliases'][:3]}")
|
||||
if bad:
|
||||
rc = 1
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user