From 61ff5364ac88c2609230b07cb3eeda8d7283c74c Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:42:01 -0600 Subject: [PATCH] T10.c2: type_census --check-casts --residue gate, residue census, ops doc, card row --- HOW_WE_WORK.md | 1 + .../corpus/tools/P10/type_census.py | 115 +++++++++++++++++- docs/ops/INDEX.md | 1 + .../p39-t10-2026-10-02-check-casts-residue.md | 7 ++ phase-ends/current/logs/T10.c2.md | 109 +++++++++++++++++ tools/type_census.py | 115 +++++++++++++++++- 6 files changed, 336 insertions(+), 12 deletions(-) create mode 100644 docs/ops/p39-t10-2026-10-02-check-casts-residue.md create mode 100644 phase-ends/current/logs/T10.c2.md diff --git a/HOW_WE_WORK.md b/HOW_WE_WORK.md index 6ea0fd5d49..88207238dd 100644 --- a/HOW_WE_WORK.md +++ b/HOW_WE_WORK.md @@ -37,6 +37,7 @@ PY = /usr/bin/python3 | phaseend_index | `PY tools/phaseend_index.py` | assemble, lint and archive a PhaseEnd | | genend_index | `PY tools/genend_index.py` | assemble and lint a GenerationEnd | | commit_task | `bash tools/commit_task.sh` | the only commit path; explicit paths, no trailers, never pushes; auto-stages phase-ends/current/{tasks,logs,research,discussions,RECAP.md,TASK_PROGRESS.md}, but leaves reply-drafting discussion records unstaged and refuses messages mentioning a drafted/posted reply; citing an issue for a fix is fine (R120) | +| type_census casts | `PY tools/type_census.py --check-casts [--residue]` | cast-and-lying gate; `--residue`: raw P/I/X/M pass only if ledgered RESIDUAL() per site; writes residue_census | | run | `bash tools/run.sh` | any command that may print >40 lines; `--bg` / `--wait` for long compute | | Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not | `Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not` | Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not | diff --git a/decomp-architect/corpus/tools/P10/type_census.py b/decomp-architect/corpus/tools/P10/type_census.py index bc086fe2d9..75748dfcb3 100644 --- a/decomp-architect/corpus/tools/P10/type_census.py +++ b/decomp-architect/corpus/tools/P10/type_census.py @@ -1919,6 +1919,23 @@ def selftest(): rz, lz = check_casts_verdict(z, [], {}, [], set()) checks.append(("check-casts all-zero OK + line", rz == 0 and lz == "check-casts: raw P=0 I=0 X=0 M=0; macros none (unbacked 0); lying=0 (ledgered 0)")) checks.append(("check-casts raw FAIL", check_casts_verdict(dict(z, X=1), [], {}, [], set())[0] == 1)) + # P39 T10: --residue — per form one covered site passes; uncovered three ways (no row; row without the site; RESIDUAL with no cause) + for f, pos in (("P", "*(u16 *)(a0 + 4) = 1;"), ("I", "*(u8 *)D_80078EB4 = 1;"), ("X", "((s16 *)a0)[2] = 0;"), ("M", "M2C_FIELD(a0, u8 *, 1) = 0;")): + w, (raw1, _) = cc(pos) + s1 = [s for s in w["sites"] if s["form"] == f] + e = dict(form=f, line=2, base=f"{s1[0]['bclass']}:{s1[0]['base']}", off=s1[0]["off"], verdict="RESIDUAL(GLOBAL)", cause="GLOBAL") + key = ("S", "src/ov_TEST/k.c", "g") + for name, lat, n_unc in (("covered", {key: dict(verdict="RESIDUAL", sites=[e])}, 0), ("no row", {}, 1), + ("site absent", {key: dict(verdict="RESIDUAL", sites=[dict(e, off=(e["off"] or 0) + 0x40)])}, 1), + ("empty cause", {key: dict(verdict="RESIDUAL", sites=[dict(e, cause="")])}, 1)): + cov, unc, cause = residue_split(w["sites"], lat) + rc_r, l_r = check_casts_verdict(raw1, [], lat, [], set(), uncovered=collections.Counter(s["form"] for s in unc)) + want = "(uncovered " + " ".join(f"{g}={int(g == f and n_unc)}" for g in ("P", "I", "X", "M")) + ")" + checks.append((f"check-casts --residue {f} {name}", len(s1) == 1 and len(unc) == n_unc and len(cov) == 1 - n_unc + and rc_r == n_unc and want in l_r and (n_unc or cause[id(cov[0])] == "GLOBAL"))) + wk, (_, mk) = cc("CAST_ALIAS(u16, a0, 4) = 2;") + covk, unck, _ = residue_split(wk["sites"], {}) + checks.append(("check-casts --residue KEPT macro under macros", len(mk) == 1 and covk == [] and unck == [])) bad = [n for n, ok in checks if not ok] print(f"type_census --selftest: {len(checks) - len(bad)}/{len(checks)} checks OK" + (f"; FAILED: {bad}" if bad else "")) return 0 if not bad else 1 @@ -1979,8 +1996,9 @@ def check_casts_inputs(): return latest, exc -def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions): - """(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV.""" +def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions, uncovered=None): + """(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV. + P39 T10 (--residue): `uncovered` {form: n} — the line adds `(uncovered P=…)` and the rc tests uncovered instead of raw.""" unbacked = 0 for s in macro_sites: row = latest.get(("S", s["tu"], s["fn"])) @@ -2002,9 +2020,80 @@ def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions macros = collections.Counter(s["name"] for s in macro_sites) mtxt = " ".join(f"{n}={macros[n]}" for n in KEPT_MACROS if macros[n] > 0) or "none" raw = [raw_by_form.get(f, 0) for f in ("P", "I", "X", "M")] - line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}; macros {mtxt} (unbacked {unbacked}); " + unc = None if uncovered is None else [uncovered.get(f, 0) for f in ("P", "I", "X", "M")] + utxt = "" if unc is None else f" (uncovered P={unc[0]} I={unc[1]} X={unc[2]} M={unc[3]})" + line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}{utxt}; macros {mtxt} (unbacked {unbacked}); " f"lying={lying} (ledgered {lying - unledgered})") - return (1 if any(raw) or unbacked or unledgered else 0), line + return (1 if any(raw if unc is None else unc) or unbacked or unledgered else 0), line + + +# --check-casts --residue (P39 T10): a raw P/I/X/M site passes only when a rung-S ledger entry RESIDUAL() covers it +# (restruct.residue_match — per site, never re-implemented here). +RESIDUE_SEGMENTS = (("ov", ["ov_*"]), ("md", ["md_*"]), ("main", ["main"])) # else "shared" (src/shared/, include/, other dirs) + + +def residue_split(sites, latest): + """(covered, uncovered, {id(covered site): ledger cause}) over the raw P/I/X/M census sites (not common.h's #define lines). The cause + is read back from the covering body's RESIDUAL entries in ledger order (annotation only; coverage is residue_match's).""" + import restruct as rs + sites = [s for s in sites if s.get("form") in ("P", "I", "X", "M") and not s.get("defline")] + covered, uncovered = rs.residue_match(latest, sites, "P,I,X,M") + pools, cause = {}, {} + for s in covered: + bk = (s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT) + if bk not in pools: + pools[bk] = collections.defaultdict(list) + for e in (latest.get(("S",) + bk) or {}).get("sites") or []: + if str(e.get("verdict") or "").startswith("RESIDUAL(") and e.get("cause"): + pools[bk][rs._residue_key(e.get("form"), e.get("base"), e.get("off"))].append(e["cause"]) + q = pools[bk][rs._residue_key(s["form"], f"{s.get('bclass')}:{s.get('base')}", s.get("off"))] + cause[id(s)] = q.pop(0) if q else "?" + return covered, uncovered, cause + + +def residue_census(covered, uncovered, cause, out_dir): + """Write /residue_census.{json,txt}: covered/uncovered by form, by cause, by segment; the top 50 map types by covered sites + (sites, bodies); how many map types cover 50/80/95% of the TYPE-NOT-CANONICAL sites. Returns the by-cause Counter.""" + import restruct as rs + forms = ("P", "I", "X", "M") + seg_of = lambda s: next((n for n, only in RESIDUE_SEGMENTS if rs.matches_only(only, s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)), "shared") + by_form = {f: dict(covered=0, uncovered=0) for f in forms} + by_seg = {n: dict(covered=0, uncovered=0) for n in [n for n, _ in RESIDUE_SEGMENTS] + ["shared"]} + for k, ss in (("covered", covered), ("uncovered", uncovered)): + for s in ss: + by_form[s["form"]][k] += 1 + by_seg[seg_of(s)][k] += 1 + by_cause = collections.Counter(cause[id(s)] for s in covered) + tsites, tbodies = collections.Counter(), collections.defaultdict(set) + tnc = collections.Counter() + for s in covered: + if s.get("type"): + tsites[s["type"]] += 1 + tbodies[s["type"]].add((s["tu"], s.get("fn"))) + if cause[id(s)] == "TYPE-NOT-CANONICAL": + tnc[s["type"]] += 1 + top = [dict(type=t, sites=n, bodies=len(tbodies[t])) for t, n in sorted(tsites.items(), key=lambda kv: (-kv[1], kv[0]))[:50]] + tnc_total, cover, acc, i = sum(tnc.values()), {}, 0, 0 + ranked = sorted(tnc.values(), reverse=True) + for pct in (50, 80, 95): + while i < len(ranked) and acc * 100 < pct * tnc_total: + acc += ranked[i] + i += 1 + cover[str(pct)] = i + doc = dict(by_form=by_form, by_cause=dict(sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))), by_segment=by_seg, + top_types=top, tnc=dict(sites=tnc_total, types=len(tnc), types_to_cover_pct=cover)) + out = REPO / out_dir + out.mkdir(parents=True, exist_ok=True) + (out / "residue_census.json").write_text(json.dumps(doc, indent=1)) + L = ["residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL() ledger entry", "", "by form (covered / uncovered):"] + L += [f" {f} {by_form[f]['covered']:>7} / {by_form[f]['uncovered']}" for f in forms] + L += ["", "by segment (covered / uncovered):"] + [f" {n:<7}{v['covered']:>7} / {v['uncovered']}" for n, v in by_seg.items()] + L += ["", "by cause (covered sites):"] + [f" {c:<22}{n:>7}" for c, n in doc["by_cause"].items()] + L += ["", "top 50 map types by covered sites (sites bodies type):"] + [f" {r['sites']:>6} {r['bodies']:>6} {r['type']}" for r in top] + L += ["", f"TYPE-NOT-CANONICAL: {tnc_total} sites over {len(tnc)} distinct map types; types to cover 50%={cover['50']} " + f"80%={cover['80']} 95%={cover['95']}"] + (out / "residue_census.txt").write_text("\n".join(L) + "\n") + return by_cause # ---------------------------------------------------------------------------------------------------------------------- def main(): @@ -2016,12 +2105,16 @@ def main(): ap.add_argument("--check", action="store_true", help="T8's gate (the invariants; exit 1 on a violation)") ap.add_argument("--check-structs", action="store_true", help="the struct-unification gate (P38 T2; exit 1 on a violation)") ap.add_argument("--check-casts", action="store_true", help="the cast-and-lying gate (P39 T2; exit 1 on a raw cast, an unbacked macro site or an unledgered lie)") + ap.add_argument("--residue", action="store_true", help="with --check-casts (P39 T10): a raw P/I/X/M site passes when a rung-S ledger " + "entry RESIDUAL() covers it (restruct.residue_match); prints the uncovered counts and a by-cause line, exits 1 on any " + "uncovered site; writes /residue_census.{json,txt} (by form, cause, segment; top 50 map types; TNC 50/80/95%%)") ap.add_argument("--selftest", action="store_true") ap.add_argument("--quiet", action="store_true") a = ap.parse_args() if a.selftest: sys.exit(selftest()) - summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites) + want_residue = a.check_casts and a.residue + summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites or want_residue) if a.quiet: print(txt.strip().split("\n")[-2]) print(txt.strip().split("\n")[-1]) @@ -2058,9 +2151,19 @@ def main(): rc = rc or rc_s if a.check_casts: latest, exc = check_casts_inputs() + unc, bc_line = None, None + if want_residue: + with open(REPO / a.out_dir / "sites.jsonl") as fh: + sites = [json.loads(l) for l in fh if l.startswith(('{"form": "P"', '{"form": "I"', '{"form": "X"', '{"form": "M"'))] + covered, uncovered, cause = residue_split(sites, latest) + unc = collections.Counter(s["form"] for s in uncovered) + by_cause = residue_census(covered, uncovered, cause, a.out_dir) + bc_line = "by-cause: " + (" ".join(f"{c}={n}" for c, n in sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))) or "none") rc_c, line = check_casts_verdict(summary["casts"]["check_raw"], summary["casts"]["macro_sites"], latest, - summary["decls"]["lying_rows"], exc) + summary["decls"]["lying_rows"], exc, uncovered=unc) print(line) + if bc_line: + print(bc_line) rc = rc or rc_c sys.exit(rc) diff --git a/docs/ops/INDEX.md b/docs/ops/INDEX.md index 602a884782..abd3935650 100644 --- a/docs/ops/INDEX.md +++ b/docs/ops/INDEX.md @@ -55,3 +55,4 @@ p38-t4-2026-09-29-restruct-head-fold-mode.md | §P38 T4 — 2026-09-29: restruct p39-t1-2026-10-01-restruct-stale-supersede-timeline-check.md | §P39 T1 — 2026-10-01: restruct stale-row supersede; timeline --check | 5 p39-t3-2026-10-02-restruct-rung-s-residue-head-types-add-pads.md | §P39 T3 — 2026-10-02: restruct rung S residue/check-residue; head_types --add-pads | 6 p39-t4-2026-10-02-restruct-only-main.md | §P39 T4 — 2026-10-02: restruct --only main | 5 +p39-t10-2026-10-02-check-casts-residue.md | §P39 T10 — 2026-10-02: type_census --check-casts --residue; restruct --forms per-site residue | 5 diff --git a/docs/ops/p39-t10-2026-10-02-check-casts-residue.md b/docs/ops/p39-t10-2026-10-02-check-casts-residue.md new file mode 100644 index 0000000000..3639920d33 --- /dev/null +++ b/docs/ops/p39-t10-2026-10-02-check-casts-residue.md @@ -0,0 +1,7 @@ +# §P39 T10 — 2026-10-02: type_census --check-casts --residue; restruct --forms / per-site --check-residue + +- `type_census.py --check-casts --residue`: a raw P/I/X/M census site (not include/common.h's `#define` lines) passes only when its body's latest rung-S ledger row (`ledger_latest`, R100) has a `RESIDUAL()` entry with a non-empty cause, same form, normalised base and off (`restruct.residue_match`; one entry covers one site; never by line). Line: `check-casts: raw P=… I=… X=… M=… (uncovered P=… I=… X=… M=…); macros … (unbacked n); lying=n (ledgered n)` + `by-cause: =n …` (covered sites, desc). Exit 1 on any uncovered, unbacked macro site or unledgered lie. Forces `sites.jsonl` (as `--sites`). +- Writes `/residue_census.{json,txt}` (default `.run/P37/census/`): covered/uncovered by form, by segment (ov `ov_*` / md `md_*` / main / else shared, `restruct.matches_only`), by cause; top 50 map types by covered sites (sites, bodies); TYPE-NOT-CANONICAL sites, distinct types, and types needed to cover 50/80/95%. +- Without `--residue`, `--check-casts` is unchanged (T2 line; exit 1 on any raw > 0). KEPT macro sites (form K) are counted under macros, never residue. +- `restruct.py --rung S --residue --forms P,I,X,M --only --label `: ledger-only rows for uncovered sites of the forms (default `P,X`); a body with a latest S row gets a merged copy (MEMBER/KEPT entries kept) plus one `RESIDUAL()` per uncovered site. `--check-residue --forms …` counts uncovered PER SITE (not per body). +- I/M causes: ABS-ADDR (base bclass abs/gaddr: fixed or symbol address), GLOBAL (bclass global); else the P/X causes (FILE-SCOPE, NO-TYPE-IN-MAP, TYPE-NOT-CANONICAL, TYPE-NOT-VISIBLE, site_class). diff --git a/phase-ends/current/logs/T10.c2.md b/phase-ends/current/logs/T10.c2.md new file mode 100644 index 0000000000..220b3cb552 --- /dev/null +++ b/phase-ends/current/logs/T10.c2.md @@ -0,0 +1,109 @@ +# T10.c2 — type_census --check-casts --residue, residue census, docs + +## Changed +- tools/type_census.py: `check_casts_verdict(..., uncovered=None)` :1999 — with `uncovered` the line adds `(uncovered P=… I=… X=… M=…)` and rc tests uncovered instead of raw; without it, byte-identical T2 line/rc. +- `RESIDUE_SEGMENTS` :2032; `residue_split(sites, latest)` :2035 → (covered, uncovered, cause) over raw P/I/X/M non-defline sites via `restruct.residue_match(latest, sites, "P,I,X,M")`; cause read back from the covering body's RESIDUAL entries (multiset, ledger order) via `rs._residue_key` — annotation only. +- `residue_census(covered, uncovered, cause, out_dir)` :2054 → `/residue_census.{json,txt}`: by form, segment (ov `ov_*` / md `md_*` / main via `rs.matches_only`, else shared), cause; top 50 map types (s["type"]) by covered sites with distinct (tu, fn) bodies; TNC sites/types and types to cover 50/80/95%. +- `--residue` arg :2108; forces `want_sites` (reads `/sites.jsonl`); prints gate line then `by-cause:` line :2155. +- selftest :1922: per form P/I/X/M: covered passes (rc 0, cause GLOBAL read back); uncovered rc 1 via no row / row with site absent (off +0x40) / RESIDUAL with empty cause; KEPT macro site (CAST_ALIAS) in macro_sites, not in covered/uncovered. 67/67. +- docs/ops/p39-t10-2026-10-02-check-casts-residue.md + docs/ops/INDEX.md line; HOW_WE_WORK.md Tools row `type_census casts` (card check: chars=6354 cap=7000). + +## Verified +- `/usr/bin/python3 tools/type_census.py --selftest` → 67/67 checks OK +- `/usr/bin/python3 tools/restruct.py --selftest` → 101/101 OK +- `type_census.py --check-casts` (no --residue) → T2 shape, RC 1: + `check-casts: raw P=257464 I=51505 X=13790 M=15130; macros CAST_ALIAS=1173 CAST_SIGN=8429 CAST_WIDTH=10410 CAST_MISALIGNED=624 CAST_NONSTRUCT=327 (unbacked 0); lying=229 (ledgered 0)` +- `type_census.py --check-casts --residue` → RC 1: + `check-casts: raw P=257464 I=51505 X=13790 M=15130 (uncovered P=0 I=35625 X=0 M=14762); macros CAST_ALIAS=1173 CAST_SIGN=8429 CAST_WIDTH=10410 CAST_MISALIGNED=624 CAST_NONSTRUCT=327 (unbacked 0); lying=229 (ledgered 0)` + `by-cause: TYPE-NOT-CANONICAL=281252 NO-TYPE-IN-MAP=2966 NO-FIELD=1773 NO-EDIT=482 INDEX=479 AGGREGATE-FIELD=369 FILE-SCOPE=172 TYPE-NOT-VISIBLE=9` +- Reconcile: by-cause sum 287502 = P+X raw 271254 + I covered 15880 + M covered 368; no `?` cause (every covered site's cause read back). +- `--help` documents `--residue`. +- `make kit-corpus && make tools-health` → RC=0, "tools-health: OK"; corpus copy decomp-architect/corpus/tools/P10/type_census.py updated and committed. + +## residue_census.txt (verbatim, .run/P37/census/residue_census.txt; gitignored, not committed) +``` +residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL() ledger entry + +by form (covered / uncovered): + P 257464 / 0 + I 15880 / 35625 + X 13790 / 0 + M 368 / 14762 + +by segment (covered / uncovered): + ov 267984 / 48334 + md 6315 / 328 + main 1751 / 321 + shared 11452 / 1404 + +by cause (covered sites): + TYPE-NOT-CANONICAL 281252 + NO-TYPE-IN-MAP 2966 + NO-FIELD 1773 + NO-EDIT 482 + INDEX 479 + AGGREGATE-FIELD 369 + FILE-SCOPE 172 + TYPE-NOT-VISIBLE 9 + +top 50 map types by covered sites (sites bodies type): + 21350 6239 Unkstruct_800D3888 + 4340 124 Unkstruct_0caecd54 + 4216 124 Unkstruct_dac960ce + 4061 4061 Unkstruct_f8ed2b9f + 3936 123 Unkstruct_a32b99a1 + 3930 131 Unkstruct_8e3dbe51 + 3271 196 Unkstruct_57a28935 + 3144 131 Unkstruct_1fa9e810 + 3100 124 Unkstruct_30b21035 + 2646 126 Unkstruct_a2c8734a + 2580 129 Unkstruct_198fc7cc + 2356 124 Unkstruct_ed8f1335 + 2213 705 Unkstruct_80126948 + 2141 535 Unkstruct_8016B4F8 + 2111 265 Unkstruct_80139C7C + 2109 317 Unkstruct_80130D48 + 2091 123 Unkstruct_2729e670 + 2016 126 Unkstruct_dd542e54 + 2000 125 Unkstruct_f49880fd + 1948 1948 Unkstruct_a8d34bce + 1926 131 Unkstruct_8016DF5C + 1875 125 Unkstruct_568f7bda + 1764 126 Unkstruct_cd1bacd2 + 1742 270 Unkstruct_8011F7A8 + 1722 123 Unkstruct_84612d18 + 1664 128 Unkstruct_57db4d07 + 1612 124 Unkstruct_4692fc17 + 1596 133 Unkstruct_eda88e53 + 1572 131 Unkstruct_4e09b6d9 + 1565 180 Unkstruct_8012D3B4 + 1548 258 Unkstruct_645b745d + 1441 131 Unkstruct_0980b9f0 + 1441 131 Unkstruct_b5c0372a + 1404 117 Unkstruct_47470ce3 + 1364 124 Unkstruct_a68f6d03 + 1336 134 Unkstruct_801345F8 + 1300 130 Unkstruct_8fdd92d1 + 1250 125 Unkstruct_110cf29e + 1240 124 Unkstruct_4db8cec8 + 1240 124 Unkstruct_d292c30d + 1240 124 Unkstruct_f0bc3b7a + 1220 122 Unkstruct_a459e3e5 + 1179 131 Unkstruct_f9a7989e + 1170 130 Unkstruct_80135260 + 1149 259 Unkstruct_801290DC + 1128 376 Unkstruct_0b30043d + 1116 124 Unkstruct_301381d4 + 1116 124 Unkstruct_4e427c01 + 1107 123 Unkstruct_4857195e + 1040 130 Unkstruct_50a3f6b4 + +TYPE-NOT-CANONICAL: 281252 sites over 13081 distinct map types; types to cover 50%=81 80%=579 95%=4384 +``` + +## Notes for the expert +- P/X uncovered 0: no bodies to report. I/M uncovered (35625 / 14762) await `restruct --residue --forms I,M` rows (T10.c1 tool). +- Covered I/M sites carry no ABS-ADDR/GLOBAL cause yet: the 15880 I + 368 M covered come from existing rows' entries (causes in the by-cause line above). +- Segment split uses matches_only order ov → md → main, else shared (src/shared/, include/, other dirs). +- residue_census.{json,txt} live under gitignored `.run/`; not force-added (brief: "written"). +- harness: a census run (both gate runs) rewrites tracked docs/struct-map.md and .run/P37/census/{type_census.*,struct_map_top.json}; those were already dirty at start and are not committed here. diff --git a/tools/type_census.py b/tools/type_census.py index bc086fe2d9..75748dfcb3 100644 --- a/tools/type_census.py +++ b/tools/type_census.py @@ -1919,6 +1919,23 @@ def selftest(): rz, lz = check_casts_verdict(z, [], {}, [], set()) checks.append(("check-casts all-zero OK + line", rz == 0 and lz == "check-casts: raw P=0 I=0 X=0 M=0; macros none (unbacked 0); lying=0 (ledgered 0)")) checks.append(("check-casts raw FAIL", check_casts_verdict(dict(z, X=1), [], {}, [], set())[0] == 1)) + # P39 T10: --residue — per form one covered site passes; uncovered three ways (no row; row without the site; RESIDUAL with no cause) + for f, pos in (("P", "*(u16 *)(a0 + 4) = 1;"), ("I", "*(u8 *)D_80078EB4 = 1;"), ("X", "((s16 *)a0)[2] = 0;"), ("M", "M2C_FIELD(a0, u8 *, 1) = 0;")): + w, (raw1, _) = cc(pos) + s1 = [s for s in w["sites"] if s["form"] == f] + e = dict(form=f, line=2, base=f"{s1[0]['bclass']}:{s1[0]['base']}", off=s1[0]["off"], verdict="RESIDUAL(GLOBAL)", cause="GLOBAL") + key = ("S", "src/ov_TEST/k.c", "g") + for name, lat, n_unc in (("covered", {key: dict(verdict="RESIDUAL", sites=[e])}, 0), ("no row", {}, 1), + ("site absent", {key: dict(verdict="RESIDUAL", sites=[dict(e, off=(e["off"] or 0) + 0x40)])}, 1), + ("empty cause", {key: dict(verdict="RESIDUAL", sites=[dict(e, cause="")])}, 1)): + cov, unc, cause = residue_split(w["sites"], lat) + rc_r, l_r = check_casts_verdict(raw1, [], lat, [], set(), uncovered=collections.Counter(s["form"] for s in unc)) + want = "(uncovered " + " ".join(f"{g}={int(g == f and n_unc)}" for g in ("P", "I", "X", "M")) + ")" + checks.append((f"check-casts --residue {f} {name}", len(s1) == 1 and len(unc) == n_unc and len(cov) == 1 - n_unc + and rc_r == n_unc and want in l_r and (n_unc or cause[id(cov[0])] == "GLOBAL"))) + wk, (_, mk) = cc("CAST_ALIAS(u16, a0, 4) = 2;") + covk, unck, _ = residue_split(wk["sites"], {}) + checks.append(("check-casts --residue KEPT macro under macros", len(mk) == 1 and covk == [] and unck == [])) bad = [n for n, ok in checks if not ok] print(f"type_census --selftest: {len(checks) - len(bad)}/{len(checks)} checks OK" + (f"; FAILED: {bad}" if bad else "")) return 0 if not bad else 1 @@ -1979,8 +1996,9 @@ def check_casts_inputs(): return latest, exc -def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions): - """(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV.""" +def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions, uncovered=None): + """(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV. + P39 T10 (--residue): `uncovered` {form: n} — the line adds `(uncovered P=…)` and the rc tests uncovered instead of raw.""" unbacked = 0 for s in macro_sites: row = latest.get(("S", s["tu"], s["fn"])) @@ -2002,9 +2020,80 @@ def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions macros = collections.Counter(s["name"] for s in macro_sites) mtxt = " ".join(f"{n}={macros[n]}" for n in KEPT_MACROS if macros[n] > 0) or "none" raw = [raw_by_form.get(f, 0) for f in ("P", "I", "X", "M")] - line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}; macros {mtxt} (unbacked {unbacked}); " + unc = None if uncovered is None else [uncovered.get(f, 0) for f in ("P", "I", "X", "M")] + utxt = "" if unc is None else f" (uncovered P={unc[0]} I={unc[1]} X={unc[2]} M={unc[3]})" + line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}{utxt}; macros {mtxt} (unbacked {unbacked}); " f"lying={lying} (ledgered {lying - unledgered})") - return (1 if any(raw) or unbacked or unledgered else 0), line + return (1 if any(raw if unc is None else unc) or unbacked or unledgered else 0), line + + +# --check-casts --residue (P39 T10): a raw P/I/X/M site passes only when a rung-S ledger entry RESIDUAL() covers it +# (restruct.residue_match — per site, never re-implemented here). +RESIDUE_SEGMENTS = (("ov", ["ov_*"]), ("md", ["md_*"]), ("main", ["main"])) # else "shared" (src/shared/, include/, other dirs) + + +def residue_split(sites, latest): + """(covered, uncovered, {id(covered site): ledger cause}) over the raw P/I/X/M census sites (not common.h's #define lines). The cause + is read back from the covering body's RESIDUAL entries in ledger order (annotation only; coverage is residue_match's).""" + import restruct as rs + sites = [s for s in sites if s.get("form") in ("P", "I", "X", "M") and not s.get("defline")] + covered, uncovered = rs.residue_match(latest, sites, "P,I,X,M") + pools, cause = {}, {} + for s in covered: + bk = (s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT) + if bk not in pools: + pools[bk] = collections.defaultdict(list) + for e in (latest.get(("S",) + bk) or {}).get("sites") or []: + if str(e.get("verdict") or "").startswith("RESIDUAL(") and e.get("cause"): + pools[bk][rs._residue_key(e.get("form"), e.get("base"), e.get("off"))].append(e["cause"]) + q = pools[bk][rs._residue_key(s["form"], f"{s.get('bclass')}:{s.get('base')}", s.get("off"))] + cause[id(s)] = q.pop(0) if q else "?" + return covered, uncovered, cause + + +def residue_census(covered, uncovered, cause, out_dir): + """Write /residue_census.{json,txt}: covered/uncovered by form, by cause, by segment; the top 50 map types by covered sites + (sites, bodies); how many map types cover 50/80/95% of the TYPE-NOT-CANONICAL sites. Returns the by-cause Counter.""" + import restruct as rs + forms = ("P", "I", "X", "M") + seg_of = lambda s: next((n for n, only in RESIDUE_SEGMENTS if rs.matches_only(only, s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)), "shared") + by_form = {f: dict(covered=0, uncovered=0) for f in forms} + by_seg = {n: dict(covered=0, uncovered=0) for n in [n for n, _ in RESIDUE_SEGMENTS] + ["shared"]} + for k, ss in (("covered", covered), ("uncovered", uncovered)): + for s in ss: + by_form[s["form"]][k] += 1 + by_seg[seg_of(s)][k] += 1 + by_cause = collections.Counter(cause[id(s)] for s in covered) + tsites, tbodies = collections.Counter(), collections.defaultdict(set) + tnc = collections.Counter() + for s in covered: + if s.get("type"): + tsites[s["type"]] += 1 + tbodies[s["type"]].add((s["tu"], s.get("fn"))) + if cause[id(s)] == "TYPE-NOT-CANONICAL": + tnc[s["type"]] += 1 + top = [dict(type=t, sites=n, bodies=len(tbodies[t])) for t, n in sorted(tsites.items(), key=lambda kv: (-kv[1], kv[0]))[:50]] + tnc_total, cover, acc, i = sum(tnc.values()), {}, 0, 0 + ranked = sorted(tnc.values(), reverse=True) + for pct in (50, 80, 95): + while i < len(ranked) and acc * 100 < pct * tnc_total: + acc += ranked[i] + i += 1 + cover[str(pct)] = i + doc = dict(by_form=by_form, by_cause=dict(sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))), by_segment=by_seg, + top_types=top, tnc=dict(sites=tnc_total, types=len(tnc), types_to_cover_pct=cover)) + out = REPO / out_dir + out.mkdir(parents=True, exist_ok=True) + (out / "residue_census.json").write_text(json.dumps(doc, indent=1)) + L = ["residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL() ledger entry", "", "by form (covered / uncovered):"] + L += [f" {f} {by_form[f]['covered']:>7} / {by_form[f]['uncovered']}" for f in forms] + L += ["", "by segment (covered / uncovered):"] + [f" {n:<7}{v['covered']:>7} / {v['uncovered']}" for n, v in by_seg.items()] + L += ["", "by cause (covered sites):"] + [f" {c:<22}{n:>7}" for c, n in doc["by_cause"].items()] + L += ["", "top 50 map types by covered sites (sites bodies type):"] + [f" {r['sites']:>6} {r['bodies']:>6} {r['type']}" for r in top] + L += ["", f"TYPE-NOT-CANONICAL: {tnc_total} sites over {len(tnc)} distinct map types; types to cover 50%={cover['50']} " + f"80%={cover['80']} 95%={cover['95']}"] + (out / "residue_census.txt").write_text("\n".join(L) + "\n") + return by_cause # ---------------------------------------------------------------------------------------------------------------------- def main(): @@ -2016,12 +2105,16 @@ def main(): ap.add_argument("--check", action="store_true", help="T8's gate (the invariants; exit 1 on a violation)") ap.add_argument("--check-structs", action="store_true", help="the struct-unification gate (P38 T2; exit 1 on a violation)") ap.add_argument("--check-casts", action="store_true", help="the cast-and-lying gate (P39 T2; exit 1 on a raw cast, an unbacked macro site or an unledgered lie)") + ap.add_argument("--residue", action="store_true", help="with --check-casts (P39 T10): a raw P/I/X/M site passes when a rung-S ledger " + "entry RESIDUAL() covers it (restruct.residue_match); prints the uncovered counts and a by-cause line, exits 1 on any " + "uncovered site; writes /residue_census.{json,txt} (by form, cause, segment; top 50 map types; TNC 50/80/95%%)") ap.add_argument("--selftest", action="store_true") ap.add_argument("--quiet", action="store_true") a = ap.parse_args() if a.selftest: sys.exit(selftest()) - summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites) + want_residue = a.check_casts and a.residue + summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites or want_residue) if a.quiet: print(txt.strip().split("\n")[-2]) print(txt.strip().split("\n")[-1]) @@ -2058,9 +2151,19 @@ def main(): rc = rc or rc_s if a.check_casts: latest, exc = check_casts_inputs() + unc, bc_line = None, None + if want_residue: + with open(REPO / a.out_dir / "sites.jsonl") as fh: + sites = [json.loads(l) for l in fh if l.startswith(('{"form": "P"', '{"form": "I"', '{"form": "X"', '{"form": "M"'))] + covered, uncovered, cause = residue_split(sites, latest) + unc = collections.Counter(s["form"] for s in uncovered) + by_cause = residue_census(covered, uncovered, cause, a.out_dir) + bc_line = "by-cause: " + (" ".join(f"{c}={n}" for c, n in sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))) or "none") rc_c, line = check_casts_verdict(summary["casts"]["check_raw"], summary["casts"]["macro_sites"], latest, - summary["decls"]["lying_rows"], exc) + summary["decls"]["lying_rows"], exc, uncovered=unc) print(line) + if bc_line: + print(bc_line) rc = rc or rc_c sys.exit(rc)