mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-04 00:20:10 -04:00
T10.c2: type_census --check-casts --residue gate, residue census, ops doc, card row
This commit is contained in:
Drew T
@@ -37,6 +37,7 @@ PY = /usr/bin/python3
|
||||
| phaseend_index | `PY tools/phaseend_index.py` | assemble, lint and archive a PhaseEnd |
|
||||
| genend_index | `PY tools/genend_index.py` | assemble and lint a GenerationEnd |
|
||||
| commit_task | `bash tools/commit_task.sh` | the only commit path; explicit paths, no trailers, never pushes; auto-stages phase-ends/current/{tasks,logs,research,discussions,RECAP.md,TASK_PROGRESS.md}, but leaves reply-drafting discussion records unstaged and refuses messages mentioning a drafted/posted reply; citing an issue for a fix is fine (R120) |
|
||||
| type_census casts | `PY tools/type_census.py --check-casts [--residue]` | cast-and-lying gate; `--residue`: raw P/I/X/M pass only if ledgered RESIDUAL(<cause>) per site; writes residue_census |
|
||||
| run | `bash tools/run.sh` | any command that may print >40 lines; `--bg` / `--wait` for long compute |
|
||||
| Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not | `Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not` | Every script under `tools/` (plus the two report make-targets), grouped by purpose — one line each. Deep HOW-TO is **not |
|
||||
|
||||
|
||||
@@ -1919,6 +1919,23 @@ def selftest():
|
||||
rz, lz = check_casts_verdict(z, [], {}, [], set())
|
||||
checks.append(("check-casts all-zero OK + line", rz == 0 and lz == "check-casts: raw P=0 I=0 X=0 M=0; macros none (unbacked 0); lying=0 (ledgered 0)"))
|
||||
checks.append(("check-casts raw FAIL", check_casts_verdict(dict(z, X=1), [], {}, [], set())[0] == 1))
|
||||
# P39 T10: --residue — per form one covered site passes; uncovered three ways (no row; row without the site; RESIDUAL with no cause)
|
||||
for f, pos in (("P", "*(u16 *)(a0 + 4) = 1;"), ("I", "*(u8 *)D_80078EB4 = 1;"), ("X", "((s16 *)a0)[2] = 0;"), ("M", "M2C_FIELD(a0, u8 *, 1) = 0;")):
|
||||
w, (raw1, _) = cc(pos)
|
||||
s1 = [s for s in w["sites"] if s["form"] == f]
|
||||
e = dict(form=f, line=2, base=f"{s1[0]['bclass']}:{s1[0]['base']}", off=s1[0]["off"], verdict="RESIDUAL(GLOBAL)", cause="GLOBAL")
|
||||
key = ("S", "src/ov_TEST/k.c", "g")
|
||||
for name, lat, n_unc in (("covered", {key: dict(verdict="RESIDUAL", sites=[e])}, 0), ("no row", {}, 1),
|
||||
("site absent", {key: dict(verdict="RESIDUAL", sites=[dict(e, off=(e["off"] or 0) + 0x40)])}, 1),
|
||||
("empty cause", {key: dict(verdict="RESIDUAL", sites=[dict(e, cause="")])}, 1)):
|
||||
cov, unc, cause = residue_split(w["sites"], lat)
|
||||
rc_r, l_r = check_casts_verdict(raw1, [], lat, [], set(), uncovered=collections.Counter(s["form"] for s in unc))
|
||||
want = "(uncovered " + " ".join(f"{g}={int(g == f and n_unc)}" for g in ("P", "I", "X", "M")) + ")"
|
||||
checks.append((f"check-casts --residue {f} {name}", len(s1) == 1 and len(unc) == n_unc and len(cov) == 1 - n_unc
|
||||
and rc_r == n_unc and want in l_r and (n_unc or cause[id(cov[0])] == "GLOBAL")))
|
||||
wk, (_, mk) = cc("CAST_ALIAS(u16, a0, 4) = 2;")
|
||||
covk, unck, _ = residue_split(wk["sites"], {})
|
||||
checks.append(("check-casts --residue KEPT macro under macros", len(mk) == 1 and covk == [] and unck == []))
|
||||
bad = [n for n, ok in checks if not ok]
|
||||
print(f"type_census --selftest: {len(checks) - len(bad)}/{len(checks)} checks OK" + (f"; FAILED: {bad}" if bad else ""))
|
||||
return 0 if not bad else 1
|
||||
@@ -1979,8 +1996,9 @@ def check_casts_inputs():
|
||||
return latest, exc
|
||||
|
||||
|
||||
def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions):
|
||||
"""(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV."""
|
||||
def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions, uncovered=None):
|
||||
"""(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV.
|
||||
P39 T10 (--residue): `uncovered` {form: n} — the line adds `(uncovered P=…)` and the rc tests uncovered instead of raw."""
|
||||
unbacked = 0
|
||||
for s in macro_sites:
|
||||
row = latest.get(("S", s["tu"], s["fn"]))
|
||||
@@ -2002,9 +2020,80 @@ def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions
|
||||
macros = collections.Counter(s["name"] for s in macro_sites)
|
||||
mtxt = " ".join(f"{n}={macros[n]}" for n in KEPT_MACROS if macros[n] > 0) or "none"
|
||||
raw = [raw_by_form.get(f, 0) for f in ("P", "I", "X", "M")]
|
||||
line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}; macros {mtxt} (unbacked {unbacked}); "
|
||||
unc = None if uncovered is None else [uncovered.get(f, 0) for f in ("P", "I", "X", "M")]
|
||||
utxt = "" if unc is None else f" (uncovered P={unc[0]} I={unc[1]} X={unc[2]} M={unc[3]})"
|
||||
line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}{utxt}; macros {mtxt} (unbacked {unbacked}); "
|
||||
f"lying={lying} (ledgered {lying - unledgered})")
|
||||
return (1 if any(raw) or unbacked or unledgered else 0), line
|
||||
return (1 if any(raw if unc is None else unc) or unbacked or unledgered else 0), line
|
||||
|
||||
|
||||
# --check-casts --residue (P39 T10): a raw P/I/X/M site passes only when a rung-S ledger entry RESIDUAL(<cause>) covers it
|
||||
# (restruct.residue_match — per site, never re-implemented here).
|
||||
RESIDUE_SEGMENTS = (("ov", ["ov_*"]), ("md", ["md_*"]), ("main", ["main"])) # else "shared" (src/shared/, include/, other dirs)
|
||||
|
||||
|
||||
def residue_split(sites, latest):
|
||||
"""(covered, uncovered, {id(covered site): ledger cause}) over the raw P/I/X/M census sites (not common.h's #define lines). The cause
|
||||
is read back from the covering body's RESIDUAL entries in ledger order (annotation only; coverage is residue_match's)."""
|
||||
import restruct as rs
|
||||
sites = [s for s in sites if s.get("form") in ("P", "I", "X", "M") and not s.get("defline")]
|
||||
covered, uncovered = rs.residue_match(latest, sites, "P,I,X,M")
|
||||
pools, cause = {}, {}
|
||||
for s in covered:
|
||||
bk = (s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)
|
||||
if bk not in pools:
|
||||
pools[bk] = collections.defaultdict(list)
|
||||
for e in (latest.get(("S",) + bk) or {}).get("sites") or []:
|
||||
if str(e.get("verdict") or "").startswith("RESIDUAL(") and e.get("cause"):
|
||||
pools[bk][rs._residue_key(e.get("form"), e.get("base"), e.get("off"))].append(e["cause"])
|
||||
q = pools[bk][rs._residue_key(s["form"], f"{s.get('bclass')}:{s.get('base')}", s.get("off"))]
|
||||
cause[id(s)] = q.pop(0) if q else "?"
|
||||
return covered, uncovered, cause
|
||||
|
||||
|
||||
def residue_census(covered, uncovered, cause, out_dir):
|
||||
"""Write <out_dir>/residue_census.{json,txt}: covered/uncovered by form, by cause, by segment; the top 50 map types by covered sites
|
||||
(sites, bodies); how many map types cover 50/80/95% of the TYPE-NOT-CANONICAL sites. Returns the by-cause Counter."""
|
||||
import restruct as rs
|
||||
forms = ("P", "I", "X", "M")
|
||||
seg_of = lambda s: next((n for n, only in RESIDUE_SEGMENTS if rs.matches_only(only, s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)), "shared")
|
||||
by_form = {f: dict(covered=0, uncovered=0) for f in forms}
|
||||
by_seg = {n: dict(covered=0, uncovered=0) for n in [n for n, _ in RESIDUE_SEGMENTS] + ["shared"]}
|
||||
for k, ss in (("covered", covered), ("uncovered", uncovered)):
|
||||
for s in ss:
|
||||
by_form[s["form"]][k] += 1
|
||||
by_seg[seg_of(s)][k] += 1
|
||||
by_cause = collections.Counter(cause[id(s)] for s in covered)
|
||||
tsites, tbodies = collections.Counter(), collections.defaultdict(set)
|
||||
tnc = collections.Counter()
|
||||
for s in covered:
|
||||
if s.get("type"):
|
||||
tsites[s["type"]] += 1
|
||||
tbodies[s["type"]].add((s["tu"], s.get("fn")))
|
||||
if cause[id(s)] == "TYPE-NOT-CANONICAL":
|
||||
tnc[s["type"]] += 1
|
||||
top = [dict(type=t, sites=n, bodies=len(tbodies[t])) for t, n in sorted(tsites.items(), key=lambda kv: (-kv[1], kv[0]))[:50]]
|
||||
tnc_total, cover, acc, i = sum(tnc.values()), {}, 0, 0
|
||||
ranked = sorted(tnc.values(), reverse=True)
|
||||
for pct in (50, 80, 95):
|
||||
while i < len(ranked) and acc * 100 < pct * tnc_total:
|
||||
acc += ranked[i]
|
||||
i += 1
|
||||
cover[str(pct)] = i
|
||||
doc = dict(by_form=by_form, by_cause=dict(sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))), by_segment=by_seg,
|
||||
top_types=top, tnc=dict(sites=tnc_total, types=len(tnc), types_to_cover_pct=cover))
|
||||
out = REPO / out_dir
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / "residue_census.json").write_text(json.dumps(doc, indent=1))
|
||||
L = ["residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL(<cause>) ledger entry", "", "by form (covered / uncovered):"]
|
||||
L += [f" {f} {by_form[f]['covered']:>7} / {by_form[f]['uncovered']}" for f in forms]
|
||||
L += ["", "by segment (covered / uncovered):"] + [f" {n:<7}{v['covered']:>7} / {v['uncovered']}" for n, v in by_seg.items()]
|
||||
L += ["", "by cause (covered sites):"] + [f" {c:<22}{n:>7}" for c, n in doc["by_cause"].items()]
|
||||
L += ["", "top 50 map types by covered sites (sites bodies type):"] + [f" {r['sites']:>6} {r['bodies']:>6} {r['type']}" for r in top]
|
||||
L += ["", f"TYPE-NOT-CANONICAL: {tnc_total} sites over {len(tnc)} distinct map types; types to cover 50%={cover['50']} "
|
||||
f"80%={cover['80']} 95%={cover['95']}"]
|
||||
(out / "residue_census.txt").write_text("\n".join(L) + "\n")
|
||||
return by_cause
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def main():
|
||||
@@ -2016,12 +2105,16 @@ def main():
|
||||
ap.add_argument("--check", action="store_true", help="T8's gate (the invariants; exit 1 on a violation)")
|
||||
ap.add_argument("--check-structs", action="store_true", help="the struct-unification gate (P38 T2; exit 1 on a violation)")
|
||||
ap.add_argument("--check-casts", action="store_true", help="the cast-and-lying gate (P39 T2; exit 1 on a raw cast, an unbacked macro site or an unledgered lie)")
|
||||
ap.add_argument("--residue", action="store_true", help="with --check-casts (P39 T10): a raw P/I/X/M site passes when a rung-S ledger "
|
||||
"entry RESIDUAL(<cause>) covers it (restruct.residue_match); prints the uncovered counts and a by-cause line, exits 1 on any "
|
||||
"uncovered site; writes <out-dir>/residue_census.{json,txt} (by form, cause, segment; top 50 map types; TNC 50/80/95%%)")
|
||||
ap.add_argument("--selftest", action="store_true")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
a = ap.parse_args()
|
||||
if a.selftest:
|
||||
sys.exit(selftest())
|
||||
summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites)
|
||||
want_residue = a.check_casts and a.residue
|
||||
summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites or want_residue)
|
||||
if a.quiet:
|
||||
print(txt.strip().split("\n")[-2])
|
||||
print(txt.strip().split("\n")[-1])
|
||||
@@ -2058,9 +2151,19 @@ def main():
|
||||
rc = rc or rc_s
|
||||
if a.check_casts:
|
||||
latest, exc = check_casts_inputs()
|
||||
unc, bc_line = None, None
|
||||
if want_residue:
|
||||
with open(REPO / a.out_dir / "sites.jsonl") as fh:
|
||||
sites = [json.loads(l) for l in fh if l.startswith(('{"form": "P"', '{"form": "I"', '{"form": "X"', '{"form": "M"'))]
|
||||
covered, uncovered, cause = residue_split(sites, latest)
|
||||
unc = collections.Counter(s["form"] for s in uncovered)
|
||||
by_cause = residue_census(covered, uncovered, cause, a.out_dir)
|
||||
bc_line = "by-cause: " + (" ".join(f"{c}={n}" for c, n in sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))) or "none")
|
||||
rc_c, line = check_casts_verdict(summary["casts"]["check_raw"], summary["casts"]["macro_sites"], latest,
|
||||
summary["decls"]["lying_rows"], exc)
|
||||
summary["decls"]["lying_rows"], exc, uncovered=unc)
|
||||
print(line)
|
||||
if bc_line:
|
||||
print(bc_line)
|
||||
rc = rc or rc_c
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
@@ -55,3 +55,4 @@ p38-t4-2026-09-29-restruct-head-fold-mode.md | §P38 T4 — 2026-09-29: restruct
|
||||
p39-t1-2026-10-01-restruct-stale-supersede-timeline-check.md | §P39 T1 — 2026-10-01: restruct stale-row supersede; timeline --check | 5
|
||||
p39-t3-2026-10-02-restruct-rung-s-residue-head-types-add-pads.md | §P39 T3 — 2026-10-02: restruct rung S residue/check-residue; head_types --add-pads | 6
|
||||
p39-t4-2026-10-02-restruct-only-main.md | §P39 T4 — 2026-10-02: restruct --only main | 5
|
||||
p39-t10-2026-10-02-check-casts-residue.md | §P39 T10 — 2026-10-02: type_census --check-casts --residue; restruct --forms per-site residue | 5
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# §P39 T10 — 2026-10-02: type_census --check-casts --residue; restruct --forms / per-site --check-residue
|
||||
|
||||
- `type_census.py --check-casts --residue`: a raw P/I/X/M census site (not include/common.h's `#define` lines) passes only when its body's latest rung-S ledger row (`ledger_latest`, R100) has a `RESIDUAL(<cause>)` entry with a non-empty cause, same form, normalised base and off (`restruct.residue_match`; one entry covers one site; never by line). Line: `check-casts: raw P=… I=… X=… M=… (uncovered P=… I=… X=… M=…); macros … (unbacked n); lying=n (ledgered n)` + `by-cause: <CAUSE>=n …` (covered sites, desc). Exit 1 on any uncovered, unbacked macro site or unledgered lie. Forces `sites.jsonl` (as `--sites`).
|
||||
- Writes `<out-dir>/residue_census.{json,txt}` (default `.run/P37/census/`): covered/uncovered by form, by segment (ov `ov_*` / md `md_*` / main / else shared, `restruct.matches_only`), by cause; top 50 map types by covered sites (sites, bodies); TYPE-NOT-CANONICAL sites, distinct types, and types needed to cover 50/80/95%.
|
||||
- Without `--residue`, `--check-casts` is unchanged (T2 line; exit 1 on any raw > 0). KEPT macro sites (form K) are counted under macros, never residue.
|
||||
- `restruct.py --rung S --residue --forms P,I,X,M --only <sel> --label <l>`: ledger-only rows for uncovered sites of the forms (default `P,X`); a body with a latest S row gets a merged copy (MEMBER/KEPT entries kept) plus one `RESIDUAL(<cause>)` per uncovered site. `--check-residue --forms …` counts uncovered PER SITE (not per body).
|
||||
- I/M causes: ABS-ADDR (base bclass abs/gaddr: fixed or symbol address), GLOBAL (bclass global); else the P/X causes (FILE-SCOPE, NO-TYPE-IN-MAP, TYPE-NOT-CANONICAL, TYPE-NOT-VISIBLE, site_class).
|
||||
@@ -0,0 +1,109 @@
|
||||
# T10.c2 — type_census --check-casts --residue, residue census, docs
|
||||
|
||||
## Changed
|
||||
- tools/type_census.py: `check_casts_verdict(..., uncovered=None)` :1999 — with `uncovered` the line adds `(uncovered P=… I=… X=… M=…)` and rc tests uncovered instead of raw; without it, byte-identical T2 line/rc.
|
||||
- `RESIDUE_SEGMENTS` :2032; `residue_split(sites, latest)` :2035 → (covered, uncovered, cause) over raw P/I/X/M non-defline sites via `restruct.residue_match(latest, sites, "P,I,X,M")`; cause read back from the covering body's RESIDUAL entries (multiset, ledger order) via `rs._residue_key` — annotation only.
|
||||
- `residue_census(covered, uncovered, cause, out_dir)` :2054 → `<out-dir>/residue_census.{json,txt}`: by form, segment (ov `ov_*` / md `md_*` / main via `rs.matches_only`, else shared), cause; top 50 map types (s["type"]) by covered sites with distinct (tu, fn) bodies; TNC sites/types and types to cover 50/80/95%.
|
||||
- `--residue` arg :2108; forces `want_sites` (reads `<out-dir>/sites.jsonl`); prints gate line then `by-cause:` line :2155.
|
||||
- selftest :1922: per form P/I/X/M: covered passes (rc 0, cause GLOBAL read back); uncovered rc 1 via no row / row with site absent (off +0x40) / RESIDUAL with empty cause; KEPT macro site (CAST_ALIAS) in macro_sites, not in covered/uncovered. 67/67.
|
||||
- docs/ops/p39-t10-2026-10-02-check-casts-residue.md + docs/ops/INDEX.md line; HOW_WE_WORK.md Tools row `type_census casts` (card check: chars=6354 cap=7000).
|
||||
|
||||
## Verified
|
||||
- `/usr/bin/python3 tools/type_census.py --selftest` → 67/67 checks OK
|
||||
- `/usr/bin/python3 tools/restruct.py --selftest` → 101/101 OK
|
||||
- `type_census.py --check-casts` (no --residue) → T2 shape, RC 1:
|
||||
`check-casts: raw P=257464 I=51505 X=13790 M=15130; macros CAST_ALIAS=1173 CAST_SIGN=8429 CAST_WIDTH=10410 CAST_MISALIGNED=624 CAST_NONSTRUCT=327 (unbacked 0); lying=229 (ledgered 0)`
|
||||
- `type_census.py --check-casts --residue` → RC 1:
|
||||
`check-casts: raw P=257464 I=51505 X=13790 M=15130 (uncovered P=0 I=35625 X=0 M=14762); macros CAST_ALIAS=1173 CAST_SIGN=8429 CAST_WIDTH=10410 CAST_MISALIGNED=624 CAST_NONSTRUCT=327 (unbacked 0); lying=229 (ledgered 0)`
|
||||
`by-cause: TYPE-NOT-CANONICAL=281252 NO-TYPE-IN-MAP=2966 NO-FIELD=1773 NO-EDIT=482 INDEX=479 AGGREGATE-FIELD=369 FILE-SCOPE=172 TYPE-NOT-VISIBLE=9`
|
||||
- Reconcile: by-cause sum 287502 = P+X raw 271254 + I covered 15880 + M covered 368; no `?` cause (every covered site's cause read back).
|
||||
- `--help` documents `--residue`.
|
||||
- `make kit-corpus && make tools-health` → RC=0, "tools-health: OK"; corpus copy decomp-architect/corpus/tools/P10/type_census.py updated and committed.
|
||||
|
||||
## residue_census.txt (verbatim, .run/P37/census/residue_census.txt; gitignored, not committed)
|
||||
```
|
||||
residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL(<cause>) ledger entry
|
||||
|
||||
by form (covered / uncovered):
|
||||
P 257464 / 0
|
||||
I 15880 / 35625
|
||||
X 13790 / 0
|
||||
M 368 / 14762
|
||||
|
||||
by segment (covered / uncovered):
|
||||
ov 267984 / 48334
|
||||
md 6315 / 328
|
||||
main 1751 / 321
|
||||
shared 11452 / 1404
|
||||
|
||||
by cause (covered sites):
|
||||
TYPE-NOT-CANONICAL 281252
|
||||
NO-TYPE-IN-MAP 2966
|
||||
NO-FIELD 1773
|
||||
NO-EDIT 482
|
||||
INDEX 479
|
||||
AGGREGATE-FIELD 369
|
||||
FILE-SCOPE 172
|
||||
TYPE-NOT-VISIBLE 9
|
||||
|
||||
top 50 map types by covered sites (sites bodies type):
|
||||
21350 6239 Unkstruct_800D3888
|
||||
4340 124 Unkstruct_0caecd54
|
||||
4216 124 Unkstruct_dac960ce
|
||||
4061 4061 Unkstruct_f8ed2b9f
|
||||
3936 123 Unkstruct_a32b99a1
|
||||
3930 131 Unkstruct_8e3dbe51
|
||||
3271 196 Unkstruct_57a28935
|
||||
3144 131 Unkstruct_1fa9e810
|
||||
3100 124 Unkstruct_30b21035
|
||||
2646 126 Unkstruct_a2c8734a
|
||||
2580 129 Unkstruct_198fc7cc
|
||||
2356 124 Unkstruct_ed8f1335
|
||||
2213 705 Unkstruct_80126948
|
||||
2141 535 Unkstruct_8016B4F8
|
||||
2111 265 Unkstruct_80139C7C
|
||||
2109 317 Unkstruct_80130D48
|
||||
2091 123 Unkstruct_2729e670
|
||||
2016 126 Unkstruct_dd542e54
|
||||
2000 125 Unkstruct_f49880fd
|
||||
1948 1948 Unkstruct_a8d34bce
|
||||
1926 131 Unkstruct_8016DF5C
|
||||
1875 125 Unkstruct_568f7bda
|
||||
1764 126 Unkstruct_cd1bacd2
|
||||
1742 270 Unkstruct_8011F7A8
|
||||
1722 123 Unkstruct_84612d18
|
||||
1664 128 Unkstruct_57db4d07
|
||||
1612 124 Unkstruct_4692fc17
|
||||
1596 133 Unkstruct_eda88e53
|
||||
1572 131 Unkstruct_4e09b6d9
|
||||
1565 180 Unkstruct_8012D3B4
|
||||
1548 258 Unkstruct_645b745d
|
||||
1441 131 Unkstruct_0980b9f0
|
||||
1441 131 Unkstruct_b5c0372a
|
||||
1404 117 Unkstruct_47470ce3
|
||||
1364 124 Unkstruct_a68f6d03
|
||||
1336 134 Unkstruct_801345F8
|
||||
1300 130 Unkstruct_8fdd92d1
|
||||
1250 125 Unkstruct_110cf29e
|
||||
1240 124 Unkstruct_4db8cec8
|
||||
1240 124 Unkstruct_d292c30d
|
||||
1240 124 Unkstruct_f0bc3b7a
|
||||
1220 122 Unkstruct_a459e3e5
|
||||
1179 131 Unkstruct_f9a7989e
|
||||
1170 130 Unkstruct_80135260
|
||||
1149 259 Unkstruct_801290DC
|
||||
1128 376 Unkstruct_0b30043d
|
||||
1116 124 Unkstruct_301381d4
|
||||
1116 124 Unkstruct_4e427c01
|
||||
1107 123 Unkstruct_4857195e
|
||||
1040 130 Unkstruct_50a3f6b4
|
||||
|
||||
TYPE-NOT-CANONICAL: 281252 sites over 13081 distinct map types; types to cover 50%=81 80%=579 95%=4384
|
||||
```
|
||||
|
||||
## Notes for the expert
|
||||
- P/X uncovered 0: no bodies to report. I/M uncovered (35625 / 14762) await `restruct --residue --forms I,M` rows (T10.c1 tool).
|
||||
- Covered I/M sites carry no ABS-ADDR/GLOBAL cause yet: the 15880 I + 368 M covered come from existing rows' entries (causes in the by-cause line above).
|
||||
- Segment split uses matches_only order ov → md → main, else shared (src/shared/, include/, other dirs).
|
||||
- residue_census.{json,txt} live under gitignored `.run/`; not force-added (brief: "written").
|
||||
- harness: a census run (both gate runs) rewrites tracked docs/struct-map.md and .run/P37/census/{type_census.*,struct_map_top.json}; those were already dirty at start and are not committed here.
|
||||
+109
-6
@@ -1919,6 +1919,23 @@ def selftest():
|
||||
rz, lz = check_casts_verdict(z, [], {}, [], set())
|
||||
checks.append(("check-casts all-zero OK + line", rz == 0 and lz == "check-casts: raw P=0 I=0 X=0 M=0; macros none (unbacked 0); lying=0 (ledgered 0)"))
|
||||
checks.append(("check-casts raw FAIL", check_casts_verdict(dict(z, X=1), [], {}, [], set())[0] == 1))
|
||||
# P39 T10: --residue — per form one covered site passes; uncovered three ways (no row; row without the site; RESIDUAL with no cause)
|
||||
for f, pos in (("P", "*(u16 *)(a0 + 4) = 1;"), ("I", "*(u8 *)D_80078EB4 = 1;"), ("X", "((s16 *)a0)[2] = 0;"), ("M", "M2C_FIELD(a0, u8 *, 1) = 0;")):
|
||||
w, (raw1, _) = cc(pos)
|
||||
s1 = [s for s in w["sites"] if s["form"] == f]
|
||||
e = dict(form=f, line=2, base=f"{s1[0]['bclass']}:{s1[0]['base']}", off=s1[0]["off"], verdict="RESIDUAL(GLOBAL)", cause="GLOBAL")
|
||||
key = ("S", "src/ov_TEST/k.c", "g")
|
||||
for name, lat, n_unc in (("covered", {key: dict(verdict="RESIDUAL", sites=[e])}, 0), ("no row", {}, 1),
|
||||
("site absent", {key: dict(verdict="RESIDUAL", sites=[dict(e, off=(e["off"] or 0) + 0x40)])}, 1),
|
||||
("empty cause", {key: dict(verdict="RESIDUAL", sites=[dict(e, cause="")])}, 1)):
|
||||
cov, unc, cause = residue_split(w["sites"], lat)
|
||||
rc_r, l_r = check_casts_verdict(raw1, [], lat, [], set(), uncovered=collections.Counter(s["form"] for s in unc))
|
||||
want = "(uncovered " + " ".join(f"{g}={int(g == f and n_unc)}" for g in ("P", "I", "X", "M")) + ")"
|
||||
checks.append((f"check-casts --residue {f} {name}", len(s1) == 1 and len(unc) == n_unc and len(cov) == 1 - n_unc
|
||||
and rc_r == n_unc and want in l_r and (n_unc or cause[id(cov[0])] == "GLOBAL")))
|
||||
wk, (_, mk) = cc("CAST_ALIAS(u16, a0, 4) = 2;")
|
||||
covk, unck, _ = residue_split(wk["sites"], {})
|
||||
checks.append(("check-casts --residue KEPT macro under macros", len(mk) == 1 and covk == [] and unck == []))
|
||||
bad = [n for n, ok in checks if not ok]
|
||||
print(f"type_census --selftest: {len(checks) - len(bad)}/{len(checks)} checks OK" + (f"; FAILED: {bad}" if bad else ""))
|
||||
return 0 if not bad else 1
|
||||
@@ -1979,8 +1996,9 @@ def check_casts_inputs():
|
||||
return latest, exc
|
||||
|
||||
|
||||
def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions):
|
||||
"""(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV."""
|
||||
def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions, uncovered=None):
|
||||
"""(rc, line) — rc 1 iff any raw form > 0, any macro site unbacked, or any lying row not ledgered in the exceptions TSV.
|
||||
P39 T10 (--residue): `uncovered` {form: n} — the line adds `(uncovered P=…)` and the rc tests uncovered instead of raw."""
|
||||
unbacked = 0
|
||||
for s in macro_sites:
|
||||
row = latest.get(("S", s["tu"], s["fn"]))
|
||||
@@ -2002,9 +2020,80 @@ def check_casts_verdict(raw_by_form, macro_sites, latest, lying_rows, exceptions
|
||||
macros = collections.Counter(s["name"] for s in macro_sites)
|
||||
mtxt = " ".join(f"{n}={macros[n]}" for n in KEPT_MACROS if macros[n] > 0) or "none"
|
||||
raw = [raw_by_form.get(f, 0) for f in ("P", "I", "X", "M")]
|
||||
line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}; macros {mtxt} (unbacked {unbacked}); "
|
||||
unc = None if uncovered is None else [uncovered.get(f, 0) for f in ("P", "I", "X", "M")]
|
||||
utxt = "" if unc is None else f" (uncovered P={unc[0]} I={unc[1]} X={unc[2]} M={unc[3]})"
|
||||
line = (f"check-casts: raw P={raw[0]} I={raw[1]} X={raw[2]} M={raw[3]}{utxt}; macros {mtxt} (unbacked {unbacked}); "
|
||||
f"lying={lying} (ledgered {lying - unledgered})")
|
||||
return (1 if any(raw) or unbacked or unledgered else 0), line
|
||||
return (1 if any(raw if unc is None else unc) or unbacked or unledgered else 0), line
|
||||
|
||||
|
||||
# --check-casts --residue (P39 T10): a raw P/I/X/M site passes only when a rung-S ledger entry RESIDUAL(<cause>) covers it
|
||||
# (restruct.residue_match — per site, never re-implemented here).
|
||||
RESIDUE_SEGMENTS = (("ov", ["ov_*"]), ("md", ["md_*"]), ("main", ["main"])) # else "shared" (src/shared/, include/, other dirs)
|
||||
|
||||
|
||||
def residue_split(sites, latest):
|
||||
"""(covered, uncovered, {id(covered site): ledger cause}) over the raw P/I/X/M census sites (not common.h's #define lines). The cause
|
||||
is read back from the covering body's RESIDUAL entries in ledger order (annotation only; coverage is residue_match's)."""
|
||||
import restruct as rs
|
||||
sites = [s for s in sites if s.get("form") in ("P", "I", "X", "M") and not s.get("defline")]
|
||||
covered, uncovered = rs.residue_match(latest, sites, "P,I,X,M")
|
||||
pools, cause = {}, {}
|
||||
for s in covered:
|
||||
bk = (s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)
|
||||
if bk not in pools:
|
||||
pools[bk] = collections.defaultdict(list)
|
||||
for e in (latest.get(("S",) + bk) or {}).get("sites") or []:
|
||||
if str(e.get("verdict") or "").startswith("RESIDUAL(") and e.get("cause"):
|
||||
pools[bk][rs._residue_key(e.get("form"), e.get("base"), e.get("off"))].append(e["cause"])
|
||||
q = pools[bk][rs._residue_key(s["form"], f"{s.get('bclass')}:{s.get('base')}", s.get("off"))]
|
||||
cause[id(s)] = q.pop(0) if q else "?"
|
||||
return covered, uncovered, cause
|
||||
|
||||
|
||||
def residue_census(covered, uncovered, cause, out_dir):
|
||||
"""Write <out_dir>/residue_census.{json,txt}: covered/uncovered by form, by cause, by segment; the top 50 map types by covered sites
|
||||
(sites, bodies); how many map types cover 50/80/95% of the TYPE-NOT-CANONICAL sites. Returns the by-cause Counter."""
|
||||
import restruct as rs
|
||||
forms = ("P", "I", "X", "M")
|
||||
seg_of = lambda s: next((n for n, only in RESIDUE_SEGMENTS if rs.matches_only(only, s["tu"], s.get("fn") or rs.RESIDUE_FILE_UNIT)), "shared")
|
||||
by_form = {f: dict(covered=0, uncovered=0) for f in forms}
|
||||
by_seg = {n: dict(covered=0, uncovered=0) for n in [n for n, _ in RESIDUE_SEGMENTS] + ["shared"]}
|
||||
for k, ss in (("covered", covered), ("uncovered", uncovered)):
|
||||
for s in ss:
|
||||
by_form[s["form"]][k] += 1
|
||||
by_seg[seg_of(s)][k] += 1
|
||||
by_cause = collections.Counter(cause[id(s)] for s in covered)
|
||||
tsites, tbodies = collections.Counter(), collections.defaultdict(set)
|
||||
tnc = collections.Counter()
|
||||
for s in covered:
|
||||
if s.get("type"):
|
||||
tsites[s["type"]] += 1
|
||||
tbodies[s["type"]].add((s["tu"], s.get("fn")))
|
||||
if cause[id(s)] == "TYPE-NOT-CANONICAL":
|
||||
tnc[s["type"]] += 1
|
||||
top = [dict(type=t, sites=n, bodies=len(tbodies[t])) for t, n in sorted(tsites.items(), key=lambda kv: (-kv[1], kv[0]))[:50]]
|
||||
tnc_total, cover, acc, i = sum(tnc.values()), {}, 0, 0
|
||||
ranked = sorted(tnc.values(), reverse=True)
|
||||
for pct in (50, 80, 95):
|
||||
while i < len(ranked) and acc * 100 < pct * tnc_total:
|
||||
acc += ranked[i]
|
||||
i += 1
|
||||
cover[str(pct)] = i
|
||||
doc = dict(by_form=by_form, by_cause=dict(sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))), by_segment=by_seg,
|
||||
top_types=top, tnc=dict(sites=tnc_total, types=len(tnc), types_to_cover_pct=cover))
|
||||
out = REPO / out_dir
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / "residue_census.json").write_text(json.dumps(doc, indent=1))
|
||||
L = ["residue census (P39 T10): raw P/I/X/M sites covered by a rung-S RESIDUAL(<cause>) ledger entry", "", "by form (covered / uncovered):"]
|
||||
L += [f" {f} {by_form[f]['covered']:>7} / {by_form[f]['uncovered']}" for f in forms]
|
||||
L += ["", "by segment (covered / uncovered):"] + [f" {n:<7}{v['covered']:>7} / {v['uncovered']}" for n, v in by_seg.items()]
|
||||
L += ["", "by cause (covered sites):"] + [f" {c:<22}{n:>7}" for c, n in doc["by_cause"].items()]
|
||||
L += ["", "top 50 map types by covered sites (sites bodies type):"] + [f" {r['sites']:>6} {r['bodies']:>6} {r['type']}" for r in top]
|
||||
L += ["", f"TYPE-NOT-CANONICAL: {tnc_total} sites over {len(tnc)} distinct map types; types to cover 50%={cover['50']} "
|
||||
f"80%={cover['80']} 95%={cover['95']}"]
|
||||
(out / "residue_census.txt").write_text("\n".join(L) + "\n")
|
||||
return by_cause
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
def main():
|
||||
@@ -2016,12 +2105,16 @@ def main():
|
||||
ap.add_argument("--check", action="store_true", help="T8's gate (the invariants; exit 1 on a violation)")
|
||||
ap.add_argument("--check-structs", action="store_true", help="the struct-unification gate (P38 T2; exit 1 on a violation)")
|
||||
ap.add_argument("--check-casts", action="store_true", help="the cast-and-lying gate (P39 T2; exit 1 on a raw cast, an unbacked macro site or an unledgered lie)")
|
||||
ap.add_argument("--residue", action="store_true", help="with --check-casts (P39 T10): a raw P/I/X/M site passes when a rung-S ledger "
|
||||
"entry RESIDUAL(<cause>) covers it (restruct.residue_match); prints the uncovered counts and a by-cause line, exits 1 on any "
|
||||
"uncovered site; writes <out-dir>/residue_census.{json,txt} (by form, cause, segment; top 50 map types; TNC 50/80/95%%)")
|
||||
ap.add_argument("--selftest", action="store_true")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
a = ap.parse_args()
|
||||
if a.selftest:
|
||||
sys.exit(selftest())
|
||||
summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites)
|
||||
want_residue = a.check_casts and a.residue
|
||||
summary, txt = run_census(a.jobs, use_cache=not a.no_cache, out_dir=a.out_dir, want_sites=a.sites or want_residue)
|
||||
if a.quiet:
|
||||
print(txt.strip().split("\n")[-2])
|
||||
print(txt.strip().split("\n")[-1])
|
||||
@@ -2058,9 +2151,19 @@ def main():
|
||||
rc = rc or rc_s
|
||||
if a.check_casts:
|
||||
latest, exc = check_casts_inputs()
|
||||
unc, bc_line = None, None
|
||||
if want_residue:
|
||||
with open(REPO / a.out_dir / "sites.jsonl") as fh:
|
||||
sites = [json.loads(l) for l in fh if l.startswith(('{"form": "P"', '{"form": "I"', '{"form": "X"', '{"form": "M"'))]
|
||||
covered, uncovered, cause = residue_split(sites, latest)
|
||||
unc = collections.Counter(s["form"] for s in uncovered)
|
||||
by_cause = residue_census(covered, uncovered, cause, a.out_dir)
|
||||
bc_line = "by-cause: " + (" ".join(f"{c}={n}" for c, n in sorted(by_cause.items(), key=lambda kv: (-kv[1], kv[0]))) or "none")
|
||||
rc_c, line = check_casts_verdict(summary["casts"]["check_raw"], summary["casts"]["macro_sites"], latest,
|
||||
summary["decls"]["lying_rows"], exc)
|
||||
summary["decls"]["lying_rows"], exc, uncovered=unc)
|
||||
print(line)
|
||||
if bc_line:
|
||||
print(bc_line)
|
||||
rc = rc or rc_c
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user