From 3ac7abee732afc1d413078f9fd23697e63289ecd Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:32:52 -0600 Subject: [PATCH] T2.c1: type_census --check-structs gate; restruct ledger_latest + audit_type_violations; actor-struct s32 --- docs/actor-struct.md | 4 +-- docs/struct-map.md | 2 +- phase-ends/current/logs/T2.c1.md | 26 +++++++++++++++ tools/restruct.py | 39 +++++++++++++--------- tools/type_census.py | 56 ++++++++++++++++++++++++++++++-- 5 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 phase-ends/current/logs/T2.c1.md diff --git a/docs/actor-struct.md b/docs/actor-struct.md index 7326eabf22..ae15199f13 100644 --- a/docs/actor-struct.md +++ b/docs/actor-struct.md @@ -24,8 +24,8 @@ kept as a Gen2 quality artifact + decomp.me enabler, **not** a build input. | +0x0B4 | u16 | **hp_cur** | =150 (the plan's "+0xB4"; Phase-3 0x80078EB4) | | +0x0B6 | u16 | bp_max | =150 | | +0x0B8 | u16 | bp_cur (stamina) | 74→50 as the player walked (BP-drain) | -| +0x108 | s16 | position X | +6 on a walk north | -| +0x10C | s16 | position Z | +10 on a walk north | +| +0x108 | s32 | position X | +6 on a walk north; 4-byte (code width 4, census control `player_block_widths`, P37) | +| +0x10C | s32 | position Z | +10 on a walk north; 4-byte (code width 4, census control `player_block_widths`, P37) | | +0xF0 | s16 | step/heading counter | +1 per walk tick | ## Pointer fields (the m2c-can't-infer set; idle RAM scan + code-inferred) diff --git a/docs/struct-map.md b/docs/struct-map.md index 0705352d99..78d3917cd7 100644 --- a/docs/struct-map.md +++ b/docs/struct-map.md @@ -1,6 +1,6 @@ # The struct map — the types the fleet's cast sites need (Phase 37 T1) -> **Generated by `tools/type_census.py` at `a8cdd934b` (2026-09-30T02:11:20Z); never typed (R75).** Every raw pointer-cast dereference in `src/` was assigned to a base (a parameter, a local, a global by value or by address, an absolute address) and the bases were clustered by EVIDENCE only — the same global symbol, an argument passed to a callee's parameter, an assignment or return, a pointer field read, a shared body's fleet-wide identity — never by a similar offset signature. Each cluster is one type the tree needs. Names are placeholders (`Unkstruct_`: the lowest global instance's address, else the lowest member function's); fields are placeholders (`unk`) until Phase 38 cites evidence. +> **Generated by `tools/type_census.py` at `09b00a443` (2026-09-30T02:32:09Z); never typed (R75).** Every raw pointer-cast dereference in `src/` was assigned to a base (a parameter, a local, a global by value or by address, an absolute address) and the bases were clustered by EVIDENCE only — the same global symbol, an argument passed to a callee's parameter, an assignment or return, a pointer field read, a shared body's fleet-wide identity — never by a similar offset signature. Each cluster is one type the tree needs. Names are placeholders (`Unkstruct_`: the lowest global instance's address, else the lowest member function's); fields are placeholders (`unk`) until Phase 38 cites evidence. **18739 types explain 498866 of 502985 dereference sites (99.2 %); 4119 sites have a base the census could not classify; 727 types carry a width conflict.** Merges by evidence: param 36454, assign 14664, arg 10168, field 9613, ret-of 3373, return 1524, assign& 1050, arg& 270. diff --git a/phase-ends/current/logs/T2.c1.md b/phase-ends/current/logs/T2.c1.md new file mode 100644 index 0000000000..a0dd7908b2 --- /dev/null +++ b/phase-ends/current/logs/T2.c1.md @@ -0,0 +1,26 @@ +# T2.c1 — type_census --check-structs + +## Changed +- `tools/type_census.py`: `--check-structs` flag (:1646, main :1679-1684); `check_structs_inputs()` (lazy `import restruct`, cycle: restruct imports type_census) + `check_structs_verdict(summary, pad_names, parse_error_decls) -> (rc, [verdict, counts, controls])` :1606-1637; `PAD_VIOL_RX` filters restruct's audit strings to `.pad: named for offset`; 3 selftests (all-zero OK rc0; parse_error_decls=3 FAIL rc1; controls 0/1 FAIL) :1593-1601; USAGE line :35. +- Controls: `PLAYER_BLOCK` gains `D_80078F08:4, D_80078F0C:4`; `PLAYER_BLOCK_DOC_DISPUTED = {}` (old value kept in a comment) :1407-1409. There is no doc parser: the disputed dict was hard-coded, so the doc fix alone could not clear it. +- `tools/restruct.py`: new `ledger_latest(rows) -> {(rung,tu,unit): row}` :395 — the one latest-row filter; `ledger_index`, `settled_units`, and apply_batch's `--redraw` vc map (a third hand-rolled latest loop, :2556) now read it. `audit_types` split: `audit_type_violations() -> (n, viol)` :2730, `audit_types` prints it (output unchanged). +- CANON_HEADERS: restruct already `CANON_HEADERS = tc.CANON_HEADERS` (:84); struct_layout has no list. No change needed. +- `docs/actor-struct.md` :27-28 +0x108/+0x10C s16 -> s32 with the P37 control citation. +- `docs/struct-map.md`: regenerated by the census run (stamp line only). + +## Verified +- `.venv/bin/python tools/type_census.py --selftest` -> 24/24 OK (21 + 3). +- `.venv/bin/python tools/restruct.py --selftest` -> restruct 54/54 OK, struct_layout 9/9. +- `--out-dir .run/P38/census --check-structs` -> exit=1 (.run/logs/t2c1_cs.log): + - `type_census --check-structs: FAIL — dup_classes=206; outside_canon=6076; dead=141; variants=40; pad_names=26; parse_error_decls=1106` + - `dup_classes=206 outside_canon=6076 dead=141 variants=40 pad_names=26 parse_error_decls=1106 controls=4/4` + - `controls: player_block_widths=OK matrix_layouts=OK prim_layout_class=OK tu_conflict_D_800A651C=OK doc_disputed=0` + - `conflicting_types=160 tu_conflict=0 types_floor_lying=1796 audit_other=0 (not gating)` +- `--check` line before (.run/logs/t2c1_check_before.log) == after (t2c1_check_after.log): `type_census --check: FAIL — 206 duplicate-layout classes; struct definitions outside the canonical type files; 502985 raw cast sites; 1796 lying declarations`. +- `restruct --audit-types` -> 1179 definitions, 26 violations, all 26 `.padX: named for offset` (audit_other=0). + +## Notes for the expert +- parse_error_decls=1106 (latest row per unit, DECL-KEPT cause `type not visible`); over ALL ledger rows the count is 1107 (one unit re-judged). `restruct --status` counts all rows: `D DECL-KEPT 3168` vs 3151 latest; it has no cause split, so it cross-checks only the DECL-KEPT total. P37's ≈1,120 is consistent. +- tu_conflict=0: the restruct ledger carries no `TU-CONFLICT` verdict (0 rows in 171,583); TU-CONFLICT lives in share_body / config/dedup_exceptions.tsv. The line reports the ledger as briefed. +- `audit_other` added to the non-gating line (the brief's "report both numbers"). +- A full census run with a changed tool stamp takes ~293 s (cache miss); run it via `tools/run.sh --bg`. diff --git a/tools/restruct.py b/tools/restruct.py index 7893f124d9..8f2972535d 100644 --- a/tools/restruct.py +++ b/tools/restruct.py @@ -392,15 +392,22 @@ def ledger_append(rows): f.write(json.dumps(r) + "\n") +def ledger_latest(rows): + """{(rung, tu, unit): row} — the LATEST row per unit (R70: the last row is the verdict). The one latest-row filter; its callers read + verdict/cause from the row (ledger_index, settled_units, type_census --check-structs).""" + latest = {} + for r in rows: + latest[(r.get("rung"), r["tu"], r["unit"])] = r + return latest + + def ledger_index(rows): """done: {(rung, tu, unit, nhash_after)}; latest: {(rung, tu, unit): verdict}.""" - done, latest = set(), {} + done = set() for r in rows: - key = (r.get("rung"), r["tu"], r["unit"]) - latest[key] = r.get("verdict") if r.get("verdict") in DONE_VERDICTS and r.get("nhash_after"): - done.add(key + (r["nhash_after"],)) - return done, latest + done.add((r.get("rung"), r["tu"], r["unit"], r["nhash_after"])) + return done, {k: r.get("verdict") for k, r in ledger_latest(rows).items()} def body_nhash(masked, d): @@ -2488,10 +2495,8 @@ def settled_units(rows, redraw=()): """{(tu, callee)} whose LATEST rung-D row is a DONE verdict not lifted by `redraw` (R70: the last row is the verdict). A redraw key is a verdict (`DECL-KEPT`) or `VERDICT:cause-substring` (`DECL-NONE:no visible definition` — the 667 rows settled before defs_visible's fleet-unique level; the plain `DECL-NONE` key redraws every "already canonical"/external pair too, the t4_D21–D36 REDRAW's cost).""" - latest = {} - for r in rows: - if r.get("rung") == "D" and r.get("unit", "").startswith("decl:"): - latest[(r["tu"], r["callee"])] = (r.get("verdict"), r.get("cause") or "") + latest = {(r["tu"], r["callee"]): (r.get("verdict"), r.get("cause") or "") + for (rung, _tu, unit), r in ledger_latest(rows).items() if rung == "D" and unit.startswith("decl:")} keys = [(k.split(":", 1)[0], k.split(":", 1)[1] if ":" in k else None) for k in redraw] def lifted(v, c): return any(v == kv and (ks is None or ks in c) for kv, ks in keys) @@ -2548,10 +2553,7 @@ def apply_batch(a): if a.redraw: # the same VERDICT[:cause-substring] keys as settled_units (the planner's side): the engine's `done` filter must lift the same # rows, or a redrawn TU is drawn and then skipped as done — "rows 0" (t4_D49/D56 on the `DECL-NONE:visible` key) - vc = {} - for r in rows: - if r.get("rung") == "D" and r.get("unit"): - vc[(r["tu"], r["unit"])] = (r.get("verdict"), r.get("cause") or "") + vc = {(tu, unit): (r.get("verdict"), r.get("cause") or "") for (rung, tu, unit), r in ledger_latest(rows).items() if rung == "D" and unit} keys = [(k.split(":", 1)[0], k.split(":", 1)[1] if ":" in k else None) for k in a.redraw] def lifted(k): v, c = vc.get((k[1], k[2]), (None, "")) @@ -2725,8 +2727,9 @@ def write_types(a): return 0 -def audit_types(a=None, quiet=False): - """struct_layout.audit_definition over every canonical type file's definitions (with their raw text for the comment/trailer checks).""" +def audit_type_violations(): + """struct_layout.audit_definition over every canonical type file's definitions (with their raw text for the comment/trailer checks): + (n definitions audited, [violation strings]). Shared by --audit-types and type_census --check-structs.""" viol, n = [], 0 for h in CANON_HEADERS: if not (REPO / h).exists(): @@ -2748,6 +2751,12 @@ def audit_types(a=None, quiet=False): text = raw[ls[d["line"] - 1]:ls[d["end"]] if d["end"] < len(ls) else len(raw)] for v in sl.audit_definition(d, res, text): viol.append(f"{h}:{d['line']} {v}") + return n, viol + + +def audit_types(a=None, quiet=False): + """--audit-types: print audit_type_violations(); exit 1 on any.""" + n, viol = audit_type_violations() if not quiet: print(f"restruct --audit-types: {n} canonical definitions audited; {len(viol)} naming/offset violation(s)") for v in viol[:40]: diff --git a/tools/type_census.py b/tools/type_census.py index f7e6959b00..62c2ff1f07 100644 --- a/tools/type_census.py +++ b/tools/type_census.py @@ -31,7 +31,8 @@ USAGE tools/type_census.py [-j N] [--out-dir .run/P37/census] [--no-cache] # the census + the map + the render tools/type_census.py --sites # also write sites.jsonl (every record) tools/type_census.py --check # T8's gate (exit 1 on any violation) - tools/type_census.py --selftest # the fixture: every form, both verdicts + tools/type_census.py --check-structs # P38's struct gate: dup/outside-canon/dead/variants/pad names/parse-error decls/controls + tools/type_census.py --selftest # the fixture: every form, both verdicts Outputs: /type_census.json, type_census.txt, struct_map.json, sites.jsonl (--sites), docs/struct-map.md (rendered). """ import argparse @@ -1403,8 +1404,9 @@ def write_parked(out_dir): # controls (R39) # ---------------------------------------------------------------------------------------------------------------------- PLAYER_BLOCK = { # docs/actor-struct.md + docs/memory-map.md §3.4 — the widths BOTH the record and the matched code agree on - "D_80078E8C": 4, "D_80078EB2": 2, "D_80078EB4": 2, "D_80078EB6": 2, "D_80078EB8": 2} -PLAYER_BLOCK_DOC_DISPUTED = {"D_80078F08": 2, "D_80078F0C": 2} # actor-struct.md says s16 at +0x108/+0x10C; the tree's byte-true declarations decide (reported, not asserted) + "D_80078E8C": 4, "D_80078EB2": 2, "D_80078EB4": 2, "D_80078EB6": 2, "D_80078EB8": 2, "D_80078F08": 4, "D_80078F0C": 4} +# was {"D_80078F08": 2, "D_80078F0C": 2} (actor-struct.md said s16 at +0x108/+0x10C; P37 census: code width 4; doc corrected to s32 at P38 T2) +PLAYER_BLOCK_DOC_DISPUTED = {} # the doc disagrees with the tree's byte-true declarations here (reported, not asserted) def controls(defs_all, sites_all, extern_data, by_text, name_layouts): out = {} @@ -1588,10 +1590,51 @@ def selftest(): checks.append(("K&R-empty extern", any(e["name"] == "func_80002000" and e["params"] == "" for e in res["extern_fns"]))) checks.append(("typedef alias fnptr", any(a["name"] == "Handler" and a["fnptr"] for a in res["typedef_aliases"]))) checks.append(("params of f1", res["fndefs"][0]["pnames"] == ["a0", "b", "c"])) + zero = dict(definitions=dict(dup_layout_classes=0, in_c_file_scope=0, in_c_block_scope=0, in_shared_fn_headers=0, other_headers=0, + dead_canon_names=0, variant_names=0), controls=dict(a=dict(ok=True), ok=1, n=1)) + rc0, l0 = check_structs_verdict(zero, 0, 0) + checks.append(("check-structs all-zero OK", rc0 == 0 and l0[0] == "type_census --check-structs: OK" and "controls=1/1" in l0[1])) + rc1, l1 = check_structs_verdict(zero, 0, 3) + checks.append(("check-structs parse_error_decls FAIL", rc1 == 1 and l1[0].startswith("type_census --check-structs: FAIL") and "parse_error_decls=3" in l1[1])) + bad_ctrl = dict(zero, controls=dict(a=dict(ok=False), ok=0, n=1)) + rc2, l2 = check_structs_verdict(bad_ctrl, 0, 0) + checks.append(("check-structs controls a