From d71d4c179ce2558478155fbcac40bca22534a98e Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:23:48 -0600 Subject: [PATCH] T5.1.c4: struct_twins.py: tier-2 twin evidence; ## Twins lists 19 SEPARATE classes --- docs/struct-twins.md | 123 +++++++++ phase-ends/current/logs/T5.1.c4.md | 9 + tools/struct_twins.py | 405 +++++++++++++++++++++++++++++ 3 files changed, 537 insertions(+) create mode 100644 phase-ends/current/logs/T5.1.c4.md create mode 100644 tools/struct_twins.py diff --git a/docs/struct-twins.md b/docs/struct-twins.md index 63650cd7fd..9972370e6e 100644 --- a/docs/struct-twins.md +++ b/docs/struct-twins.md @@ -10,4 +10,127 @@ ## Twins + + +### f1dcb3a42fc8 +names: Rec_801825D8, Rec_80184FF8, S8012C658, S8017FF9C_sp +cause: 4 names (20 B, all members placeholders) bound to 5 distinct objects, none bound to two names: ov_SC02_011/0x801890C8:sp, ov_SC04_003/0x8017FF9C:sp, ov_SC06_029/0x801825D8:rec, ov_SC06_029/0x80184FF8:rec, shared/ov/0x8012C658:sp; full list: tools/struct_twins.py --show f1dcb3a42fc8 +- `Rec_801825D8`: 1 obj — ov_SC06_029/0x801825D8:rec +- `Rec_80184FF8`: 1 obj — ov_SC06_029/0x80184FF8:rec +- `S8012C658`: 2 obj — ov_SC02_011/0x801890C8:sp, shared/ov/0x8012C658:sp +- `S8017FF9C_sp`: 1 obj — ov_SC04_003/0x8017FF9C:sp + +### b519bad67107:opaque +names: Local_8017EE34, Local_80181C84, Local_801870E8 +cause: 3 names (20 B, all members placeholders) bound to 4 distinct objects, none bound to two names: ov_SC03_105/0x8017EE34:local, ov_SC03_105/0x80181C84:local, ov_SC03_105/0x80183D38:local, ov_SC03_105/0x801870E8:local; full list: tools/struct_twins.py --show b519bad67107:opaque +- `Local_8017EE34`: 2 obj — ov_SC03_105/0x8017EE34:local, ov_SC03_105/0x80183D38:local +- `Local_80181C84`: 1 obj — ov_SC03_105/0x80181C84:local +- `Local_801870E8`: 1 obj — ov_SC03_105/0x801870E8:local + +### c80fac242f04 +names: Mat801A8E34, Mat_80183C9C, Mtx_A8CE0 +cause: 3 names (32 B, all members placeholders) bound to 4 distinct objects, none bound to two names: md_SC07_004/0x801A8CE0:mtx, md_SC07_004/0x801A8E34:buf, ov_SC06_000/0x80183C9C:mat; full list: tools/struct_twins.py --show c80fac242f04 +- `Mat801A8E34`: 1 obj — md_SC07_004/0x801A8E34:buf +- `Mat_80183C9C`: 1 obj — ov_SC06_000/0x80183C9C:mat +- `Mtx_A8CE0`: 2 obj — md_SC07_004/0x801A8CE0:mtx, __attribute__(ret) + +### 9ba0ae408dd4 +names: Drv_8017E330, Drv_8017E550, Drv_8017E878 +cause: 3 names (56 B, all members placeholders) bound to 6 distinct objects, none bound to two names: 0x801F80E4, ov_SC06_006/0x8017E330:p, ov_SC06_006/0x8017E550:a0, ov_SC06_006/0x8017E550:p, ov_SC06_006/0x8017E878:a0, ov_SC06_006/0x8017E878:p; full list: tools/struct_twins.py --show 9ba0ae408dd4 +- `Drv_8017E330`: 2 obj — 0x801F80E4, ov_SC06_006/0x8017E330:p +- `Drv_8017E550`: 2 obj — ov_SC06_006/0x8017E550:a0, ov_SC06_006/0x8017E550:p +- `Drv_8017E878`: 2 obj — ov_SC06_006/0x8017E878:a0, ov_SC06_006/0x8017E878:p + +### 8944ddcbe454:opaque +names: Blk36_800296F8, Blob36, SW20 +cause: 3 names (36 B, all members placeholders) bound to 6 distinct objects, none bound to two names: 0x800298BC(arg0), 0x80078E50, 0x80079204, 0x800D4B70, main/0x800296F8:arg0, src/resident/0x800D185C:arg0; full list: tools/struct_twins.py --show 8944ddcbe454:opaque +- `Blk36_800296F8`: 3 obj — 0x800298BC(arg0), 0x80079204, main/0x800296F8:arg0 +- `Blob36`: 2 obj — 0x80078E50, src/resident/0x800D185C:arg0 +- `SW20`: 1 obj — 0x800D4B70 + +### c2ba48790859 +names: W8, W8_, func_8002EE64_b +cause: 3 names (1 B, all members placeholders) bound to 4 distinct objects, none bound to two names: 0x80076242, 0x80076243, main/0x8002EE64:b; full list: tools/struct_twins.py --show c2ba48790859 +- `W8`: 2 obj — 0x80076242, 0x80076243 +- `W8_`: 1 obj — D_80076242_ +- `func_8002EE64_b`: 1 obj — main/0x8002EE64:b + +### ac91ae2d8aa0:opaque +names: S800D22E4, vec +cause: 2 names (28 B, all members placeholders) bound to 77 distinct objects, none bound to two names: 0x800D22E4(arg0), 0x8012931C(arg0), 0x8012AD80(arg0), 0x80185238(arg0), 0x801879E8(arg0), md_SC07_004/0x801A72DC:a0, md_SC07_004/0x801ABC8C:a0, md_SC07_004/0x801AC234:a0 …; full list: tools/struct_twins.py --show ac91ae2d8aa0:opaque +- `S800D22E4`: 1 obj — 0x800D22E4(arg0) +- `vec`: 76 obj — 0x8012931C(arg0), 0x8012AD80(arg0), 0x80185238(arg0), 0x801879E8(arg0), md_SC07_004/0x801A72DC:a0, md_SC07_004/0x801ABC8C:a0, md_SC07_004/0x801AC234:a0, md_SC07_004/0x801AC940:a1 (+68) + +### 82c6e3650290:opaque +names: Gte8_8017F2FC, Rec_80180808 +cause: 2 names (8 B, all members placeholders) bound to 3 distinct objects, none bound to two names: 0x801B1B50, ov_SC02_021/0x8017F2FC:sp10, ov_SC06_025/0x80180808:rec; full list: tools/struct_twins.py --show 82c6e3650290:opaque +- `Gte8_8017F2FC`: 1 obj — ov_SC02_021/0x8017F2FC:sp10 +- `Rec_80180808`: 2 obj — 0x801B1B50, ov_SC06_025/0x80180808:rec + +### aa66418d079c:opaque +names: Blk16_80126940, Pos16_8018743C +cause: 2 names (16 B, all members placeholders) bound to 3 distinct objects, none bound to two names: ov_SC06_000/0x80181AB4:sp10, ov_SC06_029/0x8018743C:from, ov_SC06_029/0x8018743C:to; full list: tools/struct_twins.py --show aa66418d079c:opaque +- `Blk16_80126940`: 1 obj — ov_SC06_000/0x80181AB4:sp10 +- `Pos16_8018743C`: 2 obj — ov_SC06_029/0x8018743C:from, ov_SC06_029/0x8018743C:to + +### 1dd90dc86138 +names: Ent80183F28, Ent_80184058 +cause: 2 names (236 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x80183F28(arg0), 0x80184058(arg0); full list: tools/struct_twins.py --show 1dd90dc86138 +- `Ent80183F28`: 1 obj — 0x80183F28(arg0) +- `Ent_80184058`: 1 obj — 0x80184058(arg0) + +### d60a32e09c91 +names: P8, S72 +cause: 2 names (8 B, all members placeholders) bound to 5 distinct objects, none bound to two names: 0x80185D28, 0x80185D2A, 0x80185D2C, 0x80185D2E, 0x80185D72; full list: tools/struct_twins.py --show d60a32e09c91 +- `P8`: 4 obj — 0x80185D28, 0x80185D2A, 0x80185D2C, 0x80185D2E +- `S72`: 1 obj — 0x80185D72 + +### 08944be666a1 +names: Sub8017EBC4, Sub_80181BA0 +cause: 2 names (20 B, all members placeholders) bound to 3 distinct objects, none bound to two names: ov_SC04_011/0x80181BA0:p, ov_SC06_029/0x8017EBC4:func_8012C1B8, ov_SC06_029/0x8017EBC4:sp; full list: tools/struct_twins.py --show 08944be666a1 +- `Sub8017EBC4`: 2 obj — ov_SC06_029/0x8017EBC4:func_8012C1B8, ov_SC06_029/0x8017EBC4:sp +- `Sub_80181BA0`: 1 obj — ov_SC04_011/0x80181BA0:p + +### 474e1fa40ab6 +names: Actor_8017E550, Actor_8017E878 +cause: 2 names (156 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x8017E550(arg0), 0x8017E878(arg0); full list: tools/struct_twins.py --show 474e1fa40ab6 +- `Actor_8017E550`: 1 obj — 0x8017E550(arg0) +- `Actor_8017E878`: 1 obj — 0x8017E878(arg0) + +### 12fe36cd7e07 +names: Struct801E79D8, Struct801EF318 +cause: 2 names (16 B, all members placeholders) bound to 3 distinct objects, none bound to two names: 0x801E2498, 0x801E79D8, 0x801EF318; full list: tools/struct_twins.py --show 12fe36cd7e07 +- `Struct801E79D8`: 1 obj — 0x801E79D8 +- `Struct801EF318`: 2 obj — 0x801E2498, 0x801EF318 + +### c6e867e46e6f +names: V16x4, Vec4s16_8018423C +cause: 2 names (8 B, all members placeholders) bound to 7 distinct objects, none bound to two names: 0x80188984, ov_SC03_105/0x8018423C:out, ov_SC03_105/0x8018423C:pos, ov_SC03_105/0x8018423C:tmp, ov_SC03_105/0x8018423C:vec, ov_SC06_025/0x8017F664:a, ov_SC06_025/0x8017F664:b; full list: tools/struct_twins.py --show c6e867e46e6f +- `V16x4`: 3 obj — 0x80188984, ov_SC06_025/0x8017F664:a, ov_SC06_025/0x8017F664:b +- `Vec4s16_8018423C`: 4 obj — ov_SC03_105/0x8018423C:out, ov_SC03_105/0x8018423C:pos, ov_SC03_105/0x8018423C:tmp, ov_SC03_105/0x8018423C:vec + +### 64db0612c4d4 +names: Work_801832E0, Work_80183634 +cause: 2 names (236 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x801832E0(arg0), 0x80183634(arg0); full list: tools/struct_twins.py --show 64db0612c4d4 +- `Work_801832E0`: 1 obj — 0x801832E0(arg0) +- `Work_80183634`: 1 obj — 0x80183634(arg0) + +### cc89b444f3c5 +names: EntA, Ent_SC07_010_8017AE2C +cause: 2 names (256 B, all members placeholders) bound to 4 distinct objects, none bound to two names: ov_SC07_010/0x8017E5B8:func_8012C588, ov_SC07_010/0x8017E5B8:p, ov_SC07_010/0x8017E70C:func_8012C588, ov_SC07_010/0x8017E70C:p; full list: tools/struct_twins.py --show cc89b444f3c5 +- `EntA`: 2 obj — ov_SC07_010/0x8017E5B8:func_8012C588, ov_SC07_010/0x8017E5B8:p +- `Ent_SC07_010_8017AE2C`: 2 obj — ov_SC07_010/0x8017E70C:func_8012C588, ov_SC07_010/0x8017E70C:p + +### 2816963b1259 +names: Obj800D302C, Struct800D2FEC +cause: 2 names (44 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x800D2FEC(arg0), 0x800D302C(arg0); full list: tools/struct_twins.py --show 2816963b1259 +- `Obj800D302C`: 1 obj — 0x800D302C(arg0) +- `Struct800D2FEC`: 1 obj — 0x800D2FEC(arg0) + +### 2e57efa3d051 +names: Voice336A8, VoiceF80 +cause: 2 names (72 B, all members placeholders) bound to 4 distinct objects, none bound to two names: 0x800A4988, main/0x80030F80:e, main/0x80030F80:i, main/0x800336A8:vo; full list: tools/struct_twins.py --show 2e57efa3d051 +- `Voice336A8`: 2 obj — 0x800A4988, main/0x800336A8:vo +- `VoiceF80`: 2 obj — main/0x80030F80:e, main/0x80030F80:i + ## Dead kept diff --git a/phase-ends/current/logs/T5.1.c4.md b/phase-ends/current/logs/T5.1.c4.md new file mode 100644 index 0000000000..9f25d76303 --- /dev/null +++ b/phase-ends/current/logs/T5.1.c4.md @@ -0,0 +1,9 @@ +# T5.1.c4 — tools/struct_twins.py (tier-2 twin evidence) + docs/struct-twins.md ## Twins +- New `tools/struct_twins.py`: rows = dup_gating.tsv tier-2 + rows already listed (type_census.json, names from the doc; --write idempotent). Scans src/**, include/** (comments + #lines stripped, token walk) for bindings of each name: G D_ global (address key), L local/cast base (key = census Phase-35 body identity with tier-2 names, lifted-name hex/overlay suffixes, block externs masked), P param slot (binary+fn+idx; defs also by body identity), A same routine address + same local/slot across binaries, AG global used by same-address routine across binaries, F field (no evidence). Class: SDK (base lhash == a PsyQ def's layout hash via tc.walk_file + tc.Resolver) > SHARED (object bound to ≥2 names) > UNBOUND > SEPARATE. +- `--write` rewrites only `## Twins` (header, `## Dead kept` untouched); `--show `; `--census-dir` (struct_map cluster join via body_base_type.json when a --sites census is given). +- Result (census .run/P38/census @ 4e75840da + parallel coder's uncommitted src): 80 tier-2 rows · SEPARATE 19 · SHARED 55 · SDK 5 (SVECTOR 64533110def7:opaque, MATRIX 1146b31f105c, CdlLOC 4cc70dd1aeae:opaque, VECTOR 245ecac205e6:opaque, DVECTOR a0f7100333b8:opaque) · UNBOUND 1 (6ef8724a7464: F20T, Obj20_80178D40, Sub_80155800_80155800). +- SHARED list: `.venv/bin/python tools/struct_twins.py` prints each (lhash, names, shared object); e.g. 846e0acadd6a 0x801152B0, b85caab442ac:opaque 0x800AE620, 5b156b6f1fb8 func 0x8018AFD0:p (13 overlay copies), 416a00ee0fb0 table of routine 0x8013B7AC, 06c78d2eedc6 main/0x80030730:sp10 (Blk28 local cast to C24). +- Verified: `type_census.py --out-dir .run/P38/t51/c4census --no-cache --check-structs` → FAIL (expected, other gates) `dup_classes=92 … twins_stale=0`, `twins_listed=19`; json dup rows 111 (80 t2 + 31 t1) − 19 = 92. +- Known-true: 8944ddcbe454:opaque — Blob36 → 0x80078E50 = struct_map `AT:D_80078E50` (Unkstruct_80078E50); Blk36_800296F8 → D_80079204 (src/800.c:18190/18195, no struct_map node: no cast sites); SW20 → D_800D4B70 (md_MAIN_011.c:362). +- Rejected: identity = (tu,fn,var) → 26 SEPARATE incl. overlay copies of one routine (func_8018AFD0 P_*, func_8018B23C Rec_*; diffed: differ only in lifted type names / externs / pins); SDK by census class names → broke once tier-1 SVECTOR class folded. +- Gotchas: tier-2 set moves with the parallel folds (80→? ); expert re-runs `--write` after them. SDK literal rule catches 4-byte CdlLOC/DVECTOR layouts and the 12-byte non-PsyQ VECTOR (engine_types.h notes it lacks pad) — expert may want to exclude those. docs/struct-map.md is rewritten by every census run (not committed here). Scan ~60 s. diff --git a/tools/struct_twins.py b/tools/struct_twins.py new file mode 100644 index 0000000000..e9da45f514 --- /dev/null +++ b/tools/struct_twins.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""struct_twins.py — draft the evidence for tier-2 duplicate-layout classes (P38 T5.1.c4). + +`type_census.py --check-structs` gates every tier-2 row of /dup_gating.tsv unless docs/struct-twins.md lists it under +`## Twins` with the row's names and an addressed cause (G62: layout twins without evidence stay apart, "per type, not per layout"). +This tool joins each name to the objects it is bound to in the source, and classifies each class: + + SEPARATE every name is bound to >= 1 object and no object is bound to two names of the class -> a `### ` section + SHARED two or more names bound to the same object -> not listed; printed as a fold candidate with the shared object + SDK the base layout hash equals a Sony PsyQ struct's (PSYQ names defined in the canonical type files / include/, hashed + by the census's own parser and Resolver) -> not listed (folded onto the SDK name elsewhere) + UNBOUND some name has no bound object (dead / definition-only / macro-only) -> not listed, no per-name evidence exists +Rows: dup_gating.tsv's tier-2 rows plus the ones the current `## Twins` list takes out of it (type_census.json), so --write is +idempotent. + +Objects (the binding a use of the name establishes; scanned from src/**/*.{c,h} and include/**/*.h, comments stripped): + G a D_ global declared with the type or cast to it (`extern N D_x;`, `(N *)&D_x`), keyed by address alone (the + census's AT node identity, so the same address across overlay copies is one object) + L a local / cast base inside a function body, keyed (function identity, ident); the identity is the census's normalized + body text (func_/D_ masked) with tier-2 names, lifted-name address/overlay suffixes and block-scope externs masked too, + so the overlay copies of one routine bind one object + P a parameter slot: a prototype's keyed (binary, fn, arg index), a definition's also by function identity; R the return + A same routine address + same local / slot, across binaries (the copies edited apart by levers and pins) + AG a global used by the routine at one address, counted only across binaries (per-overlay instances of one table) + F a member of an aggregate (no address) +Each G/L object is joined to its struct_map cluster through /body_base_type.json when present (`--sites` run). + +Usage: + tools/struct_twins.py [--census-dir DIR] dry run: SEPARATE/SHARED/SDK/UNBOUND counts + the SHARED list + tools/struct_twins.py --write regenerate only the `## Twins` section of docs/struct-twins.md + tools/struct_twins.py --show every name's full object list (with struct_map clusters) +""" +import argparse +import collections +import hashlib +import json +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +CENSUS_DEFAULT = ".run/P38/census" +TWINS_DOC = REPO / "docs" / "struct-twins.md" +MAX_CITES = 8 +# Sony PsyQ (libgte / libgpu / libetc / libcd / libspu / libsnd / libapi) struct typedef names +PSYQ = set("""SVECTOR VECTOR CVECTOR DVECTOR MATRIX RECT RECT32 SPOL POL3 POL4 TMESH QMESH +DR_ENV DR_MODE DR_TPAGE DR_AREA DR_OFFSET DR_TWIN DR_STP DR_MOVE DR_LOAD DR_PRIO P_TAG P_CODE +POLY_F3 POLY_F4 POLY_FT3 POLY_FT4 POLY_G3 POLY_G4 POLY_GT3 POLY_GT4 LINE_F2 LINE_F3 LINE_F4 LINE_G2 LINE_G3 LINE_G4 +SPRT SPRT_8 SPRT_16 TILE TILE_1 TILE_8 TILE_16 DISPENV DRAWENV TIM_IMAGE KANJIFONT +CdlLOC CdlFILE CdlATV CdlFILTER SpuVolume SpuVoiceAttr SpuCommonAttr SpuReverbAttr SpuExtAttr SpuEnv SpuLVoiceAttr +SndVolume SndVolume2 SndRegisterAttr VabHdr ProgAtr VagAtr DIRENTRY EXEC XF_HDR""".split()) + +TOK = re.compile(r"[A-Za-z_]\w*|0x[0-9A-Fa-f]+|\d+|\S") +COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*|\"(?:\\.|[^\"\\\n])*\"|'(?:\\.|[^'\\\n])*'", re.S) +HEX8 = re.compile(r"(80[0-9A-Fa-f]{6})$") +DSYM = re.compile(r"^D_([0-9A-Fa-f]{8})$") +NORM = re.compile(r"(?:_?[0-9A-Fa-f]{8}|_(?:SC\d\d|MAIN)_\d{3}|_md)+") # lifted-name suffixes (address / overlay) +KEYWORDS = {"struct", "union", "const", "volatile", "extern", "static", "register", "typedef", "sizeof", "return", "if", + "while", "for", "switch", "do", "else"} +CTYPES = {"s8", "u8", "s16", "u16", "s32", "u32", "s64", "u64", "int", "char", "short", "long", "unsigned", "signed", "void", + "struct", "union", "const", "volatile"} + + +def load_rows(census): + """every tier-2 row: dup_gating.tsv's (full names) plus the rows the current `## Twins` list already takes out of it + (type_census.json keeps every row but caps names at 40; a listed row's names equal its `names:` line, as the census checked)""" + rows = [] + for line in (census / "dup_gating.tsv").read_text().splitlines()[1:]: + tier, lhash, size, n, names = line.split("\t") + if tier == "2": + rows.append(dict(lhash=lhash, size=int(size), names=names.split(","))) + seen = {r["lhash"] for r in rows} + sys.path.insert(0, str(REPO / "tools")) + import type_census as tc + listed, _ = tc.load_twins(TWINS_DOC.read_text() if TWINS_DOC.exists() else "") + for r in json.loads((census / "type_census.json").read_text())["dup_layout_classes"]: + if r["tier"] != 2 or r["lhash"] in seen: + continue + names = r["names"] if len(r["names"]) == r["n_names"] else sorted((listed.get(r["lhash"]) or {}).get("names") or ()) + if len(names) != r["n_names"]: + sys.exit(f"struct_twins: tier-2 row {r['lhash']} is neither in dup_gating.tsv nor listed with its {r['n_names']} names") + rows.append(dict(lhash=r["lhash"], size=r["size"], names=list(names))) + return rows + + +def psyq_layouts(): + """base lhash -> the PsyQ names defined with that layout in the canonical type files and include/ (the census's own + parser and resolver, so the hash is the one dup_gating.tsv's rows carry)""" + sys.path.insert(0, str(REPO / "tools")) + import type_census as tc + files = list(tc.CANON_HEADERS) + sorted(str(p.relative_to(REPO)) for p in (REPO / "include").rglob("*.h")) + defs, tds = [], [] + for rel in files: + r = tc.walk_file((REPO / rel).read_text(errors="replace"), rel) + defs += r["definitions"] + tds += r["typedef_aliases"] + g = {} + for d in defs: + for nm in d["names"] + ([d["tag"]] if d["tag"] else []): + g.setdefault(nm, d) + for a in tds: + g.setdefault(a["name"], dict(kind="alias", alias_of=a["alias_of"], alias_stars=a["alias_stars"], alias_dims=a["alias_dims"])) + out = collections.defaultdict(set) + for d in defs: + own = (set(d["names"]) | ({d["tag"]} if d["tag"] else set())) & PSYQ + if own and d["kind"] != "enum": + lay = tc.Resolver(g).layout_of_fields(d["fields"], d["kind"], packed=("packed" in d["attrs"])) + if lay: + out[tc.layout_hash(lay)] |= own + return out + + +def binary_of(rel): + p = rel.split("/") + if p[0] == "src" and len(p) > 2 and p[1].startswith(("ov_", "md_")): + return p[1] + if p[0] == "src" and len(p) == 2: + return "main" + if p[0] == "src" and p[1] == "shared" and len(p) > 3: + return "shared/" + p[2] + return p[0] if p[0] != "src" else "/".join(p[:2]) + + +def addr_of(fn): + m = HEX8.search(fn) + return "0x" + m.group(1).upper() if m else fn + + +def scan_file(rel, text, wanted, sink): + """append (name, key, cite, where) for every binding of a wanted name in one file""" + text = COMMENT.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), text) + text = re.sub(r"(?m)^[ \t]*#.*$", "", text) + toks, lines, ln, last = [], [], 1, 0 + for m in TOK.finditer(text): + ln += text.count("\n", last, m.start()) + last = m.start() + toks.append(m.group(0)) + lines.append(ln) + binary = binary_of(rel) + depth, fn, fn_depth = 0, None, None + parens = [] # stack of [callee or None, comma count, brace depth] + last_group = None # (index of '(' , index of ')') of the last top-level paren group + knr = None # (fn, [param idents]) between a K&R definition's `)` and its `{` + hdrpend, fnpend, body0 = [], [], 0 # sink indexes of a header's P bindings / a body's L+P bindings; body start token + used_by, s0 = collections.defaultdict(set), len(sink) # D_ symbol -> addresses of this file's functions using it + stmt = 0 + n = len(toks) + for i, t in enumerate(toks): + if t == "(": + if not parens and depth == 0: + knr = None + callee = toks[i - 1] if i and re.match(r"[A-Za-z_]\w*$", toks[i - 1]) and toks[i - 1] not in KEYWORDS else None + parens.append([callee, 0, i]) + continue + if t == ")": + if parens: + g = parens.pop() + if not parens: + last_group = (g[2], i) + nx = toks[i + 1] if i + 1 < len(toks) else "" + if depth == 0 and g[0] and re.match(r"[A-Za-z_]\w*$", nx) and nx != "__asm__": + knr = (g[0], [x for x in toks[g[2] + 1:i] if re.match(r"[A-Za-z_]\w*$", x)]) + continue + if t == "," and parens: + parens[-1][1] += 1 + continue + if t == "{": + if depth == 0 and last_group and last_group[1] == i - 1 and last_group[0] > 0: + fn, fn_depth = toks[last_group[0] - 1], 1 + elif depth == 0 and knr: + fn, fn_depth = knr[0], 1 + if depth == 0 and fn: + fnpend, hdrpend, body0 = hdrpend, [], i + knr = None + depth += 1 + stmt = i + 1 + continue + if t == "}": + depth -= 1 + if fn and depth < fn_depth: + # the census's function identity (Phase-35 normalized text: func_/D_ masked) with the tier-2 names, address / + # overlay suffixes of lifted names and block-scope extern lines masked too, so the copies of one routine across + # overlays (which differ only in their per-copy type names and extern lists) bind one object + body, x0 = [], body0 + while x0 <= i: + if toks[x0] == "extern": + while x0 <= i and toks[x0] != ";": + x0 += 1 + else: + body.append(toks[x0]) + x0 += 1 + norm = " ".join("@T" if x in wanted else "D_" if DSYM.match(x) else "func_" if x.startswith("func_") else NORM.sub("#", x) + for x in body) + h = "F:" + hashlib.sha1(norm.encode()).hexdigest()[:12] + for ix in fnpend: # a parameter also keeps its per-binary slot (its prototypes' key) + nm, key, cite, where = sink[ix] + if key[0] == "L": + sink[ix] = (nm, ("L", h, key[-1]), cite, where) + else: + sink.append((nm, ("P", h, key[-1]), cite, where)) + # the same routine edited differently per overlay (levers, pins): same address, same local / slot + sink.append((nm, ("A", addr_of(fn), key[0], key[-1]), cite, where)) + for x in body: + if DSYM.match(x): + used_by[x].add(addr_of(fn)) + fn, fnpend = None, [] + stmt = i + 1 + continue + if t == ";": + if depth == 0 and not parens and not knr: + hdrpend = [] + stmt = i + 1 + continue + if t not in wanted: + continue + if stmt < n and toks[stmt] == "typedef": + continue + j = i + 1 + stars = 0 + while j < n and toks[j] in ("*", "const", "volatile"): + stars += toks[j] == "*" + j += 1 + nxt = toks[j] if j < n else "" + k = i - 1 + while k >= 0 and toks[k] in ("struct", "union", "const", "volatile"): + k -= 1 + prev = toks[k] if k >= 0 else "" + where = f"{rel}:{lines[i]}" + if prev == "(" and nxt == ")" and stars: # cast (N *)expr + m = j + 1 + while m < n and (toks[m] in ("&", "*", "(", ")") or toks[m] in CTYPES): + m += 1 + op = toks[m] if m < n else "" + if DSYM.match(op): + sink.append((t, ("G", op), "0x" + op[2:].upper(), where)) + elif fn and re.match(r"[A-Za-z_]\w*$", op): + fnpend.append(len(sink)) + sink.append((t, ("L", rel, fn, op), f"{binary}/{addr_of(fn)}:{op}", where)) + continue + if re.match(r"[A-Za-z_]\w*$", nxt) and nxt not in KEYWORDS: # declaration N [*]ident + after = toks[j + 1] if j + 1 < n else "" + in_params = [p for p in parens if p[0]] + if after == "(" and not in_params: # N *fn(...): the return + sink.append((t, ("R", binary, nxt), f"{addr_of(nxt)}(ret)", where)) + elif in_params: + p = in_params[-1] + (hdrpend if depth == 0 else []).append(len(sink)) + sink.append((t, ("P", binary, p[0], p[1]), f"{addr_of(p[0])}(arg{p[1]})", where)) + elif depth == 0 and knr and nxt in knr[1]: + ix = knr[1].index(nxt) + hdrpend.append(len(sink)) + sink.append((t, ("P", binary, knr[0], ix), f"{addr_of(knr[0])}(arg{ix})", where)) + elif DSYM.match(nxt): + sink.append((t, ("G", nxt), "0x" + nxt[2:].upper(), where)) + elif fn: + fnpend.append(len(sink)) + sink.append((t, ("L", rel, fn, nxt), f"{binary}/{addr_of(fn)}:{nxt}", where)) + elif depth == 0: + sink.append((t, ("G", binary, nxt), nxt, where)) + else: + sink.append((t, ("F", rel, nxt), f"field {nxt}", where)) + continue + if nxt in (")", ",") and stars and parens and parens[-1][0]: # abstract declarator in a prototype + p = parens[-1] + sink.append((t, ("P", binary, p[0], p[1]), f"{addr_of(p[0])}(arg{p[1]})", where)) + # a global's using routine: the per-overlay instances of one routine's table bind one object ("AG", counted cross-binary) + for ix in range(s0, len(sink)): + nm, key, cite, where = sink[ix] + if key[0] == "G" and len(key) == 2: + for fa in sorted(used_by.get(key[1], ())): + sink.append((nm, ("AG", fa, binary), cite, where)) + + +def struct_map_join(census): + """(G key -> clusters, (tu, fn, ident) -> cluster) from body_base_type.json, or empty maps""" + p = census / "body_base_type.json" + g, loc = collections.defaultdict(set), {} + if not p.exists(): + return g, loc, False + for body, bases in json.loads(p.read_text()).items(): + tu, _, fn = body.partition("|") + for b, e in bases.items(): + cls, _, base = b.partition(":") + if cls in ("global", "gaddr") and DSYM.match(base): + g[base].add(e["type"]) + elif cls in ("local", "param"): + loc[(tu, addr_of(fn), base)] = e["type"] + return g, loc, True + + +def collect(census): + rows = load_rows(census) + wanted = {nm for r in rows for nm in r["names"]} + sink = [] + files = sorted(p for d in ("src", "include") for p in (REPO / d).rglob("*") if p.suffix in (".c", ".h") and p.is_file()) + for p in files: + rel = str(p.relative_to(REPO)) + if p.name.startswith(".cdecl"): + continue + text = p.read_text(errors="replace") + if not set(re.findall(r"[A-Za-z_]\w*", text)) & wanted: + continue + scan_file(rel, text, wanted, sink) + bind = collections.defaultdict(dict) # name -> key -> (cite, [where]) + for name, key, cite, where in sink: + e = bind[name].setdefault(key, [cite, []]) + e[1].append(where) + psyq = psyq_layouts() + out = [] + for r in rows: + base = r["lhash"].split(":")[0] + sdk = sorted(psyq.get(base, ())) + owners = collections.defaultdict(set) + for nm in r["names"]: + for key in bind.get(nm, {}): + if key[0] == "AG": + owners[key[:2]].add((nm, key[2])) + elif key[0] != "F": + owners[key].add((nm, None)) + shared = {k: {nm for nm, _ in v} for k, v in owners.items() + if len({nm for nm, _ in v}) > 1 and (k[0] != "AG" or len({b for _, b in v}) > 1)} + unbound = [nm for nm in r["names"] if not any(k[0] not in ("F", "AG") for k in bind.get(nm, {}))] + klass = "SDK" if sdk else "SHARED" if shared else "UNBOUND" if unbound else "SEPARATE" + out.append(dict(r, klass=klass, sdk=sdk, shared=shared, unbound=unbound, + bind={nm: bind.get(nm, {}) for nm in r["names"]})) + return out + + +def cites(objs, cap=MAX_CITES): + cs = sorted({c for k, (c, _) in objs.items() if k[0] != "F"}, key=lambda c: ("0x" not in c, c)) + return cs[:cap], len(cs) + + +def section(c): + names = sorted(c["names"]) + allc = sorted({ci for nm in names for k, (ci, _) in c["bind"][nm].items() if k[0] != "F"}) + addr = [x for x in allc if "0x" in x] + nobj = len(allc) + lines = [f"### {c['lhash']}", "names: " + ", ".join(names), + f"cause: {len(names)} names ({c['size']} B, all members placeholders) bound to {nobj} distinct objects, none bound to two " + f"names: {', '.join(addr[:MAX_CITES])}{' …' if len(addr) > MAX_CITES else ''}; " + f"full list: tools/struct_twins.py --show {c['lhash']}"] + for nm in names: + cs, total = cites(c["bind"][nm]) + more = f" (+{total - len(cs)})" if total > len(cs) else "" + lines.append(f"- `{nm}`: {total} obj — {', '.join(cs)}{more}") + return "\n".join(lines) + + +def write(classes): + text = TWINS_DOC.read_text() + m = re.search(r"(?m)^## Twins[ \t]*\n", text) + if not m: + sys.exit("struct_twins: no `## Twins` heading in docs/struct-twins.md") + rest = re.search(r"(?m)^## ", text[m.end():]) + end = m.end() + rest.start() if rest else len(text) + body = "\n\n".join(section(c) for c in classes if c["klass"] == "SEPARATE") + body = ("\n\n" + body + "\n\n") if body else "\n" + TWINS_DOC.write_text(text[:m.end()] + "\n" + body + text[end:]) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--census-dir", default=CENSUS_DEFAULT) + ap.add_argument("--write", action="store_true", help="regenerate the `## Twins` section of docs/struct-twins.md") + ap.add_argument("--show", metavar="LHASH", help="every name's objects for one class") + a = ap.parse_args() + census = REPO / a.census_dir + classes = collect(census) + if a.show: + c = next((c for c in classes if c["lhash"] == a.show), None) + if not c: + sys.exit(f"struct_twins: {a.show} is not a tier-2 row of {census}/dup_gating.tsv") + g, loc, have = struct_map_join(census) + print(f"{c['lhash']} {c['size']} B · {len(c['names'])} names · {c['klass']}" + ("" if have else " (no body_base_type.json: run the census with --sites for struct_map clusters)")) + for nm in sorted(c["names"]): + print(f"{nm}:") + for k, (ci, wh) in sorted(c["bind"][nm].items(), key=lambda kv: kv[1][0]): + lk = (wh[0].rsplit(":", 1)[0], ci.split("/")[-1].rsplit(":", 1)[0], k[-1]) if k[0] == "L" else None + sm = sorted(g.get(k[1], ())) if k[0] == "G" else [loc[lk]] if lk in loc else [] + print(f" {ci:<28} {k[0]} {wh[0]}{f' (+{len(wh) - 1})' if len(wh) > 1 else ''}" + f"{' struct_map ' + ','.join(sm[:4]) if sm else ''}") + return + cnt = collections.Counter(c["klass"] for c in classes) + print(f"struct_twins: {len(classes)} tier-2 rows · SEPARATE {cnt['SEPARATE']} · SHARED {cnt['SHARED']} · SDK {cnt['SDK']} · " + f"UNBOUND {cnt['UNBOUND']}") + for c in classes: + if c["klass"] == "SHARED": + def cite_of(k, nm): + return next(ci for kk, (ci, _) in sorted(c["bind"][nm].items()) if kk[:len(k)] == k) + k, v = min(c["shared"].items(), key=lambda kv: (-len(kv[1]), cite_of(kv[0], min(kv[1])))) + ci = cite_of(k, min(v)) + (f" (routine {k[1]})" if k[0] in ("A", "AG") else "") + print(f" SHARED {c['lhash']} ({len(c['names'])} names, {len(c['shared'])} shared objects): e.g. {ci} <- {', '.join(sorted(v)[:6])}" + f"{' …' if len(v) > 6 else ''}") + elif c["klass"] == "SDK": + print(f" SDK {c['lhash']} ({len(c['names'])} names) layout of {', '.join(c['sdk'])}") + elif c["klass"] == "UNBOUND": + print(f" UNBOUND {c['lhash']} ({len(c['names'])} names): {', '.join(c['unbound'][:6])}{' …' if len(c['unbound']) > 6 else ''}") + if a.write: + write(classes) + print(f"struct_twins: wrote {cnt['SEPARATE']} `### ` sections to docs/struct-twins.md ## Twins") + + +if __name__ == "__main__": + main()