mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-04 00:20:10 -04:00
T5.1.c4: struct_twins.py: tier-2 twin evidence; ## Twins lists 19 SEPARATE classes
This commit is contained in:
Drew T
@@ -10,4 +10,127 @@
|
||||
|
||||
## Twins
|
||||
|
||||
<!-- generated by tools/struct_twins.py --write; regenerate, do not hand-edit -->
|
||||
|
||||
### f1dcb3a42fc8
|
||||
names: Rec_801825D8, Rec_80184FF8, S8012C658, S8017FF9C_sp
|
||||
cause: 4 names (20 B, all members placeholders) bound to 5 distinct objects, none bound to two names: ov_SC02_011/0x801890C8:sp, ov_SC04_003/0x8017FF9C:sp, ov_SC06_029/0x801825D8:rec, ov_SC06_029/0x80184FF8:rec, shared/ov/0x8012C658:sp; full list: tools/struct_twins.py --show f1dcb3a42fc8
|
||||
- `Rec_801825D8`: 1 obj — ov_SC06_029/0x801825D8:rec
|
||||
- `Rec_80184FF8`: 1 obj — ov_SC06_029/0x80184FF8:rec
|
||||
- `S8012C658`: 2 obj — ov_SC02_011/0x801890C8:sp, shared/ov/0x8012C658:sp
|
||||
- `S8017FF9C_sp`: 1 obj — ov_SC04_003/0x8017FF9C:sp
|
||||
|
||||
### b519bad67107:opaque
|
||||
names: Local_8017EE34, Local_80181C84, Local_801870E8
|
||||
cause: 3 names (20 B, all members placeholders) bound to 4 distinct objects, none bound to two names: ov_SC03_105/0x8017EE34:local, ov_SC03_105/0x80181C84:local, ov_SC03_105/0x80183D38:local, ov_SC03_105/0x801870E8:local; full list: tools/struct_twins.py --show b519bad67107:opaque
|
||||
- `Local_8017EE34`: 2 obj — ov_SC03_105/0x8017EE34:local, ov_SC03_105/0x80183D38:local
|
||||
- `Local_80181C84`: 1 obj — ov_SC03_105/0x80181C84:local
|
||||
- `Local_801870E8`: 1 obj — ov_SC03_105/0x801870E8:local
|
||||
|
||||
### c80fac242f04
|
||||
names: Mat801A8E34, Mat_80183C9C, Mtx_A8CE0
|
||||
cause: 3 names (32 B, all members placeholders) bound to 4 distinct objects, none bound to two names: md_SC07_004/0x801A8CE0:mtx, md_SC07_004/0x801A8E34:buf, ov_SC06_000/0x80183C9C:mat; full list: tools/struct_twins.py --show c80fac242f04
|
||||
- `Mat801A8E34`: 1 obj — md_SC07_004/0x801A8E34:buf
|
||||
- `Mat_80183C9C`: 1 obj — ov_SC06_000/0x80183C9C:mat
|
||||
- `Mtx_A8CE0`: 2 obj — md_SC07_004/0x801A8CE0:mtx, __attribute__(ret)
|
||||
|
||||
### 9ba0ae408dd4
|
||||
names: Drv_8017E330, Drv_8017E550, Drv_8017E878
|
||||
cause: 3 names (56 B, all members placeholders) bound to 6 distinct objects, none bound to two names: 0x801F80E4, ov_SC06_006/0x8017E330:p, ov_SC06_006/0x8017E550:a0, ov_SC06_006/0x8017E550:p, ov_SC06_006/0x8017E878:a0, ov_SC06_006/0x8017E878:p; full list: tools/struct_twins.py --show 9ba0ae408dd4
|
||||
- `Drv_8017E330`: 2 obj — 0x801F80E4, ov_SC06_006/0x8017E330:p
|
||||
- `Drv_8017E550`: 2 obj — ov_SC06_006/0x8017E550:a0, ov_SC06_006/0x8017E550:p
|
||||
- `Drv_8017E878`: 2 obj — ov_SC06_006/0x8017E878:a0, ov_SC06_006/0x8017E878:p
|
||||
|
||||
### 8944ddcbe454:opaque
|
||||
names: Blk36_800296F8, Blob36, SW20
|
||||
cause: 3 names (36 B, all members placeholders) bound to 6 distinct objects, none bound to two names: 0x800298BC(arg0), 0x80078E50, 0x80079204, 0x800D4B70, main/0x800296F8:arg0, src/resident/0x800D185C:arg0; full list: tools/struct_twins.py --show 8944ddcbe454:opaque
|
||||
- `Blk36_800296F8`: 3 obj — 0x800298BC(arg0), 0x80079204, main/0x800296F8:arg0
|
||||
- `Blob36`: 2 obj — 0x80078E50, src/resident/0x800D185C:arg0
|
||||
- `SW20`: 1 obj — 0x800D4B70
|
||||
|
||||
### c2ba48790859
|
||||
names: W8, W8_, func_8002EE64_b
|
||||
cause: 3 names (1 B, all members placeholders) bound to 4 distinct objects, none bound to two names: 0x80076242, 0x80076243, main/0x8002EE64:b; full list: tools/struct_twins.py --show c2ba48790859
|
||||
- `W8`: 2 obj — 0x80076242, 0x80076243
|
||||
- `W8_`: 1 obj — D_80076242_
|
||||
- `func_8002EE64_b`: 1 obj — main/0x8002EE64:b
|
||||
|
||||
### ac91ae2d8aa0:opaque
|
||||
names: S800D22E4, vec
|
||||
cause: 2 names (28 B, all members placeholders) bound to 77 distinct objects, none bound to two names: 0x800D22E4(arg0), 0x8012931C(arg0), 0x8012AD80(arg0), 0x80185238(arg0), 0x801879E8(arg0), md_SC07_004/0x801A72DC:a0, md_SC07_004/0x801ABC8C:a0, md_SC07_004/0x801AC234:a0 …; full list: tools/struct_twins.py --show ac91ae2d8aa0:opaque
|
||||
- `S800D22E4`: 1 obj — 0x800D22E4(arg0)
|
||||
- `vec`: 76 obj — 0x8012931C(arg0), 0x8012AD80(arg0), 0x80185238(arg0), 0x801879E8(arg0), md_SC07_004/0x801A72DC:a0, md_SC07_004/0x801ABC8C:a0, md_SC07_004/0x801AC234:a0, md_SC07_004/0x801AC940:a1 (+68)
|
||||
|
||||
### 82c6e3650290:opaque
|
||||
names: Gte8_8017F2FC, Rec_80180808
|
||||
cause: 2 names (8 B, all members placeholders) bound to 3 distinct objects, none bound to two names: 0x801B1B50, ov_SC02_021/0x8017F2FC:sp10, ov_SC06_025/0x80180808:rec; full list: tools/struct_twins.py --show 82c6e3650290:opaque
|
||||
- `Gte8_8017F2FC`: 1 obj — ov_SC02_021/0x8017F2FC:sp10
|
||||
- `Rec_80180808`: 2 obj — 0x801B1B50, ov_SC06_025/0x80180808:rec
|
||||
|
||||
### aa66418d079c:opaque
|
||||
names: Blk16_80126940, Pos16_8018743C
|
||||
cause: 2 names (16 B, all members placeholders) bound to 3 distinct objects, none bound to two names: ov_SC06_000/0x80181AB4:sp10, ov_SC06_029/0x8018743C:from, ov_SC06_029/0x8018743C:to; full list: tools/struct_twins.py --show aa66418d079c:opaque
|
||||
- `Blk16_80126940`: 1 obj — ov_SC06_000/0x80181AB4:sp10
|
||||
- `Pos16_8018743C`: 2 obj — ov_SC06_029/0x8018743C:from, ov_SC06_029/0x8018743C:to
|
||||
|
||||
### 1dd90dc86138
|
||||
names: Ent80183F28, Ent_80184058
|
||||
cause: 2 names (236 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x80183F28(arg0), 0x80184058(arg0); full list: tools/struct_twins.py --show 1dd90dc86138
|
||||
- `Ent80183F28`: 1 obj — 0x80183F28(arg0)
|
||||
- `Ent_80184058`: 1 obj — 0x80184058(arg0)
|
||||
|
||||
### d60a32e09c91
|
||||
names: P8, S72
|
||||
cause: 2 names (8 B, all members placeholders) bound to 5 distinct objects, none bound to two names: 0x80185D28, 0x80185D2A, 0x80185D2C, 0x80185D2E, 0x80185D72; full list: tools/struct_twins.py --show d60a32e09c91
|
||||
- `P8`: 4 obj — 0x80185D28, 0x80185D2A, 0x80185D2C, 0x80185D2E
|
||||
- `S72`: 1 obj — 0x80185D72
|
||||
|
||||
### 08944be666a1
|
||||
names: Sub8017EBC4, Sub_80181BA0
|
||||
cause: 2 names (20 B, all members placeholders) bound to 3 distinct objects, none bound to two names: ov_SC04_011/0x80181BA0:p, ov_SC06_029/0x8017EBC4:func_8012C1B8, ov_SC06_029/0x8017EBC4:sp; full list: tools/struct_twins.py --show 08944be666a1
|
||||
- `Sub8017EBC4`: 2 obj — ov_SC06_029/0x8017EBC4:func_8012C1B8, ov_SC06_029/0x8017EBC4:sp
|
||||
- `Sub_80181BA0`: 1 obj — ov_SC04_011/0x80181BA0:p
|
||||
|
||||
### 474e1fa40ab6
|
||||
names: Actor_8017E550, Actor_8017E878
|
||||
cause: 2 names (156 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x8017E550(arg0), 0x8017E878(arg0); full list: tools/struct_twins.py --show 474e1fa40ab6
|
||||
- `Actor_8017E550`: 1 obj — 0x8017E550(arg0)
|
||||
- `Actor_8017E878`: 1 obj — 0x8017E878(arg0)
|
||||
|
||||
### 12fe36cd7e07
|
||||
names: Struct801E79D8, Struct801EF318
|
||||
cause: 2 names (16 B, all members placeholders) bound to 3 distinct objects, none bound to two names: 0x801E2498, 0x801E79D8, 0x801EF318; full list: tools/struct_twins.py --show 12fe36cd7e07
|
||||
- `Struct801E79D8`: 1 obj — 0x801E79D8
|
||||
- `Struct801EF318`: 2 obj — 0x801E2498, 0x801EF318
|
||||
|
||||
### c6e867e46e6f
|
||||
names: V16x4, Vec4s16_8018423C
|
||||
cause: 2 names (8 B, all members placeholders) bound to 7 distinct objects, none bound to two names: 0x80188984, ov_SC03_105/0x8018423C:out, ov_SC03_105/0x8018423C:pos, ov_SC03_105/0x8018423C:tmp, ov_SC03_105/0x8018423C:vec, ov_SC06_025/0x8017F664:a, ov_SC06_025/0x8017F664:b; full list: tools/struct_twins.py --show c6e867e46e6f
|
||||
- `V16x4`: 3 obj — 0x80188984, ov_SC06_025/0x8017F664:a, ov_SC06_025/0x8017F664:b
|
||||
- `Vec4s16_8018423C`: 4 obj — ov_SC03_105/0x8018423C:out, ov_SC03_105/0x8018423C:pos, ov_SC03_105/0x8018423C:tmp, ov_SC03_105/0x8018423C:vec
|
||||
|
||||
### 64db0612c4d4
|
||||
names: Work_801832E0, Work_80183634
|
||||
cause: 2 names (236 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x801832E0(arg0), 0x80183634(arg0); full list: tools/struct_twins.py --show 64db0612c4d4
|
||||
- `Work_801832E0`: 1 obj — 0x801832E0(arg0)
|
||||
- `Work_80183634`: 1 obj — 0x80183634(arg0)
|
||||
|
||||
### cc89b444f3c5
|
||||
names: EntA, Ent_SC07_010_8017AE2C
|
||||
cause: 2 names (256 B, all members placeholders) bound to 4 distinct objects, none bound to two names: ov_SC07_010/0x8017E5B8:func_8012C588, ov_SC07_010/0x8017E5B8:p, ov_SC07_010/0x8017E70C:func_8012C588, ov_SC07_010/0x8017E70C:p; full list: tools/struct_twins.py --show cc89b444f3c5
|
||||
- `EntA`: 2 obj — ov_SC07_010/0x8017E5B8:func_8012C588, ov_SC07_010/0x8017E5B8:p
|
||||
- `Ent_SC07_010_8017AE2C`: 2 obj — ov_SC07_010/0x8017E70C:func_8012C588, ov_SC07_010/0x8017E70C:p
|
||||
|
||||
### 2816963b1259
|
||||
names: Obj800D302C, Struct800D2FEC
|
||||
cause: 2 names (44 B, all members placeholders) bound to 2 distinct objects, none bound to two names: 0x800D2FEC(arg0), 0x800D302C(arg0); full list: tools/struct_twins.py --show 2816963b1259
|
||||
- `Obj800D302C`: 1 obj — 0x800D302C(arg0)
|
||||
- `Struct800D2FEC`: 1 obj — 0x800D2FEC(arg0)
|
||||
|
||||
### 2e57efa3d051
|
||||
names: Voice336A8, VoiceF80
|
||||
cause: 2 names (72 B, all members placeholders) bound to 4 distinct objects, none bound to two names: 0x800A4988, main/0x80030F80:e, main/0x80030F80:i, main/0x800336A8:vo; full list: tools/struct_twins.py --show 2e57efa3d051
|
||||
- `Voice336A8`: 2 obj — 0x800A4988, main/0x800336A8:vo
|
||||
- `VoiceF80`: 2 obj — main/0x80030F80:e, main/0x80030F80:i
|
||||
|
||||
## Dead kept
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# T5.1.c4 — tools/struct_twins.py (tier-2 twin evidence) + docs/struct-twins.md ## Twins
|
||||
- New `tools/struct_twins.py`: rows = dup_gating.tsv tier-2 + rows already listed (type_census.json, names from the doc; --write idempotent). Scans src/**, include/** (comments + #lines stripped, token walk) for bindings of each name: G D_ global (address key), L local/cast base (key = census Phase-35 body identity with tier-2 names, lifted-name hex/overlay suffixes, block externs masked), P param slot (binary+fn+idx; defs also by body identity), A same routine address + same local/slot across binaries, AG global used by same-address routine across binaries, F field (no evidence). Class: SDK (base lhash == a PsyQ def's layout hash via tc.walk_file + tc.Resolver) > SHARED (object bound to ≥2 names) > UNBOUND > SEPARATE.
|
||||
- `--write` rewrites only `## Twins` (header, `## Dead kept` untouched); `--show <lhash>`; `--census-dir` (struct_map cluster join via body_base_type.json when a --sites census is given).
|
||||
- Result (census .run/P38/census @ 4e75840da + parallel coder's uncommitted src): 80 tier-2 rows · SEPARATE 19 · SHARED 55 · SDK 5 (SVECTOR 64533110def7:opaque, MATRIX 1146b31f105c, CdlLOC 4cc70dd1aeae:opaque, VECTOR 245ecac205e6:opaque, DVECTOR a0f7100333b8:opaque) · UNBOUND 1 (6ef8724a7464: F20T, Obj20_80178D40, Sub_80155800_80155800).
|
||||
- SHARED list: `.venv/bin/python tools/struct_twins.py` prints each (lhash, names, shared object); e.g. 846e0acadd6a 0x801152B0, b85caab442ac:opaque 0x800AE620, 5b156b6f1fb8 func 0x8018AFD0:p (13 overlay copies), 416a00ee0fb0 table of routine 0x8013B7AC, 06c78d2eedc6 main/0x80030730:sp10 (Blk28 local cast to C24).
|
||||
- Verified: `type_census.py --out-dir .run/P38/t51/c4census --no-cache --check-structs` → FAIL (expected, other gates) `dup_classes=92 … twins_stale=0`, `twins_listed=19`; json dup rows 111 (80 t2 + 31 t1) − 19 = 92.
|
||||
- Known-true: 8944ddcbe454:opaque — Blob36 → 0x80078E50 = struct_map `AT:D_80078E50` (Unkstruct_80078E50); Blk36_800296F8 → D_80079204 (src/800.c:18190/18195, no struct_map node: no cast sites); SW20 → D_800D4B70 (md_MAIN_011.c:362).
|
||||
- Rejected: identity = (tu,fn,var) → 26 SEPARATE incl. overlay copies of one routine (func_8018AFD0 P_*, func_8018B23C Rec_*; diffed: differ only in lifted type names / externs / pins); SDK by census class names → broke once tier-1 SVECTOR class folded.
|
||||
- Gotchas: tier-2 set moves with the parallel folds (80→? ); expert re-runs `--write` after them. SDK literal rule catches 4-byte CdlLOC/DVECTOR layouts and the 12-byte non-PsyQ VECTOR (engine_types.h notes it lacks pad) — expert may want to exclude those. docs/struct-map.md is rewritten by every census run (not committed here). Scan ~60 s.
|
||||
@@ -0,0 +1,405 @@
|
||||
#!/usr/bin/env python3
|
||||
"""struct_twins.py — draft the evidence for tier-2 duplicate-layout classes (P38 T5.1.c4).
|
||||
|
||||
`type_census.py --check-structs` gates every tier-2 row of <census>/dup_gating.tsv unless docs/struct-twins.md lists it under
|
||||
`## Twins` with the row's names and an addressed cause (G62: layout twins without evidence stay apart, "per type, not per layout").
|
||||
This tool joins each name to the objects it is bound to in the source, and classifies each class:
|
||||
|
||||
SEPARATE every name is bound to >= 1 object and no object is bound to two names of the class -> a `### <lhash>` section
|
||||
SHARED two or more names bound to the same object -> not listed; printed as a fold candidate with the shared object
|
||||
SDK the base layout hash equals a Sony PsyQ struct's (PSYQ names defined in the canonical type files / include/, hashed
|
||||
by the census's own parser and Resolver) -> not listed (folded onto the SDK name elsewhere)
|
||||
UNBOUND some name has no bound object (dead / definition-only / macro-only) -> not listed, no per-name evidence exists
|
||||
Rows: dup_gating.tsv's tier-2 rows plus the ones the current `## Twins` list takes out of it (type_census.json), so --write is
|
||||
idempotent.
|
||||
|
||||
Objects (the binding a use of the name establishes; scanned from src/**/*.{c,h} and include/**/*.h, comments stripped):
|
||||
G a D_<addr> global declared with the type or cast to it (`extern N D_x;`, `(N *)&D_x`), keyed by address alone (the
|
||||
census's AT node identity, so the same address across overlay copies is one object)
|
||||
L a local / cast base inside a function body, keyed (function identity, ident); the identity is the census's normalized
|
||||
body text (func_/D_ masked) with tier-2 names, lifted-name address/overlay suffixes and block-scope externs masked too,
|
||||
so the overlay copies of one routine bind one object
|
||||
P a parameter slot: a prototype's keyed (binary, fn, arg index), a definition's also by function identity; R the return
|
||||
A same routine address + same local / slot, across binaries (the copies edited apart by levers and pins)
|
||||
AG a global used by the routine at one address, counted only across binaries (per-overlay instances of one table)
|
||||
F a member of an aggregate (no address)
|
||||
Each G/L object is joined to its struct_map cluster through <census>/body_base_type.json when present (`--sites` run).
|
||||
|
||||
Usage:
|
||||
tools/struct_twins.py [--census-dir DIR] dry run: SEPARATE/SHARED/SDK/UNBOUND counts + the SHARED list
|
||||
tools/struct_twins.py --write regenerate only the `## Twins` section of docs/struct-twins.md
|
||||
tools/struct_twins.py --show <lhash> every name's full object list (with struct_map clusters)
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
CENSUS_DEFAULT = ".run/P38/census"
|
||||
TWINS_DOC = REPO / "docs" / "struct-twins.md"
|
||||
MAX_CITES = 8
|
||||
# Sony PsyQ (libgte / libgpu / libetc / libcd / libspu / libsnd / libapi) struct typedef names
|
||||
PSYQ = set("""SVECTOR VECTOR CVECTOR DVECTOR MATRIX RECT RECT32 SPOL POL3 POL4 TMESH QMESH
|
||||
DR_ENV DR_MODE DR_TPAGE DR_AREA DR_OFFSET DR_TWIN DR_STP DR_MOVE DR_LOAD DR_PRIO P_TAG P_CODE
|
||||
POLY_F3 POLY_F4 POLY_FT3 POLY_FT4 POLY_G3 POLY_G4 POLY_GT3 POLY_GT4 LINE_F2 LINE_F3 LINE_F4 LINE_G2 LINE_G3 LINE_G4
|
||||
SPRT SPRT_8 SPRT_16 TILE TILE_1 TILE_8 TILE_16 DISPENV DRAWENV TIM_IMAGE KANJIFONT
|
||||
CdlLOC CdlFILE CdlATV CdlFILTER SpuVolume SpuVoiceAttr SpuCommonAttr SpuReverbAttr SpuExtAttr SpuEnv SpuLVoiceAttr
|
||||
SndVolume SndVolume2 SndRegisterAttr VabHdr ProgAtr VagAtr DIRENTRY EXEC XF_HDR""".split())
|
||||
|
||||
TOK = re.compile(r"[A-Za-z_]\w*|0x[0-9A-Fa-f]+|\d+|\S")
|
||||
COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*|\"(?:\\.|[^\"\\\n])*\"|'(?:\\.|[^'\\\n])*'", re.S)
|
||||
HEX8 = re.compile(r"(80[0-9A-Fa-f]{6})$")
|
||||
DSYM = re.compile(r"^D_([0-9A-Fa-f]{8})$")
|
||||
NORM = re.compile(r"(?:_?[0-9A-Fa-f]{8}|_(?:SC\d\d|MAIN)_\d{3}|_md)+") # lifted-name suffixes (address / overlay)
|
||||
KEYWORDS = {"struct", "union", "const", "volatile", "extern", "static", "register", "typedef", "sizeof", "return", "if",
|
||||
"while", "for", "switch", "do", "else"}
|
||||
CTYPES = {"s8", "u8", "s16", "u16", "s32", "u32", "s64", "u64", "int", "char", "short", "long", "unsigned", "signed", "void",
|
||||
"struct", "union", "const", "volatile"}
|
||||
|
||||
|
||||
def load_rows(census):
|
||||
"""every tier-2 row: dup_gating.tsv's (full names) plus the rows the current `## Twins` list already takes out of it
|
||||
(type_census.json keeps every row but caps names at 40; a listed row's names equal its `names:` line, as the census checked)"""
|
||||
rows = []
|
||||
for line in (census / "dup_gating.tsv").read_text().splitlines()[1:]:
|
||||
tier, lhash, size, n, names = line.split("\t")
|
||||
if tier == "2":
|
||||
rows.append(dict(lhash=lhash, size=int(size), names=names.split(",")))
|
||||
seen = {r["lhash"] for r in rows}
|
||||
sys.path.insert(0, str(REPO / "tools"))
|
||||
import type_census as tc
|
||||
listed, _ = tc.load_twins(TWINS_DOC.read_text() if TWINS_DOC.exists() else "")
|
||||
for r in json.loads((census / "type_census.json").read_text())["dup_layout_classes"]:
|
||||
if r["tier"] != 2 or r["lhash"] in seen:
|
||||
continue
|
||||
names = r["names"] if len(r["names"]) == r["n_names"] else sorted((listed.get(r["lhash"]) or {}).get("names") or ())
|
||||
if len(names) != r["n_names"]:
|
||||
sys.exit(f"struct_twins: tier-2 row {r['lhash']} is neither in dup_gating.tsv nor listed with its {r['n_names']} names")
|
||||
rows.append(dict(lhash=r["lhash"], size=r["size"], names=list(names)))
|
||||
return rows
|
||||
|
||||
|
||||
def psyq_layouts():
|
||||
"""base lhash -> the PsyQ names defined with that layout in the canonical type files and include/ (the census's own
|
||||
parser and resolver, so the hash is the one dup_gating.tsv's rows carry)"""
|
||||
sys.path.insert(0, str(REPO / "tools"))
|
||||
import type_census as tc
|
||||
files = list(tc.CANON_HEADERS) + sorted(str(p.relative_to(REPO)) for p in (REPO / "include").rglob("*.h"))
|
||||
defs, tds = [], []
|
||||
for rel in files:
|
||||
r = tc.walk_file((REPO / rel).read_text(errors="replace"), rel)
|
||||
defs += r["definitions"]
|
||||
tds += r["typedef_aliases"]
|
||||
g = {}
|
||||
for d in defs:
|
||||
for nm in d["names"] + ([d["tag"]] if d["tag"] else []):
|
||||
g.setdefault(nm, d)
|
||||
for a in tds:
|
||||
g.setdefault(a["name"], dict(kind="alias", alias_of=a["alias_of"], alias_stars=a["alias_stars"], alias_dims=a["alias_dims"]))
|
||||
out = collections.defaultdict(set)
|
||||
for d in defs:
|
||||
own = (set(d["names"]) | ({d["tag"]} if d["tag"] else set())) & PSYQ
|
||||
if own and d["kind"] != "enum":
|
||||
lay = tc.Resolver(g).layout_of_fields(d["fields"], d["kind"], packed=("packed" in d["attrs"]))
|
||||
if lay:
|
||||
out[tc.layout_hash(lay)] |= own
|
||||
return out
|
||||
|
||||
|
||||
def binary_of(rel):
|
||||
p = rel.split("/")
|
||||
if p[0] == "src" and len(p) > 2 and p[1].startswith(("ov_", "md_")):
|
||||
return p[1]
|
||||
if p[0] == "src" and len(p) == 2:
|
||||
return "main"
|
||||
if p[0] == "src" and p[1] == "shared" and len(p) > 3:
|
||||
return "shared/" + p[2]
|
||||
return p[0] if p[0] != "src" else "/".join(p[:2])
|
||||
|
||||
|
||||
def addr_of(fn):
|
||||
m = HEX8.search(fn)
|
||||
return "0x" + m.group(1).upper() if m else fn
|
||||
|
||||
|
||||
def scan_file(rel, text, wanted, sink):
|
||||
"""append (name, key, cite, where) for every binding of a wanted name in one file"""
|
||||
text = COMMENT.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), text)
|
||||
text = re.sub(r"(?m)^[ \t]*#.*$", "", text)
|
||||
toks, lines, ln, last = [], [], 1, 0
|
||||
for m in TOK.finditer(text):
|
||||
ln += text.count("\n", last, m.start())
|
||||
last = m.start()
|
||||
toks.append(m.group(0))
|
||||
lines.append(ln)
|
||||
binary = binary_of(rel)
|
||||
depth, fn, fn_depth = 0, None, None
|
||||
parens = [] # stack of [callee or None, comma count, brace depth]
|
||||
last_group = None # (index of '(' , index of ')') of the last top-level paren group
|
||||
knr = None # (fn, [param idents]) between a K&R definition's `)` and its `{`
|
||||
hdrpend, fnpend, body0 = [], [], 0 # sink indexes of a header's P bindings / a body's L+P bindings; body start token
|
||||
used_by, s0 = collections.defaultdict(set), len(sink) # D_ symbol -> addresses of this file's functions using it
|
||||
stmt = 0
|
||||
n = len(toks)
|
||||
for i, t in enumerate(toks):
|
||||
if t == "(":
|
||||
if not parens and depth == 0:
|
||||
knr = None
|
||||
callee = toks[i - 1] if i and re.match(r"[A-Za-z_]\w*$", toks[i - 1]) and toks[i - 1] not in KEYWORDS else None
|
||||
parens.append([callee, 0, i])
|
||||
continue
|
||||
if t == ")":
|
||||
if parens:
|
||||
g = parens.pop()
|
||||
if not parens:
|
||||
last_group = (g[2], i)
|
||||
nx = toks[i + 1] if i + 1 < len(toks) else ""
|
||||
if depth == 0 and g[0] and re.match(r"[A-Za-z_]\w*$", nx) and nx != "__asm__":
|
||||
knr = (g[0], [x for x in toks[g[2] + 1:i] if re.match(r"[A-Za-z_]\w*$", x)])
|
||||
continue
|
||||
if t == "," and parens:
|
||||
parens[-1][1] += 1
|
||||
continue
|
||||
if t == "{":
|
||||
if depth == 0 and last_group and last_group[1] == i - 1 and last_group[0] > 0:
|
||||
fn, fn_depth = toks[last_group[0] - 1], 1
|
||||
elif depth == 0 and knr:
|
||||
fn, fn_depth = knr[0], 1
|
||||
if depth == 0 and fn:
|
||||
fnpend, hdrpend, body0 = hdrpend, [], i
|
||||
knr = None
|
||||
depth += 1
|
||||
stmt = i + 1
|
||||
continue
|
||||
if t == "}":
|
||||
depth -= 1
|
||||
if fn and depth < fn_depth:
|
||||
# the census's function identity (Phase-35 normalized text: func_/D_ masked) with the tier-2 names, address /
|
||||
# overlay suffixes of lifted names and block-scope extern lines masked too, so the copies of one routine across
|
||||
# overlays (which differ only in their per-copy type names and extern lists) bind one object
|
||||
body, x0 = [], body0
|
||||
while x0 <= i:
|
||||
if toks[x0] == "extern":
|
||||
while x0 <= i and toks[x0] != ";":
|
||||
x0 += 1
|
||||
else:
|
||||
body.append(toks[x0])
|
||||
x0 += 1
|
||||
norm = " ".join("@T" if x in wanted else "D_" if DSYM.match(x) else "func_" if x.startswith("func_") else NORM.sub("#", x)
|
||||
for x in body)
|
||||
h = "F:" + hashlib.sha1(norm.encode()).hexdigest()[:12]
|
||||
for ix in fnpend: # a parameter also keeps its per-binary slot (its prototypes' key)
|
||||
nm, key, cite, where = sink[ix]
|
||||
if key[0] == "L":
|
||||
sink[ix] = (nm, ("L", h, key[-1]), cite, where)
|
||||
else:
|
||||
sink.append((nm, ("P", h, key[-1]), cite, where))
|
||||
# the same routine edited differently per overlay (levers, pins): same address, same local / slot
|
||||
sink.append((nm, ("A", addr_of(fn), key[0], key[-1]), cite, where))
|
||||
for x in body:
|
||||
if DSYM.match(x):
|
||||
used_by[x].add(addr_of(fn))
|
||||
fn, fnpend = None, []
|
||||
stmt = i + 1
|
||||
continue
|
||||
if t == ";":
|
||||
if depth == 0 and not parens and not knr:
|
||||
hdrpend = []
|
||||
stmt = i + 1
|
||||
continue
|
||||
if t not in wanted:
|
||||
continue
|
||||
if stmt < n and toks[stmt] == "typedef":
|
||||
continue
|
||||
j = i + 1
|
||||
stars = 0
|
||||
while j < n and toks[j] in ("*", "const", "volatile"):
|
||||
stars += toks[j] == "*"
|
||||
j += 1
|
||||
nxt = toks[j] if j < n else ""
|
||||
k = i - 1
|
||||
while k >= 0 and toks[k] in ("struct", "union", "const", "volatile"):
|
||||
k -= 1
|
||||
prev = toks[k] if k >= 0 else ""
|
||||
where = f"{rel}:{lines[i]}"
|
||||
if prev == "(" and nxt == ")" and stars: # cast (N *)expr
|
||||
m = j + 1
|
||||
while m < n and (toks[m] in ("&", "*", "(", ")") or toks[m] in CTYPES):
|
||||
m += 1
|
||||
op = toks[m] if m < n else ""
|
||||
if DSYM.match(op):
|
||||
sink.append((t, ("G", op), "0x" + op[2:].upper(), where))
|
||||
elif fn and re.match(r"[A-Za-z_]\w*$", op):
|
||||
fnpend.append(len(sink))
|
||||
sink.append((t, ("L", rel, fn, op), f"{binary}/{addr_of(fn)}:{op}", where))
|
||||
continue
|
||||
if re.match(r"[A-Za-z_]\w*$", nxt) and nxt not in KEYWORDS: # declaration N [*]ident
|
||||
after = toks[j + 1] if j + 1 < n else ""
|
||||
in_params = [p for p in parens if p[0]]
|
||||
if after == "(" and not in_params: # N *fn(...): the return
|
||||
sink.append((t, ("R", binary, nxt), f"{addr_of(nxt)}(ret)", where))
|
||||
elif in_params:
|
||||
p = in_params[-1]
|
||||
(hdrpend if depth == 0 else []).append(len(sink))
|
||||
sink.append((t, ("P", binary, p[0], p[1]), f"{addr_of(p[0])}(arg{p[1]})", where))
|
||||
elif depth == 0 and knr and nxt in knr[1]:
|
||||
ix = knr[1].index(nxt)
|
||||
hdrpend.append(len(sink))
|
||||
sink.append((t, ("P", binary, knr[0], ix), f"{addr_of(knr[0])}(arg{ix})", where))
|
||||
elif DSYM.match(nxt):
|
||||
sink.append((t, ("G", nxt), "0x" + nxt[2:].upper(), where))
|
||||
elif fn:
|
||||
fnpend.append(len(sink))
|
||||
sink.append((t, ("L", rel, fn, nxt), f"{binary}/{addr_of(fn)}:{nxt}", where))
|
||||
elif depth == 0:
|
||||
sink.append((t, ("G", binary, nxt), nxt, where))
|
||||
else:
|
||||
sink.append((t, ("F", rel, nxt), f"field {nxt}", where))
|
||||
continue
|
||||
if nxt in (")", ",") and stars and parens and parens[-1][0]: # abstract declarator in a prototype
|
||||
p = parens[-1]
|
||||
sink.append((t, ("P", binary, p[0], p[1]), f"{addr_of(p[0])}(arg{p[1]})", where))
|
||||
# a global's using routine: the per-overlay instances of one routine's table bind one object ("AG", counted cross-binary)
|
||||
for ix in range(s0, len(sink)):
|
||||
nm, key, cite, where = sink[ix]
|
||||
if key[0] == "G" and len(key) == 2:
|
||||
for fa in sorted(used_by.get(key[1], ())):
|
||||
sink.append((nm, ("AG", fa, binary), cite, where))
|
||||
|
||||
|
||||
def struct_map_join(census):
|
||||
"""(G key -> clusters, (tu, fn, ident) -> cluster) from body_base_type.json, or empty maps"""
|
||||
p = census / "body_base_type.json"
|
||||
g, loc = collections.defaultdict(set), {}
|
||||
if not p.exists():
|
||||
return g, loc, False
|
||||
for body, bases in json.loads(p.read_text()).items():
|
||||
tu, _, fn = body.partition("|")
|
||||
for b, e in bases.items():
|
||||
cls, _, base = b.partition(":")
|
||||
if cls in ("global", "gaddr") and DSYM.match(base):
|
||||
g[base].add(e["type"])
|
||||
elif cls in ("local", "param"):
|
||||
loc[(tu, addr_of(fn), base)] = e["type"]
|
||||
return g, loc, True
|
||||
|
||||
|
||||
def collect(census):
|
||||
rows = load_rows(census)
|
||||
wanted = {nm for r in rows for nm in r["names"]}
|
||||
sink = []
|
||||
files = sorted(p for d in ("src", "include") for p in (REPO / d).rglob("*") if p.suffix in (".c", ".h") and p.is_file())
|
||||
for p in files:
|
||||
rel = str(p.relative_to(REPO))
|
||||
if p.name.startswith(".cdecl"):
|
||||
continue
|
||||
text = p.read_text(errors="replace")
|
||||
if not set(re.findall(r"[A-Za-z_]\w*", text)) & wanted:
|
||||
continue
|
||||
scan_file(rel, text, wanted, sink)
|
||||
bind = collections.defaultdict(dict) # name -> key -> (cite, [where])
|
||||
for name, key, cite, where in sink:
|
||||
e = bind[name].setdefault(key, [cite, []])
|
||||
e[1].append(where)
|
||||
psyq = psyq_layouts()
|
||||
out = []
|
||||
for r in rows:
|
||||
base = r["lhash"].split(":")[0]
|
||||
sdk = sorted(psyq.get(base, ()))
|
||||
owners = collections.defaultdict(set)
|
||||
for nm in r["names"]:
|
||||
for key in bind.get(nm, {}):
|
||||
if key[0] == "AG":
|
||||
owners[key[:2]].add((nm, key[2]))
|
||||
elif key[0] != "F":
|
||||
owners[key].add((nm, None))
|
||||
shared = {k: {nm for nm, _ in v} for k, v in owners.items()
|
||||
if len({nm for nm, _ in v}) > 1 and (k[0] != "AG" or len({b for _, b in v}) > 1)}
|
||||
unbound = [nm for nm in r["names"] if not any(k[0] not in ("F", "AG") for k in bind.get(nm, {}))]
|
||||
klass = "SDK" if sdk else "SHARED" if shared else "UNBOUND" if unbound else "SEPARATE"
|
||||
out.append(dict(r, klass=klass, sdk=sdk, shared=shared, unbound=unbound,
|
||||
bind={nm: bind.get(nm, {}) for nm in r["names"]}))
|
||||
return out
|
||||
|
||||
|
||||
def cites(objs, cap=MAX_CITES):
|
||||
cs = sorted({c for k, (c, _) in objs.items() if k[0] != "F"}, key=lambda c: ("0x" not in c, c))
|
||||
return cs[:cap], len(cs)
|
||||
|
||||
|
||||
def section(c):
|
||||
names = sorted(c["names"])
|
||||
allc = sorted({ci for nm in names for k, (ci, _) in c["bind"][nm].items() if k[0] != "F"})
|
||||
addr = [x for x in allc if "0x" in x]
|
||||
nobj = len(allc)
|
||||
lines = [f"### {c['lhash']}", "names: " + ", ".join(names),
|
||||
f"cause: {len(names)} names ({c['size']} B, all members placeholders) bound to {nobj} distinct objects, none bound to two "
|
||||
f"names: {', '.join(addr[:MAX_CITES])}{' …' if len(addr) > MAX_CITES else ''}; "
|
||||
f"full list: tools/struct_twins.py --show {c['lhash']}"]
|
||||
for nm in names:
|
||||
cs, total = cites(c["bind"][nm])
|
||||
more = f" (+{total - len(cs)})" if total > len(cs) else ""
|
||||
lines.append(f"- `{nm}`: {total} obj — {', '.join(cs)}{more}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write(classes):
|
||||
text = TWINS_DOC.read_text()
|
||||
m = re.search(r"(?m)^## Twins[ \t]*\n", text)
|
||||
if not m:
|
||||
sys.exit("struct_twins: no `## Twins` heading in docs/struct-twins.md")
|
||||
rest = re.search(r"(?m)^## ", text[m.end():])
|
||||
end = m.end() + rest.start() if rest else len(text)
|
||||
body = "\n\n".join(section(c) for c in classes if c["klass"] == "SEPARATE")
|
||||
body = ("<!-- generated by tools/struct_twins.py --write; regenerate, do not hand-edit -->\n\n" + body + "\n\n") if body else "\n"
|
||||
TWINS_DOC.write_text(text[:m.end()] + "\n" + body + text[end:])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--census-dir", default=CENSUS_DEFAULT)
|
||||
ap.add_argument("--write", action="store_true", help="regenerate the `## Twins` section of docs/struct-twins.md")
|
||||
ap.add_argument("--show", metavar="LHASH", help="every name's objects for one class")
|
||||
a = ap.parse_args()
|
||||
census = REPO / a.census_dir
|
||||
classes = collect(census)
|
||||
if a.show:
|
||||
c = next((c for c in classes if c["lhash"] == a.show), None)
|
||||
if not c:
|
||||
sys.exit(f"struct_twins: {a.show} is not a tier-2 row of {census}/dup_gating.tsv")
|
||||
g, loc, have = struct_map_join(census)
|
||||
print(f"{c['lhash']} {c['size']} B · {len(c['names'])} names · {c['klass']}" + ("" if have else " (no body_base_type.json: run the census with --sites for struct_map clusters)"))
|
||||
for nm in sorted(c["names"]):
|
||||
print(f"{nm}:")
|
||||
for k, (ci, wh) in sorted(c["bind"][nm].items(), key=lambda kv: kv[1][0]):
|
||||
lk = (wh[0].rsplit(":", 1)[0], ci.split("/")[-1].rsplit(":", 1)[0], k[-1]) if k[0] == "L" else None
|
||||
sm = sorted(g.get(k[1], ())) if k[0] == "G" else [loc[lk]] if lk in loc else []
|
||||
print(f" {ci:<28} {k[0]} {wh[0]}{f' (+{len(wh) - 1})' if len(wh) > 1 else ''}"
|
||||
f"{' struct_map ' + ','.join(sm[:4]) if sm else ''}")
|
||||
return
|
||||
cnt = collections.Counter(c["klass"] for c in classes)
|
||||
print(f"struct_twins: {len(classes)} tier-2 rows · SEPARATE {cnt['SEPARATE']} · SHARED {cnt['SHARED']} · SDK {cnt['SDK']} · "
|
||||
f"UNBOUND {cnt['UNBOUND']}")
|
||||
for c in classes:
|
||||
if c["klass"] == "SHARED":
|
||||
def cite_of(k, nm):
|
||||
return next(ci for kk, (ci, _) in sorted(c["bind"][nm].items()) if kk[:len(k)] == k)
|
||||
k, v = min(c["shared"].items(), key=lambda kv: (-len(kv[1]), cite_of(kv[0], min(kv[1]))))
|
||||
ci = cite_of(k, min(v)) + (f" (routine {k[1]})" if k[0] in ("A", "AG") else "")
|
||||
print(f" SHARED {c['lhash']} ({len(c['names'])} names, {len(c['shared'])} shared objects): e.g. {ci} <- {', '.join(sorted(v)[:6])}"
|
||||
f"{' …' if len(v) > 6 else ''}")
|
||||
elif c["klass"] == "SDK":
|
||||
print(f" SDK {c['lhash']} ({len(c['names'])} names) layout of {', '.join(c['sdk'])}")
|
||||
elif c["klass"] == "UNBOUND":
|
||||
print(f" UNBOUND {c['lhash']} ({len(c['names'])} names): {', '.join(c['unbound'][:6])}{' …' if len(c['unbound']) > 6 else ''}")
|
||||
if a.write:
|
||||
write(classes)
|
||||
print(f"struct_twins: wrote {cnt['SEPARATE']} `### <lhash>` sections to docs/struct-twins.md ## Twins")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user