mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-28 06:49:47 -04:00
feat(phase-24): T7b/T2+T3 — reconcile_decls.py (data-symbol canonical reconcile); byte-proven on func_80129CF8
- tools/reconcile_decls.py: the DATA-symbol analog of cast_call_sites.py. A fleet-majority canonical-type ORACLE (engine_core.h-authoritative + plurality across all overlays+resident — the picker gen_harvest_targets lacked) + a byte-neutral access-cast TRANSFORM (array/struct/ scalar/ptr taxonomy, single-pass so no double-wrap). Mechanizes the manual giant decl-reconcile that func_80129CF8 got by hand. - wired into gate_stage after cast_call_sites (canon -> cast -> reconcile_decls -> byte-gate); idempotent / no-op on drafts without a data conflict -> can't regress the wave; the whole-binary byte-gate stays the sole arbiter (G3/P9). - VALIDATED: oracle 8/8 correct picks for func_80129CF8's symbols; full loose giant -> reconcile_decls -> swapped into the DEFINE_ macro -> make build BINARY=ov_SC01_077 = BYTE-IDENTICAL d19c9580; idempotent (0 reconciled) on the already-canonical committed draft.
This commit is contained in:
Drew T
@@ -165,6 +165,11 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_
|
||||
cast_extra = (["--src-file", src_file] if src_file else None)
|
||||
d1 = _xform("canon_resident_calls.py", binary, drafts, "-cn")
|
||||
d1 = _xform("cast_call_sites.py", binary, d1, "-cast", extra=cast_extra)
|
||||
# data-symbol analog of cast_call_sites: rewrite each loose D_XXXX extern -> fleet-canonical +
|
||||
# byte-neutral access cast (§33, T7b). No-op/idempotent on drafts without a data-decl conflict, so
|
||||
# it can't regress the wave; the byte-gate is still the sole arbiter. Chiefly unblocks GIANTS
|
||||
# (hoisted global base arrays) but composes harmlessly for every draft.
|
||||
d1 = _xform("reconcile_decls.py", binary, d1, "-rc", extra=cast_extra)
|
||||
verified = _gate1(binary, src, asm, out, good_sha, d1, verified_out, failed_out)
|
||||
|
||||
d = d1
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""reconcile_decls.py — the DATA-symbol analog of cast_call_sites.py (Phase 24 T7b, cookbook §33).
|
||||
|
||||
A freshly-matched giant / wave draft byte-matches STANDALONE with its own guessed decls, but to bank
|
||||
×1 in the overlay TU (and then propagate ×134 via a shared `DEFINE_func_*` macro) its callee/data
|
||||
externs must be FLEET-CANONICAL — otherwise `conflicting types` vs engine_core.h / sibling decls
|
||||
(a COMPILE error, not a byte miss). `cast_call_sites.py` handles this for callee FUNCTIONS
|
||||
(decl->canonical + fn-ptr cast at the call). This tool does the same for DATA symbols `D_XXXX`:
|
||||
rewrite the draft's `extern <T_draft> D_x...;` -> the fleet-canonical decl, and inject a byte-neutral
|
||||
cast at each ACCESS so the load/store opcode is unchanged. gcc-2.7.2 folds the compile-time
|
||||
pointer/scalar cast of a known symbol, so the body bytes are identical.
|
||||
|
||||
This is a PURE draft-text transform (like sig_unify / cast_call_sites / canon_resident_calls): the
|
||||
whole-binary `harvest_verify` byte-gate remains the sole arbiter (G3/P9) — a wrong reconcile just
|
||||
fails the gate and reverts, so it trial-and-errors safely. It does NOT touch callee FUNCTION decls
|
||||
(that's cast_call_sites/sig_unify — compose them). It changes only the draft's OWN data externs +
|
||||
the access sites; it never edits the fleet's canonical decls.
|
||||
|
||||
Byte-neutral cast taxonomy (proven by hand on func_80129CF8; the intended type is what the DRAFT
|
||||
declared, the canonical is the fleet's — cast every use to reproduce the intended access):
|
||||
draft array Ed[] vs canonical array Ec[] -> decl Ec[]; D_x[i] -> ((Ed*)D_x)[i]; D_x -> (Ed*)D_x
|
||||
draft array Ed[] vs canonical struct/scalar -> decl canon; D_x -> (Ed*)&D_x (address-of the object)
|
||||
draft scalar Td vs canonical scalar Tc -> decl Tc; D_x -> *(Td*)&D_x (forces the Td-width opcode)
|
||||
draft ptr P* D_x vs canonical scalar Tc -> decl Tc; write D_x=v -> D_x=(Tc)(v); read D_x -> (P*)D_x
|
||||
|
||||
Pipeline (cookbook §33):
|
||||
draft -> canon_resident_calls -> cast_call_sites -> reconcile_decls -> sig_unify -> harvest_verify --chunk 1
|
||||
(callee funcs) (data symbols) (def sig) (the byte-gate)
|
||||
|
||||
Usage:
|
||||
tools/reconcile_decls.py --overlay ov_SC01_077 --src-file src/ov_SC01_077/ov_SC01_077_a.c \
|
||||
--in .run/drafts-giant --out .run/drafts-giant-rc
|
||||
tools/reconcile_decls.py --print-canon D_80126948 # inspect the oracle's pick for one symbol
|
||||
"""
|
||||
import argparse, os, re, glob, importlib.util, collections
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def _load(mod, rel):
|
||||
spec = importlib.util.spec_from_file_location(mod, os.path.join(REPO, rel))
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
return m
|
||||
|
||||
|
||||
_ght = _load('ght', 'tools/gen_harvest_targets.py') # reuse collect_data_decls / the D_ regexes
|
||||
|
||||
# `extern <type> D_XXXX;` or `extern <type> D_XXXX[];` (optional trailing /* comment */). <type> may
|
||||
# carry a trailing `*` (pointer) which we fold into the element type.
|
||||
DATA_DECL_LINE_RE = re.compile(
|
||||
r'^([ \t]*)extern\s+([A-Za-z_][\w \t]*?)\s*(\*?)\s*\b(D_[0-9A-Fa-f]+)\b\s*(\[\s*\])?\s*;'
|
||||
r'[ \t]*(?:/\*[^\n]*\*/)?[ \t]*$')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the canonical-type ORACLE (T2)
|
||||
def canonical_data_map(extra_src=None):
|
||||
"""Fleet-majority canonical decl per data symbol. Returns {D_name: 'extern <decl>;'}.
|
||||
|
||||
Precedence: an engine_core.h macro decl is AUTHORITATIVE (it's the shared/propagated set every
|
||||
overlay co-instantiates — conforming to it is what avoids the per-overlay macro conflict). Else
|
||||
the plurality vote across all overlays + resident (ties -> the lexicographically-first spelling,
|
||||
deterministic). This is the picker the design-doc flagged as missing (gen_harvest_targets read
|
||||
only engine_core.h + ONE overlay, first-seen-wins, and never saw a fleet disagreement)."""
|
||||
ec = os.path.join(REPO, 'src/shared/engine_core.h')
|
||||
# 1. engine_core.h — authoritative (a symbol declared inside any DEFINE_ macro)
|
||||
ec_decls = _ght.collect_data_decls([ec]) # {D_name: 'extern ...;'} first-seen
|
||||
# 2. plurality across the fleet (all overlays + resident) for symbols engine_core.h lacks
|
||||
votes = collections.defaultdict(collections.Counter)
|
||||
fleet = (glob.glob(os.path.join(REPO, 'src/ov_*/*.c'))
|
||||
+ [os.path.join(REPO, 'src/resident/resident.c')])
|
||||
if extra_src:
|
||||
fleet.append(os.path.join(REPO, extra_src))
|
||||
for p in fleet:
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
txt = open(p).read()
|
||||
for m in _ght.DATA_DECL_RE.finditer(txt):
|
||||
decl = re.sub(r'\s+', ' ', m.group(1)).strip()
|
||||
nm = re.search(r'D_[0-9A-Fa-f]+', decl)
|
||||
if nm:
|
||||
votes[nm.group(0)][f'extern {decl};'] += 1
|
||||
canon = {}
|
||||
names = set(ec_decls) | set(votes)
|
||||
for nm in names:
|
||||
if nm in ec_decls:
|
||||
canon[nm] = ec_decls[nm] # authoritative
|
||||
else:
|
||||
# plurality; tie -> lexicographically-first decl spelling (deterministic)
|
||||
best = sorted(votes[nm].items(), key=lambda kv: (-kv[1], kv[0]))[0][0]
|
||||
canon[nm] = best
|
||||
return canon
|
||||
|
||||
|
||||
def parse_data_decl(decl):
|
||||
"""'extern u8 D_x[];' -> (name, elem='u8', is_array=True, is_ptr=False).
|
||||
'extern struct BigCopy D_x;' -> (name, 'struct BigCopy', False, False).
|
||||
'extern s32 *D_x;' -> (name, 's32', False, is_ptr=True)."""
|
||||
m = DATA_DECL_LINE_RE.match(decl) or DATA_DECL_LINE_RE.match(' ' + decl.strip())
|
||||
if not m:
|
||||
return None
|
||||
_indent, base, star, name, arr = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5)
|
||||
return name, re.sub(r'\s+', ' ', base).strip(), bool(arr), bool(star)
|
||||
|
||||
|
||||
def _norm_type(elem, is_array, is_ptr):
|
||||
"""Spelling-insensitive key for 'do these decls differ' — collapse int-family aliases (same
|
||||
width+conv, casting them is a no-op), normalise whitespace. Array vs ptr vs scalar are distinct."""
|
||||
e = re.sub(r'\s+', ' ', elem).strip()
|
||||
e = re.sub(r'\b(s32|u32|int|unsigned int|unsigned|long|unsigned long|u_long)\b', 'int', e)
|
||||
kind = 'arr' if is_array else ('ptr' if is_ptr else 'sca')
|
||||
return (e, kind)
|
||||
|
||||
|
||||
def _base_ptr_expr(name, ielem, canon_is_array):
|
||||
"""The draft's intended array/base pointer, expressed under the canonical storage decl.
|
||||
canonical array -> the array decays: (Ed*)D_x
|
||||
canonical struct/scalar -> take its address: (Ed*)&D_x"""
|
||||
return f'(({ielem} *){name})' if canon_is_array else f'(({ielem} *)&{name})'
|
||||
|
||||
|
||||
def data_access_subs(name, idecl, cdecl):
|
||||
"""One (compiled-regex, replacement-fn) that reproduces the DRAFT's intended access to `name`
|
||||
under the CANONICAL storage decl in a SINGLE pass (re.sub never re-scans its own output, so a
|
||||
symbol appearing in several forms on one line can't double-wrap). [] if type-compatible, None if
|
||||
unparseable. The regex captures an optional leading `&` and a following `[` to pick the form."""
|
||||
ip = parse_data_decl(idecl)
|
||||
cp = parse_data_decl(cdecl)
|
||||
if not ip or not cp:
|
||||
return None # unparseable -> caller logs, leaves as-is
|
||||
_, ielem, iarr, iptr = ip
|
||||
_, celem, carr, cptr = cp
|
||||
if _norm_type(ielem, iarr, iptr) == _norm_type(celem, carr, cptr):
|
||||
return [] # already compatible
|
||||
rx = re.compile(rf'(&?)\b{re.escape(name)}\b(\s*\[)?')
|
||||
if iarr:
|
||||
base = _base_ptr_expr(name, ielem, carr) # (Ed*)D_x or (Ed*)&D_x (canon array vs not)
|
||||
def repl(m):
|
||||
amp, idx = m.group(1), m.group(2)
|
||||
if idx: # D_x[i] / &D_x[i] -> [&]base[i]
|
||||
return f'{amp}{base}{idx}'
|
||||
return base # bare D_x / &D_x -> base (array address IS the base ptr)
|
||||
else:
|
||||
# scalar (signedness/width) OR ptr-vs-scalar: force the intended-width/type access via a
|
||||
# cast-lvalue `*(<intended> *)&D_x` (valid as both lvalue and rvalue; ptr-intended -> `ielem **`).
|
||||
star = ' *' if iptr else ''
|
||||
acc = f'(*({ielem}{star} *)&{name})'
|
||||
def repl(m):
|
||||
amp, idx = m.group(1), m.group(2)
|
||||
return f'{amp}{acc}{idx or ""}'
|
||||
return [(rx, repl)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the transform (T3)
|
||||
def transform(text, canon):
|
||||
"""Return (new_text, n_data_reconciled, notes). For each data symbol the draft declares with a
|
||||
type differing from the fleet-canonical: rewrite the decl -> canonical + cast every access."""
|
||||
lines = text.split('\n')
|
||||
plan = {} # D_name -> (canonical_decl_line, [subs])
|
||||
decl_idx = {} # D_name -> line index of its decl
|
||||
notes = []
|
||||
for i, ln in enumerate(lines):
|
||||
m = DATA_DECL_LINE_RE.match(ln)
|
||||
if not m:
|
||||
continue
|
||||
indent, name = m.group(1), m.group(4)
|
||||
idecl = ln.strip()
|
||||
if name not in canon:
|
||||
continue # no fleet decl -> draft's is the only one, keep
|
||||
cdecl = canon[name]
|
||||
subs = data_access_subs(name, idecl, cdecl)
|
||||
if subs is None:
|
||||
notes.append(f'{name}: UNPARSEABLE decl, left as-is'); continue
|
||||
if not subs:
|
||||
continue # already compatible -> nothing to do
|
||||
plan[name] = (f'{indent}{cdecl}', subs)
|
||||
decl_idx[name] = i
|
||||
|
||||
if not plan:
|
||||
return text, 0, notes
|
||||
|
||||
out = []
|
||||
for i, ln in enumerate(lines):
|
||||
dname = next((n for n, j in decl_idx.items() if j == i), None)
|
||||
if dname is not None:
|
||||
out.append(plan[dname][0]) # replace the decl line with the canonical decl
|
||||
continue
|
||||
if DATA_DECL_LINE_RE.match(ln): # some OTHER data decl -> never cast in a decl
|
||||
out.append(ln); continue
|
||||
for name, (_c, subs) in plan.items(): # body line -> rewrite this symbol's accesses
|
||||
for rx, rep in subs:
|
||||
ln = rx.sub(rep, ln)
|
||||
out.append(ln)
|
||||
return '\n'.join(out), len(plan), notes
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--overlay', default='ov_SC01_077')
|
||||
ap.add_argument('--src-file', dest='src_file', default=None,
|
||||
help='overlay .c whose local decls also vote (default: none; the fleet scan '
|
||||
'already covers all overlays)')
|
||||
ap.add_argument('--in', dest='indir')
|
||||
ap.add_argument('--out', dest='outdir')
|
||||
ap.add_argument('--print-canon', metavar='D_XXXX', help='print the oracle pick for one symbol and exit')
|
||||
a = ap.parse_args()
|
||||
|
||||
canon = canonical_data_map(a.src_file)
|
||||
if a.print_canon:
|
||||
print(canon.get(a.print_canon, f'(no fleet decl for {a.print_canon})'))
|
||||
return
|
||||
if not (a.indir and a.outdir):
|
||||
ap.error('need --in and --out (or --print-canon)')
|
||||
|
||||
os.makedirs(os.path.join(REPO, a.outdir), exist_ok=True)
|
||||
drafts = touched = total = 0
|
||||
for p in sorted(glob.glob(os.path.join(REPO, a.indir, '*.c'))):
|
||||
new, k, notes = transform(open(p).read(), canon)
|
||||
open(os.path.join(REPO, a.outdir, os.path.basename(p)), 'w').write(new)
|
||||
drafts += 1
|
||||
if k:
|
||||
touched += 1; total += k
|
||||
for n in notes:
|
||||
print(f' [{os.path.basename(p)}] {n}')
|
||||
print(f'canonical data decls: {len(canon)}; drafts: {drafts}; '
|
||||
f'reconciled: {touched} draft(s), {total} data symbol(s)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user