#!/usr/bin/env python3 """cast_call_sites.py — the §17a-1 per-site function-pointer cast recovery (Phase 20, cookbook §20). THE wave-at-scale gate cap (the §20 loose-typing call-graph wall). At the reach-134 tail a draft passes `match_one` (compiles STANDALONE with its own externs, masks jal/%hi/%lo) but FAILS the whole-binary gate: a callee (e.g. func_80153C74) is DEFINED in this overlay's TU with one signature (the engine_core.h `DEFINE_func_80153C74()` body `void func_80153C74(s16,s16)`), while the draft declares it `extern void func_80153C74(int,int)` for its own byte-match -> in-TU `conflicting types for func_80153C74` (a COMPILE error, not a byte miss). Every cheaper lever fails here (cookbook §20, all 0): `sig_unify`/`canon_draft_decls` rewrite the draft's callee extern to the ONE canonical sig but leave the CALL `func_X(a,b)` -> the args get converted to the canonical (narrow) param types -> DIFFERENT codegen -> byte miss; `fix_arity_callers` is the caller side, not this; no-proto `void f()` is incompatible with the overlay's narrow def `void f(s16,s16)`; strip-externs gives implicit-int. The ONLY fix (§17a-1, byte-proven Phase 18 Step-1): for each conflicting callee, do BOTH — (1) rewrite the draft's callee DECL line to the canonical (conflict-free, keeps the symbol in scope as a forward declaration before the call), and (2) CAST every CALL site to the draft's INTENDED signature: func_X(a, b) -> (((*)())func_X)(a, b) x = func_X(a) -> x = (((*)())func_X)(a) gcc-2.7.2 folds the cast of a known function symbol back to a direct `jal func_X` with the draft's calling convention, so the body bytes are unchanged — there is no global decl that can conflict, and the call codegen matches the byte target. (Decl lines are never cast.) This is a PURE draft-text transform (like sig_unify / canon_resident_calls): the whole-binary `harvest_verify` byte-gate remains the sole arbiter (G3/P9) — a wrong transform just fails the gate and reverts. It SUPERSEDES sig_unify for the callee-conflict class (it does sig_unify's extern canonicalization PLUS the call-site cast). Run it on the canon_resident_calls output (intended sigs intact); the DEF-side caller-conflict class is still `fix_arity_callers`' job (composes). Pipeline (cookbook §20): draft -> canon_resident_calls -> cast_call_sites -> harvest_verify --chunk 1 (then fix_arity_callers for any residual DEF-side `conflicting types ... (void)` caller conflicts) Usage: tools/cast_call_sites.py --overlay ov_SC01_077 --in .run/drafts-t6-cn --out .run/drafts-t6-cast """ import argparse, os, re, glob, importlib.util REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) def _load(mod, rel): spec = importlib.util.spec_from_file_location(mod, os.path.join(REPO, rel)) m = importlib.util.module_from_spec(spec) spec.loader.exec_module(m) return m _ght = _load('ght', 'tools/gen_harvest_targets.py') _su = _load('su', 'tools/sig_unify.py') # reuse split_top_commas / param_type _cdecl = _load('cdecl', 'tools/cdecl.py') # THE declaration oracle (Phase 26-A, §51g) _corpus = _load('corpus', 'tools/corpus.py') # THE corpus oracle — which TU holds a stub def tu_for(overlay, fn, override=None): """The TU this draft will actually be SPLICED INTO — DERIVED from the corpus, not hand-passed. `--src-file` was an OPTIONAL flag defaulting to `src//.c`. Every caller that did not know to set it — and none of them know about the Phase-26 `_jr_` carve files — was therefore canonicalizing drafts against the MAIN .c while `harvest_verify` (fixed in A3) correctly spliced them into the jr split. The recovery passes were reconciling against a DIFFERENT TRANSLATION UNIT than the one that would compile the code, and the heavy Phase-26 cores live in exactly those jr files. The INCLUDE_ASM line is self-describing, and corpus.stubs() reads it. Ask, don't assume.""" if override: return override try: addr = int(fn[5:], 16) # a curated (non-func_ADDR) name has no address here except ValueError: return os.path.join(REPO, f'src/{overlay}/{overlay}.c') # CorpusError PROPAGATES (R32/R35, P30 S1e). This was a bare `except Exception: pass`, which on # any oracle failure fell through to the default `.c` — silently reinstating the exact bug # the docstring above says this function exists to fix (reconciling against the WRONG TU, where # the heavy Phase-26 jr cores live). A wrong-TU reconcile fails the gate, and this phase's base # rate is ~24k PLUMBING vs 4,917 DIFF, so it would read as a codegen wall. Verified safe: at # P30 S1e HEAD, corpus.stubs raises for 0 of 140 binaries — if it fires, the tree really is # inconsistent and reconciling through it is worse than stopping. st = _corpus.stubs(overlay).get(addr) if st: return os.path.join(REPO, st.path) return os.path.join(REPO, f'src/{overlay}/{overlay}.c') # A function declaration line (prototype ending in `;`, NOT a definition): optional `extern`, # a type, func_X, a param list, `;`, optional trailing /* comment */. Matches both the # `extern void func_X(int,int);` form and the bare `s32 func_X(s32,void*,s32);` m2c form. DECL_LINE_RE = re.compile( # The callee NAME is any identifier, not just `func_XXXXXXXX` (Phase 29 T78). A named PsyQ # symbol (`RotTransPers`, `ApplyMatrixSV`) is a callee like any other, and keying on the func_ # form made every one of them structurally invisible to this tool — see canonical_map. r'^([ \t]*)(extern\s+)?([A-Za-z_][\w \t\*]*?)\b([A-Za-z_]\w*)\s*\(([^;{]*)\)\s*;' r'[ \t]*(?:/\*[^\n]*\*/)?[ \t]*$') # A declaration's type-specifier can NEVER begin with a statement keyword. Without this guard the # pattern above reads a RETURN STATEMENT as a prototype — `return` is a perfectly good `[A-Za-z_]\w*` # where a type is expected: # # return func_8012CB64((s32)out, -0xC0, 0x40, -0x60, 0); # ^^^^^^ captured as the return TYPE, func_8012CB64 as the declared name # # …so the "rewrite this decl to the canonical signature" path REPLACED the statement with # `extern s32 func_8012CB64(s32,s32,s32,s32,s32);`, deleting the return. The C89 fallout is a bare # `parse error before 'extern'` (a declaration after a statement in a block), which reads as a # plumbing failure of the DRAFT rather than a defect in the tool that wrote it. # # Byte-measured (P30 S40): this silently destroyed the return in 9/9 staged members of family # 0x801848dc and was 34 of the 39 failures in that sweep — a 0% that looked like a wall. # # NOTE cdecl CANNOT adjudicate this (checked, do not "fix" it by routing here): `cdecl.parse()` is a # declarator-grammar parser that ASSUMES it was handed a declaration — it reports `return func_X(…);` # as declaring func_X, and `if (f(a));` as declaring `if`. Statement-vs-declaration is a question it # does not answer, so the keyword guard belongs here. (§134's law still holds — route line-SHAPE # masking through cdecl._mask — but "is this text a declaration at all" is a different question.) _STMT_KEYWORDS = frozenset(( 'return', 'if', 'else', 'while', 'for', 'do', 'switch', 'case', 'default', 'break', 'continue', 'goto', 'sizeof', )) def _is_decl_match(m): """True iff a DECL_LINE_RE match is really a declaration (not a statement wearing its shape).""" if not m: return False head = (m.group(3) or '').strip().split() return not (head and head[0] in _STMT_KEYWORDS) def parse_sig(ret, params): """('void', 'int a0, int a1') -> ('void', ['int','int']); '' / 'void' params -> [].""" ret = re.sub(r'\s+', ' ', ret).strip() params = params.strip() if params in ('', 'void'): ptypes = [] else: ptypes = [_su.param_type(x) for x in _su.split_top_commas(params)] return ret, ptypes def norm_sig(ret, ptypes): """A spelling-insensitive-ish key for 'do these sigs differ': collapse whitespace, treat the typedef int aliases as int (s32/int/long are byte-identical params -> casting them is a no-op, so we must NOT count them as a difference and emit a pointless cast).""" def canon_t(t): t = re.sub(r'\s+', ' ', t).strip() t = re.sub(r'\s*\*', ' *', t).strip() # int-family aliases collapse (same calling-convention / same width) t = re.sub(r'\b(s32|u32|int|unsigned int|unsigned|long|unsigned long|u_long)\b', 'int', t) return t return canon_t(ret), tuple(canon_t(t) for t in ptypes) def cast_type(dret, dptypes): """'void',['int','int'] -> 'void (*)(int, int)'; 's32',[] -> 's32 (*)(void)'.""" inner = ', '.join(dptypes) if dptypes else 'void' return f'{dret} (*)({inner})' def canonical_map(overlay, src_file=None): """addr-int -> canonical sig string ' func_X()' — DERIVED from what cc1 actually sees in the TU (`cdecl.tu_scope`, i.e. cpp), not scraped out of the raw text. WHY THIS CHANGED (Phase 26-A, R33; cookbook §51g LAW 7) ------------------------------------------------------ It used to union three raw-text scanners: collect_extern_sigs([engine_core.h, the .c]) + collect_define_sigs(ec) + collect_inline_sigs(c) and NONE of them can see a MACRO-INJECTED declaration — an `extern` inside a `DEFINE_func_*` macro body, which becomes a genuine file-scope declaration of every TU that invokes the macro. engine_core.h is 23,546 continuation lines inside 1,801 such macros, so this was not an edge case. The cost, measured: `func_801387B8` (a stub in 134 TUs) calls `func_80138DE0`, which its TU declares — via a macro expansion — as `s32 (s32, s32, s32)`. The draft declares `s32 (s32)`. The map returned NOTHING for that callee, so transform() took the `if addr not in canon: continue # pure stub callee -> no conflict, leave it` branch — on a premise that was simply false — and the draft died with `conflicting types`, which reads downstream as an intrinsic compiler wall. cpp answers it exactly, in 54 ms. NAMED SYMBOLS TOO, AND THE KEY IS THE NAME (Phase 29 T78). This used to require `re.fullmatch(r'func_[0-9A-Fa-f]{8}', name)` and key by the parsed address, so every CURATED or PsyQ-library callee was invisible: `func_8012F40C`'s family failed 138x on `conflicting types for RotTransPers` and this tool — the one lever for exactly that class — could not see the symbol at all. Curated naming is something this project does MORE of as RE quality improves, so a func_-only predicate rots by design (the same shape as `stub_map`'s, Phase 26-A). """ c_path = src_file or os.path.join(REPO, f'src/{overlay}/{overlay}.c') sigs = {} for name, d in _cdecl.tu_scope(c_path).items(): if d.kind != 'func': continue sigs[name] = d.declaration(storage='').rstrip(';').strip() return sigs, c_path def split_sig_string(s, fn=None): """'void func_X(s16 a0, s16 a1)' -> ('void', 's16 a0, s16 a1') (no extern / no ;). ANY identifier, not just `func_XXXXXXXX` (Phase 29 T78). This was the THIRD place keyed on the func_ name form, and the one that survived the first two fixes: with `canonical_map` and `transform` widened, `RotTransPers` reached this function and got `None`, so `transform` took its `if not csig: continue` branch and cast nothing — a silent skip that looked exactly like "no conflict found". Pass `fn` to anchor on the known callee; otherwise the last identifier before the parameter list is used.""" s = s.strip() if fn: m = re.match(rf'(.*?)\b{re.escape(fn)}\s*\((.*)\)\s*$', s, re.S) else: m = re.match(r'(.*?)\b[A-Za-z_]\w*\s*\((.*)\)\s*$', s, re.S) if not m: return None return m.group(1).strip(), m.group(2).strip() _ABOVE = set() # names the TU declares ABOVE the splice point (set by main/gate_stage) def _no_conflict(canon_sig, draft_line, fn): """Will cc1 accept the TU's declaration of `fn` alongside the draft's? (cdecl.compatible, which is validated against the REAL gcc-2.7.2 cc1 on 1,485 live corpus pairs — `cdecl.py --compat`.) This replaces `norm_sig(...) == norm_sig(...)`, which collapsed the int family (s32|u32|int| unsigned|long) to ONE token and therefore called a SIGNEDNESS change "already compatible" and emitted no rewrite — while cc1 rejects that redeclaration outright. It was right about codegen (same width, same load) and wrong about the C FRONT END, which never reaches codegen. Order matters, and only cc1 could have told us so: a no-prototype decl followed by a narrow-param prototype CONFLICTS, but the reverse order is ACCEPTED (§51g / the Phase-15 narrow-param wall). So the TU's decl goes first iff it is declared above the splice point. """ try: c = _cdecl.parse(f'extern {canon_sig};')[0] d = _cdecl.parse(draft_line.strip())[0] except (_cdecl.CDeclError, IndexError): return False # unparseable -> be safe, reconcile it a, b = (c, d) if fn in _ABOVE else (d, c) # TU order return _cdecl.compatible(a, b) def transform(text, self_fn, canon): """Return (new_text, n_callees_cast). For each callee whose canonical TU sig CONFLICTS with the draft's intended sig: rewrite the decl line to canonical + cast every call to the intended sig.""" lines = text.split('\n') # Pass 1: scan decl lines -> intended sigs; decide which callees conflict (need casting). to_cast = {} # func_Y -> cast_type string (draft's intended sig) canon_decl = {} # func_Y -> canonical decl line replacement (indentation preserved) decl_line_idx = {} # func_Y -> set of line indices that are its decl line(s) for i, ln in enumerate(lines): m = DECL_LINE_RE.match(ln) if not _is_decl_match(m): # a statement wearing a prototype's shape is NOT a decl continue indent, _extern, ret, fn, params = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5) if fn == self_fn: continue if fn not in canon: # pure stub callee: draft's decl is the only continue # one -> no conflict, leave it (don't cast) dret, dptypes = parse_sig(ret, params) csig = split_sig_string(canon[fn], fn) if not csig: continue cret, cptypes = parse_sig(*csig) if _no_conflict(canon[fn], ln, fn): continue # cc1 accepts both -> no rewrite, no cast to_cast[fn] = cast_type(dret, dptypes) canon_decl[fn] = f'{indent}extern {canon[fn]};' decl_line_idx.setdefault(fn, set()).add(i) if not to_cast: return text, 0 # Pass 2: rebuild. Decl lines of cast-callees -> canonical decl. Body lines -> cast the calls. call_res = {fn: re.compile(r'\b' + re.escape(fn) + r'\s*\(') for fn in to_cast} out = [] for i, ln in enumerate(lines): replaced_decl = None for fn, idxs in decl_line_idx.items(): if i in idxs: replaced_decl = fn break if replaced_decl is not None: out.append(canon_decl[replaced_decl]) # the canonical forward decl (no cast here) continue if _is_decl_match(DECL_LINE_RE.match(ln)): # some OTHER decl line -> never cast in a decl out.append(ln) continue for fn, cty in to_cast.items(): # body line -> cast every call site ln = call_res[fn].sub(f'(({cty}){fn})(', ln) out.append(ln) return '\n'.join(out), len(to_cast) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--overlay', default='ov_SC01_077') ap.add_argument('--src-file', dest='src_file', default=None, help='overlay .c whose local decls the draft must match (default main; use the ' '_a.c/_o0.c split file for drafts that land there)') ap.add_argument('--in', dest='indir', required=True) ap.add_argument('--out', dest='outdir', required=True) a = ap.parse_args() override = a.src_file and os.path.join(REPO, a.src_file) os.makedirs(os.path.join(REPO, a.outdir), exist_ok=True) drafts = touched = total_callees = 0 cache, ncanon = {}, 0 for p in sorted(glob.glob(os.path.join(REPO, a.indir, '*.c'))): fn = os.path.basename(p)[:-2] # PER-DRAFT: canonicalize against the TU that will actually compile it (derived), not # against whatever .c the caller happened to name. tu = tu_for(a.overlay, fn, override) if tu not in cache: cache[tu] = canonical_map(a.overlay, tu)[0] canon = cache[tu] ncanon = max(ncanon, len(canon)) # TU order for the no-prototype rule: which of the TU's decls precede this splice point? # (cdecl caches the cpp run, so this is ~free per draft.) _ABOVE.clear() _ABOVE.update(_cdecl.tu_scope(tu, above=fn)) new, k = transform(open(p).read(), fn, canon) open(os.path.join(REPO, a.outdir, os.path.basename(p)), 'w').write(new) drafts += 1 if k: touched += 1 total_callees += k print(f'canonical sigs: {ncanon} (per-TU, derived); TUs: {len(cache)}; drafts: {drafts}; ' f'cast-recovered: {touched} draft(s), {total_callees} callee(s)') if __name__ == '__main__': main()