From df7bca2e6592cfed1b698e44cf2f9d0b8cdef548 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:36:36 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-31):=20tools/reconcile=5Fslate.py=20?= =?UTF-8?q?=E2=80=94=20drive=20a=20slate=20to=200-dropped=20BEFORE=20the?= =?UTF-8?q?=20first=20rebuild?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 68%->95% lever from §176h.C2, mechanized. Reconciliation belongs INSIDE the wave: a banked draft's declarations become the TU's, so a sibling clash hardens into a file clash and post-bank recovery is measurably worse (18 parked drafts still MATCH, only 1 survived after their wave banked vs 5 before). AUTO-FIXES, each re-verified with match_one and REVERTED if a byte moves (a declaration change is a codegen change, §176f): * COSMETIC-TYPEDEF two names for a structurally identical struct -> adopt the other. Compared by BODY, never by name (OtBlk_80015498 == OtBlk_80016450; Elem12 != B12). This body comparison is also the answer to §176h.C's spelled-name limit. * SIGNEDNESS / ALIAS / ARRAY-VS-SCALAR -> adopt the TU's spelling, fixing the use site. * DEFPARAMS (NEW LEVER) -> adopt the TU's parameter types on the DEFINITION and re-narrow with a shadowing local: `void f(s32 a0_p) { s16 a0 = (s16)a0_p; }`. One textual insertion instead of rewriting every use site, and the cast emits the same sll/sra pair. Byte-identical on both cases tried. REFUSES, with named reasons, because these are decisions and not edits: DIFFERENT-STRUCT (two real layouts for one symbol), IMMOVABLE-TU-DECL (gate_main reverts src/, so it needs its own commit + rebuild + R22), DEF-SIDE-RETURN (adopting the TU's return type usually costs the match -- measured on func_8001ABBC), and BROKE-MATCH for anything its own verification rejects. Measured on wave P's leftover slate: 6 -> 9 compatible, 3 auto-reconciled, 2 repairs reverted by the tool's own byte check, 7 named for a human. build_wave_atlas: --rank mass (main's gate cost is per SLATE, so ranking groups by member count silently collapses a wide band to the smallest functions -- measured 60 cards/2,604 ins where 46 cards/4,829 ins were available), and the selector no longer counts ITS OWN OUTPUT as already-waved (re-running with identical filters had been shrinking the pool 60 -> 46). --- tools/build_wave_atlas.py | 21 +++- tools/reconcile_slate.py | 244 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 tools/reconcile_slate.py diff --git a/tools/build_wave_atlas.py b/tools/build_wave_atlas.py index 59d448a8d..38a73c686 100644 --- a/tools/build_wave_atlas.py +++ b/tools/build_wave_atlas.py @@ -20,7 +20,7 @@ MUST NOT run while a gate is in flight (R35 — corpus.stubs() misreports substi Usage: build_wave_atlas.py [N] [--max-bins K] [--min-ins M] [--levers a,b,c] """ -import json, sys, collections, subprocess, argparse, glob +import json, os, sys, collections, subprocess, argparse, glob sys.path.insert(0, 'tools') import corpus @@ -41,6 +41,14 @@ ap.add_argument('--exclude-bins', default='', 'INCREMENTALLY, because its extract runs psyq_integrate/ld_interleave and ' 'rewrites the .ld. Draft main like any binary; gate it with ' 'tools/gate_main.py, never gate_lane/gate_stage.)') +ap.add_argument('--rank', choices=('groups','mass'), default='groups', + help="'groups' (default) ranks gate groups by MEMBER COUNT -- right for overlays, " + "where every (binary,TU) group costs its own rebuild. 'mass' ranks purely by " + "instruction size across all groups -- right for MAIN, whose gate cost is per " + "SLATE, not per TU: with 'groups' a wide --min-ins band fills from the " + "biggest-by-count group, which is the SMALLEST-by-instruction one, and the " + "wave silently collapses to tiny functions (measured: 60 cards / 2,604 ins " + "avg 43, when 46 cards / 4,829 ins avg 105 were available).") ap.add_argument('--target-ins', type=int, default=0, help='size the wave by INSTRUCTION MASS: keep drawing cards until this many ' 'instructions are selected (still capped by n). The public metric is ' @@ -61,7 +69,11 @@ if busy.returncode == 0 and busy.stdout.strip(): # R32/R33: derive the already-waved set from what is ON DISK, never from a hardcoded wave-letter # list (the literal 'a'..'l' silently missed waves m and n and would have re-issued their cards). -PRIORS = sorted(glob.glob('.run/wave_*_cards.json')) +# Exclude OUR OWN output: the glob matches it, so re-running the selector after an aborted or +# re-tuned build marked the previous attempt's cards as 'already waved' and silently shrank the +# pool (measured: 46 candidates instead of 60 on a re-run with identical filters). +PRIORS = [p for p in sorted(glob.glob('.run/wave_*_cards.json')) + if os.path.abspath(p) != os.path.abspath(a.out)] taken = set() for p in PRIORS: try: @@ -126,7 +138,10 @@ for g in atlas['groups']: by_tu = collections.defaultdict(list) for c in cands: by_tu[(c['binary'], c['tu'])].append(c) -ranked = sorted(by_tu, key=lambda k: -len(by_tu[k]))[:a.max_bins] +if a.rank == 'mass': + ranked = sorted(by_tu, key=lambda k: -sum(c['nins'] for c in by_tu[k]))[:a.max_bins] +else: + ranked = sorted(by_tu, key=lambda k: -len(by_tu[k]))[:a.max_bins] # principle 3 (P31 S52): SIZE A WAVE BY INSTRUCTIONS, NOT BY CARDS. The public metric is # instruction-weighted, so a wave is worth what its instructions are worth: the 12-42-ins card diff --git a/tools/reconcile_slate.py b/tools/reconcile_slate.py new file mode 100644 index 000000000..4e50ac464 --- /dev/null +++ b/tools/reconcile_slate.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""reconcile_slate.py — drive a main slate to `N -> N compatible, 0 dropped` BEFORE any rebuild. + +WHY (P31 S52, cookbook §176g/§176h.C2). Wave P drafted at 97% and banked 68%. The whole gap was +declaration plumbing, and the fix is NOT to bank the clean drafts and recover the rest later -- +that is measurably backwards, because a banked draft's declarations become the TU's, so a +sibling-vs-sibling clash (settleable by editing either side) hardens into a file-vs-draft clash +(settleable only by editing the draft, and sometimes not at all). Of 18 parked drafts still +verifying MATCH, only 1 survived the conflict check after their wave banked, versus 5 before. + +So reconciliation belongs INSIDE the wave, before the first gate. This tool automates the part +that is mechanical, and -- just as importantly -- REFUSES the part that is not, naming what a +human must decide. + +WHAT IT FIXES AUTOMATICALLY (each re-verified with match_one; a repair that changes a byte is +reverted, because a declaration change is a codegen change -- §176f): + * COSMETIC-TYPEDEF two names for a structurally IDENTICAL struct -> adopt the other name. + Compared by BODY, never by name: OtBlk_80015498 and OtBlk_80016450 are the + same {s32 a; s32 b[4];}, while Elem12 and B12 genuinely differ and are + refused. (This body comparison is also the answer to §176h.C's limit.) + * SIGNEDNESS `extern u16 D_x` vs `extern s16 D_x` -> adopt the TU's spelling. + * ARRAY-VS-SCALAR `u8 D_x[]` vs `u8 D_x` -> adopt the TU's, fixing the use site + (`D_x[i]` <-> `(&D_x)[i]`). + * ALIAS `short` vs `s16` -> adopt the TU's spelling (no code change at all). + +WHAT IT REFUSES, AND WHY (these are decisions, not edits): + * DIFFERENT-STRUCT two genuinely different layouts for one symbol -- someone modelled it wrong. + * IMMOVABLE-TU-DECL the draft needs the TU's own declaration changed. gate_main REVERTS src/ + before every build, so this can never ride inside a slate: it needs its own + commit + rebuild + R22. + * DEF-SIDE-RETURN the TU prototypes the function with a different RETURN type. Adopting it + usually breaks the match (measured on func_8001ABBC: `void` cost the match). + +Usage: + reconcile_slate.py [--apply] [--max-rounds N] + default is a DRY RUN listing what it would do. + --apply rewrites the DRAFTS (never the tree) and loops until 0 dropped or no progress. +""" +import argparse, json, os, re, subprocess, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import corpus +import gate_main as gm + +SIG = re.compile(r"^\('(?P[^']*)', '(?P[^']*)'\)$") + + +FNSIG = re.compile(r"^\('(?P[^']*)', \((?P.*)\)\)$") + + +def _parse_fnsig(sig): + """('void', ('s32', 'u8*')) -> ('void', ['s32','u8*']); (None, None) if not a function sig.""" + m = FNSIG.match(sig) + if not m: + return None, None + params = [p.strip().strip("'") for p in m.group('params').split(',') if p.strip()] + return m.group('ret'), params + + +def _apply_tu_params(path, fn, mine, theirs): + """§176d, mechanized: adopt the TU's parameter types on the DEFINITION and re-narrow inside + the body, so the emitted bytes are unchanged. + + void f(s16 a0) -> void f(s32 a0_p) { s16 a0 = (s16)a0_p; } + + A shadowing local is used rather than rewriting every use site: it is one textual insertion, + the body is untouched, and the cast emits the same sll/sra pair the narrow parameter did. The + caller re-verifies with match_one and reverts if a single byte moves, so attempting it is + free.""" + src = open(path).read() + m = re.search(r'^([A-Za-z_][\w \t\*]*?)\b%s\s*\(([^;{]*)\)\s*\{' % re.escape(fn), src, re.M) + if not m or len(mine) != len(theirs): + return False + decls = [d.strip() for d in m.group(2).split(',')] + if len(decls) != len(mine): + return False + new_decls, shims = [], [] + for d, mt, tt in zip(decls, mine, theirs): + nm = re.findall(r'(\w+)\s*$', d) + if not nm: + return False + nm = nm[0] + if mt == tt: + new_decls.append(d) + continue + new_decls.append('%s %s_p' % (tt.replace('*', ' *'), nm)) + shims.append(' %s %s = (%s)%s_p;' % (mt.replace('*', ' *'), nm, + mt.replace('*', ' *'), nm)) + if not shims: + return False + head = '%s%s(%s) {\n%s' % (m.group(1), fn, ', '.join(new_decls), '\n'.join(shims)) + open(path, 'w').write(src[:m.start()] + head + src[m.end():]) + return True + + +def _bodies(text): + out = {} + for pat in (gm.TYPEDEF_BLOCK, gm.TYPEDEF_PLAIN): + for m in pat.finditer(text): + out.setdefault(m.group(1), re.sub(r'\s+', ' ', m.group(0))) + return out + + +def _same_struct(a_name, a_body, b_name, b_body): + """Structurally identical modulo the typedef's own name?""" + if not (a_body and b_body): + return False + return (re.sub(r'\b%s\b' % re.escape(a_name), 'X', a_body) == + re.sub(r'\b%s\b' % re.escape(b_name), 'X', b_body)) + + +def verify(fn, draft): + st = {s.symbol: s for s in corpus.stubs('main').values()}.get(fn) + if not st: + return False + r = subprocess.run(['.venv/bin/python', 'tools/match_one.py', fn, '--c', draft, + '--asm-subdir', st.asm_dir], capture_output=True, text=True) + o = r.stdout + r.stderr + return 'MATCH' in o and 'NOMATCH' not in o + + +def classify(drop, draft_text, tu_bodies, draft_bodies): + """-> (kind, detail). kind in {cosmetic, signedness, array, alias, refuse-*}""" + kept, this = drop['kept'], drop['this'] + mk, mt = SIG.match(kept), SIG.match(this) + if drop['symbol'] == drop['fn']: + # FUNCTION conflict. SIG only parses DATA declarations -- a function signature is + # ('void', ('s32', ...)), whose tail is a tuple, not a quoted string -- so every function + # conflict used to fall through to "DEF-SIDE-RETURN" even when only the PARAMETERS + # differed. That distinction decides whether the case is fixable: differing params yield + # to the §176d lever (adopt the TU's types, narrow inside the body), differing RETURNS + # usually cost the match (measured: `void` on func_8001ABBC). + kr, kp = _parse_fnsig(kept) + tr, tp = _parse_fnsig(this) + if kr is not None and kr == tr and kp != tp: + return 'defparams', (tp, kp) + return 'refuse-DEF-SIDE-RETURN', f'{this} vs TU {kept}' + if not (mk and mt): + return 'refuse-SIGNATURE', f'{this} vs {kept}' + kr, tr = mk['ret'].rstrip('*'), mt['ret'].rstrip('*') + # local struct types on both sides? + if kr in tu_bodies and tr in draft_bodies: + if _same_struct(kr, tu_bodies[kr], tr, draft_bodies[tr]): + return 'cosmetic', (tr, kr) + return 'refuse-DIFFERENT-STRUCT', f'{tr} and {kr} have different layouts' + if mk['tail'] != mt['tail']: + return 'array', (mt['ret'], mk['ret'], mk['tail']) + if gm._alias(mk['ret']) == gm._alias(mt['ret']): + return 'alias', (mt['ret'], mk['ret']) + if {kr, tr} <= {'u8', 's8', 'u16', 's16', 'u32', 's32'}: + return 'signedness', (mt['ret'], mk['ret']) + return 'refuse-TYPE', f'{mt["ret"]} vs {mk["ret"]}' + + +def repair(kind, detail, sym, path): + """Rewrite the draft. Returns True if the file changed.""" + src = open(path).read() + if kind == 'cosmetic': + old, new = detail + out = re.sub(r'\b%s\b' % re.escape(old), new, src) + elif kind in ('signedness', 'alias'): + old, new = detail + out = re.sub(r'(extern\s+)%s(\s+%s\b)' % (re.escape(old), re.escape(sym)), + r'\g<1>%s\g<2>' % new, src) + elif kind == 'defparams': + mine, theirs = detail + return _apply_tu_params(path, sym, mine, theirs) + elif kind == 'array': + _mine, tu_ret, tu_tail = detail + if tu_tail == '[]': # TU says array, draft says scalar + out = re.sub(r'(extern\s+[^;]*\b%s\b)\s*;' % re.escape(sym), r'\1[];', src) + out = re.sub(r'&%s\b' % re.escape(sym), sym, out) + else: # TU says scalar, draft says array + out = re.sub(r'(extern\s+[^;]*\b%s\b)\s*\[\s*\]\s*;' % re.escape(sym), r'\1;', src) + out = re.sub(r'(? {len(kept)} compatible, {len(dropped)} dropped') + if not dropped: + break + progress = 0 + for d in dropped: + path = next(e['draft'] for e in slate if e['fn'] == d['fn']) + text = open(path).read() + kind, detail = classify(d, text, tu_bodies, _bodies(text)) + if kind.startswith('refuse'): + refusals.append((d['fn'], d['symbol'], kind, detail)) + continue + if not a.apply: + print(f" would fix {d['fn']:22s} {d['symbol']:16s} {kind}") + progress += 1 + continue + before = text + if not repair(kind, detail, d['symbol'], path): + refusals.append((d['fn'], d['symbol'], 'refuse-NO-EDIT', str(detail))) + continue + if verify(d['fn'], path): + print(f" FIXED {d['fn']:22s} {d['symbol']:16s} {kind} (re-verified MATCH)") + fixed_total.append(d['fn']) + progress += 1 + else: + open(path, 'w').write(before) # a repair that moves a byte is not a repair + refusals.append((d['fn'], d['symbol'], 'refuse-BROKE-MATCH', kind)) + print(f" revert {d['fn']:22s} {d['symbol']:16s} {kind} broke the match") + if not a.apply or not progress: + break + + kept, dropped = gm.resolve_conflicts(slate) + print(f'\nFINAL: {len(kept)}/{len(slate)} compatible, {len(dropped)} dropped; ' + f'{len(fixed_total)} auto-reconciled') + if refusals: + print('\nNEEDS A HUMAN DECISION (not mechanical):') + seen = set() + for fn, sym, kind, detail in refusals: + if (fn, sym) in seen: + continue + seen.add((fn, sym)) + print(f' {kind:24s} {fn:22s} {sym:16s} {detail}') + json.dump([e for e in slate if e['fn'] in {k['fn'] for k in kept}], + open(a.slate.replace('.json', '_reconciled.json'), 'w'), indent=1) + print(f"\nreconciled slate -> {a.slate.replace('.json', '_reconciled.json')}") + + +if __name__ == '__main__': + main()