mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
698f2959ae
* `binof = {c["fn"]: c["binary"]}` was last-writer-wins, and `status`, `det` and `subof`
had the same shape — a draft of a name carried by two binaries was stamped with
whichever card came last and then reloc-checked against the OTHER binary's symbols.
* Resolve per draft instead: the shard's own target list first
(`.run/wave_<tag>_targets.<i>.json` = `targets[i::workers]`, each row carrying its
binary), a unique-name card second, a counted refusal when neither can answer (R43).
* R39 negative control over every historical wave: 42,655 drafts, 0 regressions,
2,317 (5.4%) previously mis-stamped; 2,107 homonym card names fleet-wide.
Intra-shard ambiguity: 0 of 50,684 (shard, name) pairs over 302,370 shard files.
docs: §408 — §406 refuted as a sweep (0 MATCH / 14 applied, 0 / 210). The 134-member
census counted main's 960 LINKED library stubs and matched a symmetric SHAPE; derived
from the mine-vs-target residual the addressable set is 15 / 210. Decision-log entry
records the pivot: 64 of 210 (30.5%) already match standalone, so the frontier's
largest lane is §376 integration, not codegen.
tools/weave_sweep.py — the derived-selector sweep (R32 coverage, R41 denominators,
--lever-all ablation control).
242 lines
11 KiB
Python
242 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""§406 PROLOGUE-WEAVE SWEEP — one known lever, applied where the BYTES say it belongs.
|
|
|
|
THE CLASS (cookbook §406). gcc-2.7.2 sched2 sinks a function's prologue `sw $ra` down to just above
|
|
the first branch: bb0 is all constant-address MEMs, so `memrefs_conflict_p` (sched.c:614) finds no
|
|
dependence, `sw $ra` is ready early and `schedule_select`'s `potential_hazard` (memory beats ALU,
|
|
sched.c:2616) promotes it. The lever is a NON-volatile `__asm__("" : : : "memory")` as the first
|
|
STATEMENT of the body: the BLK clobber gives `sw $ra` a successor so it is only ready after the load
|
|
is picked, and it lands back in the prologue. Twelve other variants are measured INERT (§406).
|
|
|
|
SELECTION IS DERIVED, NOT PATTERN-MATCHED (R33/R34). The §406 census — "sw $s0 / move $s0,$a0 / sw
|
|
$ra in the first 24 lines of the target" — is the shape of the FAMILY, not of the DEFECT, and it is
|
|
symmetric: `main/func_8002EED8` has that shape with the target's `sw $ra` ALREADY SUNK, where this
|
|
lever pushes exactly the wrong way. The defect is a DISAGREEMENT, so we read it off the residual:
|
|
|
|
WEAVE-SUNK mine sinks `sw ra` BELOW where the target keeps it -> the lever applies
|
|
WEAVE-EARLY mine keeps it above where the target sank it -> the lever is BACKWARDS
|
|
(neither) the `sw $ra` slot is not the residual -> not this class
|
|
|
|
Stages: baseline-score every member's stored drafts -> classify from the residual -> apply the lever
|
|
to WEAVE-SUNK only -> re-score. Nothing here drafts; every body already exists.
|
|
|
|
python3 tools/weave_sweep.py --limit 5 # R37 probe
|
|
python3 tools/weave_sweep.py --members .run/S71_frontier.json # the whole real frontier
|
|
|
|
R32: every input member gets a row or an explicit reason, asserted at the end.
|
|
R41: every rate is printed with its denominator.
|
|
"""
|
|
import argparse, hashlib, json, os, re, subprocess, sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
CLOBBER = '__asm__("" : : : "memory");'
|
|
MINE_RA = re.compile(r'\bsw\s+ra\b') # objdump form (mine)
|
|
TGT_RA = re.compile(r'\bsw\s+\$ra\b') # splat .s form (target)
|
|
KEYWORD = re.compile(r'^\s*(return|if|while|for|do|switch|goto|break|continue|else|case|default)\b')
|
|
# A STATEMENT at body top level. Everything else that ends in ';' is treated as a declaration —
|
|
# including `register u8 *p __asm__("$16");`, which an earlier decl-side regex missed and so put the
|
|
# clobber ABOVE the declarations, where C89 rejects every one of them (cc1-fail on 2 of 5 probes).
|
|
STMT = re.compile(r'^\s*(?:[*(!&+\-{}]|'
|
|
r'[A-Za-z_]\w*\s*(?:\[|\(|\.|->|\+\+|--|=[^=]|[+\-*/%&|^]=|<<=|>>=))')
|
|
|
|
|
|
def score(fn, cfile, asm_dir, timeout=240):
|
|
cmd = ['python3', 'tools/match_one.py', fn, '--c', cfile, '--asm-subdir', asm_dir, '--json']
|
|
try:
|
|
p = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
return {'status': 'timeout', 'closeness': None, 'residual': []}
|
|
for line in reversed(p.stdout.strip().split('\n')):
|
|
line = line.strip()
|
|
if line.startswith('{'):
|
|
try:
|
|
return json.loads(line)
|
|
except ValueError:
|
|
pass
|
|
return {'status': 'no-json', 'closeness': None, 'residual': [],
|
|
'err': (p.stderr or p.stdout)[-200:]}
|
|
|
|
|
|
def is_match(s):
|
|
return (s or {}).get('status', '').lower() == 'match'
|
|
|
|
|
|
def classify(res):
|
|
"""Direction of the `sw $ra` disagreement, read off the residual [(i, mine, tgt), ...]."""
|
|
mi = next((i for i, me, tg in res if MINE_RA.search(me)), None)
|
|
ti = next((i for i, me, tg in res if TGT_RA.search(tg)), None)
|
|
if mi is None and ti is None:
|
|
return 'no-ra-residual'
|
|
if mi is None or ti is None:
|
|
return 'ra-one-sided' # one stream's sw ra moved into a slot the other matches
|
|
if mi > ti:
|
|
return 'WEAVE-SUNK' # mine below target -> the lever pulls it back up
|
|
if mi < ti:
|
|
return 'WEAVE-EARLY' # mine above target -> the lever is backwards
|
|
return 'ra-same-index'
|
|
|
|
|
|
def insert_clobber(src, fn, skip=0):
|
|
"""Clobber as the first statement of fn's body, skipping `skip` further candidate lines."""
|
|
if CLOBBER.replace(' ', '') in src.replace(' ', ''):
|
|
return None, 'already-has-clobber'
|
|
m = re.search(r'^[^\n;#]*?\b%s\s*\([^;{]*\)\s*\{' % re.escape(fn), src, re.M | re.S)
|
|
if not m:
|
|
return None, 'no-definition'
|
|
lines = src.split('\n')
|
|
i = src[:m.end()].count('\n') + 1
|
|
left = skip
|
|
while i < len(lines):
|
|
s = lines[i].strip()
|
|
if not s or s.startswith('//') or s.startswith('/*') or s.startswith('*/') or s.startswith('*'):
|
|
i += 1; continue
|
|
if s == '}':
|
|
return None, 'empty-body'
|
|
if STMT.match(lines[i]) or KEYWORD.match(lines[i]):
|
|
if left == 0:
|
|
break
|
|
left -= 1
|
|
i += 1
|
|
if i >= len(lines):
|
|
return None, 'no-statement'
|
|
ind = re.match(r'\s*', lines[i]).group(0) or ' '
|
|
body = lines[:i] + [
|
|
'%s/* §406: non-volatile memory clobber. The prologue `sw $ra` has no memory' % ind,
|
|
'%s * dependence in bb0, so sched2 promotes it over the ALU candidates and sinks' % ind,
|
|
'%s * it above the branch; the BLK clobber gives it a successor. */' % ind,
|
|
ind + CLOBBER, ''] + lines[i:]
|
|
return '\n'.join(body), None
|
|
|
|
|
|
def dedup(paths, cap):
|
|
seen, out = set(), []
|
|
for p in paths:
|
|
try:
|
|
h = hashlib.sha1(open(p, 'rb').read()).hexdigest()
|
|
except OSError:
|
|
continue
|
|
if h in seen:
|
|
continue
|
|
seen.add(h); out.append(p)
|
|
if len(out) >= cap:
|
|
break
|
|
return out
|
|
|
|
|
|
def run_member(m, workdir, cap, lever_all):
|
|
fn, asm_dir = m['symbol'], m['asm_dir']
|
|
r = dict(binary=m['binary'], addr=m['addr'], fn=fn, asm_dir=asm_dir, lever=None, klass=None)
|
|
drafts = dedup(m.get('drafts', []), cap)
|
|
r['ndrafts_scored'] = len(drafts)
|
|
if not drafts:
|
|
r['baseline'] = None; r['reason'] = 'no-draft-file'; return r
|
|
best = None
|
|
for d in drafts:
|
|
s = score(fn, d, asm_dir)
|
|
c = s.get('closeness')
|
|
key = c if isinstance(c, (int, float)) else 10 ** 9
|
|
if best is None or key < best[0]:
|
|
best = (key, d, s)
|
|
if is_match(s):
|
|
break
|
|
_, bd, bs = best
|
|
r['baseline'] = {'draft': bd, 'status': bs.get('status'), 'closeness': bs.get('closeness'),
|
|
'residual_n': len(bs.get('residual') or []), 'nins': bs.get('nins')}
|
|
if is_match(bs):
|
|
r['klass'] = 'baseline-MATCH'; r['reason'] = 'standalone match — an integration target, '\
|
|
'not a lever target (a standalone match is not a bank)'
|
|
return r
|
|
r['klass'] = classify(bs.get('residual') or [])
|
|
if r['klass'] != 'WEAVE-SUNK' and not lever_all:
|
|
r['reason'] = 'not-weave-sunk'; return r
|
|
os.makedirs(workdir, exist_ok=True)
|
|
src = open(bd, errors='replace').read()
|
|
for skip in range(0, 4): # self-correcting: walk past a mis-read decl
|
|
new, why = insert_clobber(src, fn, skip)
|
|
if new is None:
|
|
r['reason'] = 'lever-not-applicable:' + why; return r
|
|
lp = os.path.join(workdir, '%s.%s.lever%d.c' % (m['binary'], fn, skip))
|
|
open(lp, 'w').write(new)
|
|
s = score(fn, lp, asm_dir)
|
|
if s.get('status') != 'cc1-fail':
|
|
r['lever'] = {'draft': lp, 'skip': skip, 'status': s.get('status'),
|
|
'closeness': s.get('closeness')}
|
|
return r
|
|
r['reason'] = 'lever-cc1-fail-all-positions'
|
|
return r
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument('--members', default='.run/S71_frontier.json')
|
|
ap.add_argument('--out', default='.run/S71_weave.jsonl')
|
|
ap.add_argument('--workdir', default='.run/S71_weave')
|
|
ap.add_argument('--limit', type=int, default=0)
|
|
ap.add_argument('--only', default=None, help='comma-separated binary:0xADDR')
|
|
ap.add_argument('--drafts-per-fn', type=int, default=4)
|
|
ap.add_argument('--workers', type=int, default=12)
|
|
ap.add_argument('--lever-all', action='store_true',
|
|
help='apply the lever regardless of class (the ABLATION control: it must do '
|
|
'worse on WEAVE-EARLY than on WEAVE-SUNK, or the classifier is noise)')
|
|
a = ap.parse_args()
|
|
|
|
members = json.load(open(a.members))
|
|
if a.only:
|
|
want = {(t.split(':')[0], int(t.split(':')[1], 0)) for t in a.only.split(',')}
|
|
members = [m for m in members if (m['binary'], m['addr']) in want]
|
|
if a.limit:
|
|
seen, first, rest = set(), [], []
|
|
for m in members:
|
|
(first if m['binary'] not in seen else rest).append(m); seen.add(m['binary'])
|
|
members = (first + rest)[:a.limit]
|
|
n_in = len(members)
|
|
print('weave_sweep: %d members / %d workers / <=%d drafts each%s'
|
|
% (n_in, a.workers, a.drafts_per_fn, ' / LEVER-ALL ablation' if a.lever_all else ''),
|
|
flush=True)
|
|
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=a.workers) as ex:
|
|
futs = [ex.submit(run_member, m, a.workdir, a.drafts_per_fn, a.lever_all) for m in members]
|
|
for k, f in enumerate(futs, 1):
|
|
r = f.result(); results.append(r)
|
|
b = r['baseline'] or {}; l = r['lever'] or {}
|
|
print('[%3d/%d] %-14s %-16s base=%s/%-5s %-15s lever=%s/%s %s'
|
|
% (k, n_in, r['binary'], r['fn'], b.get('status'), b.get('closeness'),
|
|
r.get('klass'), l.get('status'), l.get('closeness'), r.get('reason', '')),
|
|
flush=True)
|
|
with open(a.out, 'w') as fh:
|
|
for r in results:
|
|
fh.write(json.dumps(r) + '\n')
|
|
assert len(results) == n_in, 'COVERAGE: %d rows for %d members' % (len(results), n_in)
|
|
|
|
from collections import Counter
|
|
cls = Counter(r['klass'] for r in results)
|
|
tried = [r for r in results if r['lever']]
|
|
hit = [r for r in tried if is_match(r['lever'])]
|
|
better = [r for r in tried
|
|
if isinstance(r['lever'].get('closeness'), (int, float))
|
|
and isinstance((r['baseline'] or {}).get('closeness'), (int, float))
|
|
and r['lever']['closeness'] < r['baseline']['closeness']]
|
|
print('\n== §406 weave sweep (R41: rates carry their denominators) ==')
|
|
print('members in : %d' % n_in)
|
|
for k, v in cls.most_common():
|
|
print(' class %-18s : %d / %d' % (k, v, n_in))
|
|
print('lever applied : %d / %d' % (len(tried), n_in))
|
|
if tried:
|
|
print('lever MATCH : %d / %d applied (%d / %d members)'
|
|
% (len(hit), len(tried), len(hit), n_in))
|
|
print('lever improved, still near : %d / %d applied' % (len(better), len(tried)))
|
|
for r in hit:
|
|
print(' MATCH %-14s %s' % (r['binary'], r['fn']))
|
|
for r in results:
|
|
if str(r.get('reason', '')).startswith('lever-'):
|
|
print(' skipped %-14s %-16s %s' % (r['binary'], r['fn'], r['reason']))
|
|
print('rows -> %s' % a.out)
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|