mirror/BFM-decomp
1
0
Fork 0
mirror of https://github.com/Druthulu/BFM-decomp synced 2026-09-28 06:49:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(tools): t5_distill_args takes repeated --wave/--wfdir pairs so one distill covers several waves; dedups by (binary,fn,arm) (R48) and skips un-gated waves loudly (P31 S63 T5.5)

Browse Source
This commit is contained in:
Drew T
2026-08-26 21:12:10 -06:00
parent c865841dbe
commit a197f45a9c
1 changed files with 46 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/t5_distill_args.py
+46 -27
View File
@@ -1,7 +1,10 @@
#!/usr/bin/env python3
"""tools/t5_distill_args.py — build the R16 distill slate for a Claude wave (P31 S63 T5).
Usage: t5_distill_args.py --wave .run/t5a --wfdir <workflow transcript dir> [--label 'P31 S63 t5a'] [--all-arms]
Usage: t5_distill_args.py --wave .run/t5a --wfdir <dir> [--wave .run/t5b --wfdir <dir> ...]
[--label 'P31 S63'] [--all-arms] [--out <path>]
--wave/--wfdir may be repeated and are paired POSITIONALLY (the Nth wave with the Nth wfdir), so one
distill run can cover several waves; the slates are concatenated and de-duplicated by (binary, fn).
Only BYTE-PROVEN transcripts are distilled (the distiller's contract: "the whole-binary byte-gate
ACCEPTED the final draft, so the final body is ground truth"), so the slate = the wave's banked set
@@ -16,39 +19,55 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); os.chdir(REP
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('--wave', required=True)
ap.add_argument('--wfdir', required=True, help="the drafting run's transcript dir (holds journal.jsonl + agent-*.jsonl)")
ap.add_argument('--wave', required=True, action='append',
help='wave dir; repeatable, paired positionally with --wfdir')
ap.add_argument('--wfdir', required=True, action='append',
help="the drafting run's transcript dir (holds journal.jsonl + agent-*.jsonl); repeatable")
ap.add_argument('--label', default='', help="provenance string for the cookbook header, e.g. 'P31 S63 t5a'")
ap.add_argument('--all-arms', action='store_true')
ap.add_argument('--out', default='')
a = ap.parse_args()
jd = json.load(open(os.path.join(a.wave, 'judge.json')))
banked_by_arm = {arm: set(v.get('banked', [])) for arm, v in jd.get('arms', {}).items()}
banked = set().union(*banked_by_arm.values()) if banked_by_arm else set()
union = set(jd.get('union', {}).get('banked', []))
picked = jd.get('union', {}).get('picked', {})
jp = os.path.join(a.wfdir, 'journal.jsonl')
if not os.path.exists(jp): sys.exit('no journal.jsonl in %s' % a.wfdir)
rows = [json.loads(l) for l in open(jp) if l.strip()]
tr = {}
for r in rows:
res = r.get('result') or {}
if isinstance(res, dict) and res.get('fn'):
tr[(res['fn'], res.get('arm'))] = (os.path.join(a.wfdir, 'agent-%s.jsonl' % r['agentId']), res.get('note', ''))
out, missing = [], []
for fn in sorted(banked):
arms = sorted(banked_by_arm) if a.all_arms else [picked.get(fn) or next(x for x in banked_by_arm if fn in banked_by_arm[x])]
for arm in arms:
if fn not in banked_by_arm.get(arm, ()): continue
hit = tr.get((fn, arm))
if not hit or not os.path.exists(hit[0]): missing.append('%s/%s' % (arm, fn)); continue
out.append({'fn': fn, 'arm': arm, 'transcript': hit[0], 'note': hit[1]})
print('banked %d (union-gated %d); transcripts found %d; label %r' % (len(banked), len(union), len(out), a.label))
if len(a.wave) != len(a.wfdir):
sys.exit('REFUSED: %d --wave but %d --wfdir; they pair positionally' % (len(a.wave), len(a.wfdir)))
out, missing, seen, tot_banked, tot_union = [], [], set(), 0, 0
for wave, wfdir in zip(a.wave, a.wfdir):
jp = os.path.join(wave, 'judge.json')
if not os.path.exists(jp):
print('SKIP %s: no judge.json (wave not gated yet)' % wave); continue
jd = json.load(open(jp))
banked_by_arm = {arm: set(v.get('banked', [])) for arm, v in jd.get('arms', {}).items()}
banked = set().union(*banked_by_arm.values()) if banked_by_arm else set()
union = set(jd.get('union', {}).get('banked', []))
picked = jd.get('union', {}).get('picked', {})
tot_banked += len(banked); tot_union += len(union)
T = {t['name']: t for t in json.load(open(os.path.join(wave, 'targets.json')))}
jl = os.path.join(wfdir, 'journal.jsonl')
if not os.path.exists(jl):
sys.exit('no journal.jsonl in %s' % wfdir)
tr = {}
for line in open(jl):
if '"result"' not in line: continue
r = json.loads(line); res = r.get('result') or {}
if isinstance(res, dict) and res.get('fn'):
tr[(res['fn'], res.get('arm'))] = (os.path.join(wfdir, 'agent-%s.jsonl' % r['agentId']), res.get('note', ''))
for fn in sorted(banked):
arms = sorted(banked_by_arm) if a.all_arms else [picked.get(fn) or next(x for x in banked_by_arm if fn in banked_by_arm[x])]
for arm in arms:
if fn not in banked_by_arm.get(arm, ()): continue
key = (T.get(fn, {}).get('binary', '?'), fn, arm)
if key in seen: continue # the same fn can be drawn by two waves (R48: key by binary too)
seen.add(key)
hit = tr.get((fn, arm))
if not hit or not os.path.exists(hit[0]):
missing.append('%s/%s/%s' % (os.path.basename(wave), arm, fn)); continue
out.append({'fn': fn, 'arm': arm, 'transcript': hit[0], 'note': hit[1]})
print('waves %d; banked %d (union-gated %d); transcripts found %d; label %r'
% (len(a.wave), tot_banked, tot_union, len(out), a.label))
if missing:
print('NO TRANSCRIPT for %d banked draft(s) — NOT distilled (R32, reported not skipped): %s'
% (len(missing), ' '.join(missing[:20])))
p = a.out or os.path.join(a.wave, 'distill_args.json')
json.dump({'targets': out, 'label': a.label or os.path.basename(a.wave.rstrip('/'))}, open(p, 'w'), indent=1)
p = a.out or os.path.join(a.wave[0], 'distill_args.json')
json.dump({'targets': out, 'label': a.label or os.path.basename(a.wave[0].rstrip('/'))}, open(p, 'w'), indent=1)
print('wrote %s' % p)
if __name__ == '__main__':
main()