mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
feat(phase-22): eval_lora held-out scorer (validated) + train_lora --max-steps
eval_lora.py: feeds a fine-tuned model the LEAN asm of held-out BANKED test fns, compiles its C, compares to the corpus target reloc-masked (banked fns have no .s — corpus carries the asm). Validated: known-good corpus C scores MATCH vs its own asm. train_lora --max-steps for smoke runs. GPU confirmed (3080 Ti, torch 2.10+cu128); install + corpus-filter (638/75 clean pairs) done; smoke-train running.
This commit is contained in:
Drew T
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""eval_lora.py — gate-true eval of a fine-tuned model on HELD-OUT banked functions.
|
||||
|
||||
The held-out test pairs (datasets/match_pairs/test.jsonl) are banked, so they have no .s — but the
|
||||
corpus carries each one's target asm (with reloc markers). So: feed the model the LEAN asm prompt
|
||||
(same shape as training), compile its C with the pinned toolchain, objdump it, and compare to the
|
||||
corpus target (relocation-masked, exactly like match_one). Same format on both sides (objdump) → no
|
||||
train/eval format confound; functions the model never trained on → real generalization signal.
|
||||
|
||||
Reports match / near / fail vs the stock-local floor (~0). Optional --iters feeds the diff back.
|
||||
|
||||
LEAN=1 API_BASE=http://<host>:1234/v1 MODEL=bfm-match-7b \
|
||||
.venv/bin/python tools/eval_lora.py --test datasets/match_pairs/test.jsonl --iters 3
|
||||
"""
|
||||
import argparse, json, os, re, struct, subprocess, sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import api_draft # call_api, build_user_lean, extract_code, LEAN_SYS (module-level argparse-free)
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
CPP = 'mipsel-linux-gnu-cpp'; CC1 = 'tools/bin/gcc-2.7.2-psx/cc1'; MASPSX = 'tools/maspsx/maspsx.py'
|
||||
AS = 'mipsel-linux-gnu-as'; OBJDUMP = 'mipsel-linux-gnu-objdump'; PYV = '.venv/bin/python'
|
||||
CPPF = '-lang-c -Iinclude -undef -Wall -fno-builtin -Dmips -D__GNUC__=2 -D__OPTIMIZE__ -Dpsx -D_PSYQ -D_MIPSEL -D_LANGUAGE_C'.split()
|
||||
CC1F = '-quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker'.split()
|
||||
ASF = '-Iinclude -march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0'.split()
|
||||
_TD = re.compile(r'^[ \t]*typedef\b.*\b(u8|u16|u32|u64|s8|s16|s32|s64|f32|f64)[ \t]*;[ \t]*\n', re.M)
|
||||
|
||||
|
||||
def compile_obj(c, wd):
|
||||
"""pinned pipeline -> object path, or None on compile failure."""
|
||||
os.makedirs(wd, exist_ok=True)
|
||||
open(os.path.join(wd, 't.c'), 'w').write('#include "common.h"\n' + _TD.sub('', c))
|
||||
p = subprocess.run([CPP] + CPPF + [os.path.join(wd, 't.c')], capture_output=True, cwd=REPO)
|
||||
if p.returncode: return None
|
||||
p = subprocess.run([CC1] + CC1F, input=p.stdout, capture_output=True, cwd=REPO)
|
||||
if p.returncode: return None
|
||||
p = subprocess.run([PYV, MASPSX, '--aspsx-version=2.56', '--expand-div'], input=p.stdout, capture_output=True, cwd=REPO)
|
||||
if p.returncode: return None
|
||||
p = subprocess.run([AS] + ASF + ['-o', os.path.join(wd, 't.o')], input=p.stdout, capture_output=True, cwd=REPO)
|
||||
return os.path.join(wd, 't.o') if p.returncode == 0 else None
|
||||
|
||||
|
||||
def my_insns(obj, fn):
|
||||
d = subprocess.run([OBJDUMP, '-dr', '-j', '.text', obj], capture_output=True, text=True, cwd=REPO).stdout
|
||||
words, masks, infn = [], [], False
|
||||
for line in d.splitlines():
|
||||
h = re.match(r'^[0-9a-f]+ <([^>]+)>:', line)
|
||||
if h:
|
||||
infn = (h.group(1) == fn); continue
|
||||
if not infn: continue
|
||||
mi = re.match(r'\s+[0-9a-f]+:\s+([0-9a-f]{8})\s+', line)
|
||||
if mi:
|
||||
w = int(mi.group(1), 16); words.append(w)
|
||||
masks.append(0 if (w >> 26) in (2, 3) else 0xFFFFFFFF)
|
||||
elif 'R_MIPS' in line and words:
|
||||
masks[-1] = 0 if '_26' in line else 0xFFFF0000
|
||||
return words, masks
|
||||
|
||||
|
||||
def parse_target(asm):
|
||||
"""corpus asm (/* vaddr LEWORD */ mnem ; R_MIPS_*) -> (words, masks)."""
|
||||
words, masks = [], []
|
||||
for line in asm.splitlines():
|
||||
m = re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})\s*\*/\s*(.*)', line)
|
||||
if not m: continue
|
||||
words.append(struct.unpack('<I', bytes.fromhex(m.group(1)))[0])
|
||||
masks.append(0 if 'R_MIPS' in m.group(2) and '_26' in m.group(2)
|
||||
else (0xFFFF0000 if 'R_MIPS' in m.group(2) else 0xFFFFFFFF))
|
||||
return words, masks
|
||||
|
||||
|
||||
def diff(mw, mmask, tw, tmask):
|
||||
n = max(len(mw), len(tw)); ds = []
|
||||
for i in range(n):
|
||||
me = (mw[i] & mmask[i]) if i < len(mw) else None
|
||||
tg = (tw[i] & tmask[i]) if i < len(tw) else None
|
||||
if me != tg:
|
||||
ds.append('idx %d: mine=%s target=%s' % (i, '%08x' % mw[i] if i < len(mw) else '--',
|
||||
'%08x' % tw[i] if i < len(tw) else '--'))
|
||||
return ds
|
||||
|
||||
|
||||
def eval_one(r, iters, wd):
|
||||
fn, asm = r['fn'], r['asm']
|
||||
tw, tmask = parse_target(asm)
|
||||
msgs = [{'role': 'system', 'content': api_draft.LEAN_SYS},
|
||||
{'role': 'user', 'content': api_draft.build_user_lean(r, asm, '')}]
|
||||
best = 10 ** 9
|
||||
for i in range(max(1, iters)):
|
||||
code = api_draft.extract_code(api_draft.call_api(msgs))
|
||||
if not code.strip(): break
|
||||
obj = compile_obj(code, wd)
|
||||
if not obj:
|
||||
ds = ['compile-fail']; close = 10 ** 8
|
||||
else:
|
||||
mw, mmask = my_insns(obj, fn)
|
||||
mismatch = diff(mw, mmask, tw, tmask)
|
||||
close = 0 if (not mismatch and len(mw) == len(tw)) else (len(mismatch) or 1)
|
||||
ds = mismatch
|
||||
best = min(best, close)
|
||||
if close == 0: return 0
|
||||
if i < iters - 1:
|
||||
msgs += [{'role': 'assistant', 'content': '```c\n' + code + '```'},
|
||||
{'role': 'user', 'content': 'Not byte-identical. Diff:\n' + '\n'.join(ds[:40])
|
||||
+ '\nFix and reply with the corrected full function in ONE ```c block.'}]
|
||||
return best
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--test', default='datasets/match_pairs/test.jsonl')
|
||||
ap.add_argument('--iters', type=int, default=3)
|
||||
ap.add_argument('--limit', type=int, default=0)
|
||||
a = ap.parse_args()
|
||||
rows = [json.loads(l) for l in open(os.path.join(REPO, a.test))]
|
||||
if a.limit: rows = rows[:a.limit]
|
||||
print('eval_lora: %s @ %s -> %d held-out fns, %d iters\n' % (api_draft.MODEL, api_draft.API_BASE, len(rows), a.iters))
|
||||
match = near = fail = 0
|
||||
for r in rows:
|
||||
c = eval_one(r, a.iters, os.path.join(REPO, '.run/_eval_lora'))
|
||||
tag = 'MATCH' if c == 0 else ('compile/near-fail' if c >= 10 ** 8 else 'near %d' % c)
|
||||
if c == 0: match += 1
|
||||
elif c >= 10 ** 8: fail += 1
|
||||
else: near += 1
|
||||
print(' %-16s %s' % (r['fn'], tag))
|
||||
n = len(rows)
|
||||
print('\n=== eval: MATCH %d/%d near %d fail %d (stock-local floor: ~0) ===' % (match, n, near, fail))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+4
-2
@@ -31,6 +31,7 @@ def main():
|
||||
ap.add_argument('--out', default='models/bfm-match-7b')
|
||||
ap.add_argument('--rank', type=int, default=16)
|
||||
ap.add_argument('--epochs', type=float, default=3.0)
|
||||
ap.add_argument('--max-steps', type=int, default=0, help='>0 overrides epochs (use for a smoke run)')
|
||||
ap.add_argument('--lr', type=float, default=2e-4)
|
||||
ap.add_argument('--maxlen', type=int, default=4096, help='token cap; longest pairs are ~giant asm')
|
||||
ap.add_argument('--no-gguf', action='store_true', help='skip the merged-GGUF export step')
|
||||
@@ -63,8 +64,9 @@ def main():
|
||||
args=SFTConfig(
|
||||
dataset_text_field='text', max_seq_length=a.maxlen,
|
||||
per_device_train_batch_size=2, gradient_accumulation_steps=8,
|
||||
warmup_ratio=0.03, num_train_epochs=a.epochs, learning_rate=a.lr,
|
||||
logging_steps=10, optim='adamw_8bit', weight_decay=0.01,
|
||||
warmup_ratio=0.03, num_train_epochs=a.epochs,
|
||||
max_steps=(a.max_steps if a.max_steps > 0 else -1), learning_rate=a.lr,
|
||||
logging_steps=1 if a.max_steps else 10, optim='adamw_8bit', weight_decay=0.01,
|
||||
lr_scheduler_type='cosine', seed=3407,
|
||||
output_dir=os.path.join(REPO, a.out, 'ckpt')))
|
||||
trainer.train()
|
||||
|
||||
Reference in New Issue
Block a user