diff --git a/tools/eval_lora.py b/tools/eval_lora.py new file mode 100644 index 000000000..a501253ad --- /dev/null +++ b/tools/eval_lora.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""eval_lora.py — gate-true eval of a fine-tuned model on HELD-OUT banked functions. + +The held-out test pairs (datasets/match_pairs/test.jsonl) are banked, so they have no .s — but the +corpus carries each one's target asm (with reloc markers). So: feed the model the LEAN asm prompt +(same shape as training), compile its C with the pinned toolchain, objdump it, and compare to the +corpus target (relocation-masked, exactly like match_one). Same format on both sides (objdump) → no +train/eval format confound; functions the model never trained on → real generalization signal. + +Reports match / near / fail vs the stock-local floor (~0). Optional --iters feeds the diff back. + + LEAN=1 API_BASE=http://:1234/v1 MODEL=bfm-match-7b \ + .venv/bin/python tools/eval_lora.py --test datasets/match_pairs/test.jsonl --iters 3 +""" +import argparse, json, os, re, struct, subprocess, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import api_draft # call_api, build_user_lean, extract_code, LEAN_SYS (module-level argparse-free) + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CPP = 'mipsel-linux-gnu-cpp'; CC1 = 'tools/bin/gcc-2.7.2-psx/cc1'; MASPSX = 'tools/maspsx/maspsx.py' +AS = 'mipsel-linux-gnu-as'; OBJDUMP = 'mipsel-linux-gnu-objdump'; PYV = '.venv/bin/python' +CPPF = '-lang-c -Iinclude -undef -Wall -fno-builtin -Dmips -D__GNUC__=2 -D__OPTIMIZE__ -Dpsx -D_PSYQ -D_MIPSEL -D_LANGUAGE_C'.split() +CC1F = '-quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker'.split() +ASF = '-Iinclude -march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0'.split() +_TD = re.compile(r'^[ \t]*typedef\b.*\b(u8|u16|u32|u64|s8|s16|s32|s64|f32|f64)[ \t]*;[ \t]*\n', re.M) + + +def compile_obj(c, wd): + """pinned pipeline -> object path, or None on compile failure.""" + os.makedirs(wd, exist_ok=True) + open(os.path.join(wd, 't.c'), 'w').write('#include "common.h"\n' + _TD.sub('', c)) + p = subprocess.run([CPP] + CPPF + [os.path.join(wd, 't.c')], capture_output=True, cwd=REPO) + if p.returncode: return None + p = subprocess.run([CC1] + CC1F, input=p.stdout, capture_output=True, cwd=REPO) + if p.returncode: return None + p = subprocess.run([PYV, MASPSX, '--aspsx-version=2.56', '--expand-div'], input=p.stdout, capture_output=True, cwd=REPO) + if p.returncode: return None + p = subprocess.run([AS] + ASF + ['-o', os.path.join(wd, 't.o')], input=p.stdout, capture_output=True, cwd=REPO) + return os.path.join(wd, 't.o') if p.returncode == 0 else None + + +def my_insns(obj, fn): + d = subprocess.run([OBJDUMP, '-dr', '-j', '.text', obj], capture_output=True, text=True, cwd=REPO).stdout + words, masks, infn = [], [], False + for line in d.splitlines(): + h = re.match(r'^[0-9a-f]+ <([^>]+)>:', line) + if h: + infn = (h.group(1) == fn); continue + if not infn: continue + mi = re.match(r'\s+[0-9a-f]+:\s+([0-9a-f]{8})\s+', line) + if mi: + w = int(mi.group(1), 16); words.append(w) + masks.append(0 if (w >> 26) in (2, 3) else 0xFFFFFFFF) + elif 'R_MIPS' in line and words: + masks[-1] = 0 if '_26' in line else 0xFFFF0000 + return words, masks + + +def parse_target(asm): + """corpus asm (/* vaddr LEWORD */ mnem ; R_MIPS_*) -> (words, masks).""" + words, masks = [], [] + for line in asm.splitlines(): + m = re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})\s*\*/\s*(.*)', line) + if not m: continue + words.append(struct.unpack(' %d held-out fns, %d iters\n' % (api_draft.MODEL, api_draft.API_BASE, len(rows), a.iters)) + match = near = fail = 0 + for r in rows: + c = eval_one(r, a.iters, os.path.join(REPO, '.run/_eval_lora')) + tag = 'MATCH' if c == 0 else ('compile/near-fail' if c >= 10 ** 8 else 'near %d' % c) + if c == 0: match += 1 + elif c >= 10 ** 8: fail += 1 + else: near += 1 + print(' %-16s %s' % (r['fn'], tag)) + n = len(rows) + print('\n=== eval: MATCH %d/%d near %d fail %d (stock-local floor: ~0) ===' % (match, n, near, fail)) + + +if __name__ == '__main__': + main() diff --git a/tools/train_lora.py b/tools/train_lora.py index 29b1803a4..7123110c0 100644 --- a/tools/train_lora.py +++ b/tools/train_lora.py @@ -31,6 +31,7 @@ def main(): ap.add_argument('--out', default='models/bfm-match-7b') ap.add_argument('--rank', type=int, default=16) ap.add_argument('--epochs', type=float, default=3.0) + ap.add_argument('--max-steps', type=int, default=0, help='>0 overrides epochs (use for a smoke run)') ap.add_argument('--lr', type=float, default=2e-4) ap.add_argument('--maxlen', type=int, default=4096, help='token cap; longest pairs are ~giant asm') ap.add_argument('--no-gguf', action='store_true', help='skip the merged-GGUF export step') @@ -63,8 +64,9 @@ def main(): args=SFTConfig( dataset_text_field='text', max_seq_length=a.maxlen, per_device_train_batch_size=2, gradient_accumulation_steps=8, - warmup_ratio=0.03, num_train_epochs=a.epochs, learning_rate=a.lr, - logging_steps=10, optim='adamw_8bit', weight_decay=0.01, + warmup_ratio=0.03, num_train_epochs=a.epochs, + max_steps=(a.max_steps if a.max_steps > 0 else -1), learning_rate=a.lr, + logging_steps=1 if a.max_steps else 10, optim='adamw_8bit', weight_decay=0.01, lr_scheduler_type='cosine', seed=3407, output_dir=os.path.join(REPO, a.out, 'ckpt'))) trainer.train()