mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-28 14:59:48 -04:00
feat(phase-22): lora_grind.py — the free local-model mass-run (draft source for the grinder flywheel)
Rotates over every binary with open small stubs, drafts with the fine-tuned model (api_draft LEAN, GPU via LM Studio), banks via gate_stage, propagates fleet-wide periodically. Near-misses -> backlog -> grinder.py permuter closes regalloc/schedule residuals (synergy). Writes a classified near-miss histogram (the 'missing idioms' signal -> corpus-v3 priorities). STOP/heartbeat/stats like grinder.py. Smoke OK; histogram shows struct(49)+schedule/regalloc(94) dominate the backlog.
This commit is contained in:
Drew T
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lora_grind.py — the free local-model MASS-RUN (the LoRA draft source for the grinder flywheel).
|
||||
|
||||
Rotates over every binary with open small stubs, drafts each with the fine-tuned local model
|
||||
(tools/api_draft.py LEAN, served by LM Studio on the GPU), and banks the byte-matches through the
|
||||
shared gate_stage (sole arbiter, G3/P9). Near-misses land in the backlog -> grinder.py (the permuter)
|
||||
closes the regalloc/schedule residuals the LoRA leaves -> SYNERGY. Banks grow the corpus for the next
|
||||
retrain; the classified near-miss histogram (written to the stats file) is the readout of which idioms
|
||||
the model still misses -> the corpus-v3/v4 priority list (the data-driven flywheel).
|
||||
|
||||
LLM cost: $0 (local). Runs unattended for days alongside grinder.py under auto_supervisor.sh.
|
||||
SAFE EXIT: touch .run/auto/STOP (tools/auto_stop.sh) — finishes the current batch+gate, exits 0.
|
||||
HEARTBEAT: .run/auto/lora_grind_heartbeat.json. STATS: .run/auto/lora_grind_stats.json.
|
||||
|
||||
Env: API_BASE, MODEL (the served fine-tuned model). Usage:
|
||||
API_BASE=http://192.168.1.113:1234/v1 MODEL=bfm-local/bfm-match-7b-v2 \
|
||||
tools/lora_grind.py [--max-nins 15] [--batch 12] [--iters 3] [--propagate-every 8] [--once]
|
||||
"""
|
||||
import argparse, glob, json, os, re, subprocess, sys, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gate_stage, backlog
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
AUTODIR = os.path.join(REPO, ".run/auto")
|
||||
STOP = os.path.join(AUTODIR, "STOP")
|
||||
HB = os.path.join(AUTODIR, "lora_grind_heartbeat.json")
|
||||
STATS = os.path.join(AUTODIR, "lora_grind_stats.json")
|
||||
TRIED = os.path.join(AUTODIR, "lora_grind_tried.json")
|
||||
DRAFTS = os.path.join(AUTODIR, "lora_grind_drafts")
|
||||
STUB_RE = re.compile(r'INCLUDE_ASM\([^,]+,\s*(func_[0-9A-Fa-f]+)\)')
|
||||
|
||||
|
||||
def log(m):
|
||||
print("[lora_grind] " + m, flush=True)
|
||||
|
||||
|
||||
def binaries():
|
||||
"""every binary with a locked good-sha (config/check.<b>.sha)."""
|
||||
return sorted(os.path.basename(f)[6:-4] for f in glob.glob(os.path.join(REPO, "config/check.*.sha")))
|
||||
|
||||
|
||||
def good_sha(b):
|
||||
p = os.path.join(REPO, "config/check.%s.sha" % b)
|
||||
return open(p).read().strip() if os.path.exists(p) else None
|
||||
|
||||
|
||||
def nins(s_path):
|
||||
return sum(1 for l in open(s_path)
|
||||
if re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+[0-9A-Fa-f]+\s+[0-9A-Fa-f]{8}\s*\*/', l))
|
||||
|
||||
|
||||
def open_stubs(b, max_nins, tried):
|
||||
"""still-INCLUDE_ASM funcs for binary b (main + split .c) whose .s exists and is <= max_nins ins."""
|
||||
stubbed = set()
|
||||
for cf in glob.glob(os.path.join(REPO, "src/%s/%s*.c" % (b, b))):
|
||||
stubbed |= set(STUB_RE.findall(open(cf).read()))
|
||||
out = []
|
||||
for fn in stubbed:
|
||||
if fn in tried:
|
||||
continue
|
||||
for sd in glob.glob(os.path.join(REPO, "asm/%s/nonmatchings/*/%s.s" % (b, fn))):
|
||||
n = nins(sd)
|
||||
if 0 < n <= max_nins:
|
||||
out.append({"name": fn, "addr": "0x" + fn[5:].lower(), "nins": n,
|
||||
"class": "WAVE", "asm": os.path.relpath(sd, REPO), "ghidra_c": ""})
|
||||
break
|
||||
out.sort(key=lambda t: t["nins"])
|
||||
return out
|
||||
|
||||
|
||||
def draft(targets, api_base, model, iters):
|
||||
"""run the fine-tuned model (LEAN) over a batch -> DRAFTS dir; returns the dir."""
|
||||
os.makedirs(DRAFTS, exist_ok=True)
|
||||
for f in glob.glob(os.path.join(DRAFTS, "*.c")):
|
||||
os.remove(f)
|
||||
tf = os.path.join(AUTODIR, "lora_grind_targets.json")
|
||||
json.dump(targets, open(tf, "w"))
|
||||
env = dict(os.environ, LEAN="1", NORMALIZE_ASM="0", API_BASE=api_base, MODEL=model, TEMP="0.2")
|
||||
subprocess.run([".venv/bin/python", "tools/api_draft.py", "--targets", tf,
|
||||
"--out", os.path.relpath(DRAFTS, REPO), "--iters", str(iters)],
|
||||
cwd=REPO, env=env, capture_output=True, timeout=3600)
|
||||
return DRAFTS
|
||||
|
||||
|
||||
def gate_binary(b, draftdir):
|
||||
"""gate_stage the drafts for binary b — main .c, then any split (_a/_o0) with --src-file.
|
||||
propagate=False here (the loop runs a periodic propagate sweep); commit=True banks to git."""
|
||||
banked = []
|
||||
gs = good_sha(b)
|
||||
if not gs:
|
||||
return banked
|
||||
# main
|
||||
r = gate_stage.run_gate(os.path.relpath(draftdir, REPO), binary=b, good_sha=gs,
|
||||
propagate=False, source_tag="lora-grind", commit=True)
|
||||
banked += r.get("verified", [])
|
||||
# split files
|
||||
for split in glob.glob(os.path.join(REPO, "src/%s/%s_*.c" % (b, b))):
|
||||
name = os.path.basename(split)[:-2] # e.g. ov_SC01_077_a
|
||||
sub = "asm/%s/nonmatchings/%s" % (b, name)
|
||||
if not os.path.isdir(os.path.join(REPO, sub)):
|
||||
continue
|
||||
r = gate_stage.run_gate(os.path.relpath(draftdir, REPO), binary=b,
|
||||
src="src/%s/%s.c" % (b, name), asm=sub, good_sha=gs,
|
||||
propagate=False, source_tag="lora-grind", commit=True,
|
||||
src_file=name + ".c")
|
||||
banked += r.get("verified", [])
|
||||
return banked
|
||||
|
||||
|
||||
def near_class_hist():
|
||||
"""the backlog's open near-miss histogram by residual class — the 'missing idioms' signal."""
|
||||
import collections
|
||||
h = collections.Counter()
|
||||
for r in backlog.load_best():
|
||||
if r.get("status") == "near":
|
||||
h[(r.get("klass") or "other").split()[0]] += 1
|
||||
return dict(h)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--max-nins", type=int, default=15)
|
||||
ap.add_argument("--batch", type=int, default=12)
|
||||
ap.add_argument("--iters", type=int, default=3)
|
||||
ap.add_argument("--propagate-every", type=int, default=8, help="run a fleet propagate sweep every N batches")
|
||||
ap.add_argument("--idle-secs", type=int, default=120)
|
||||
ap.add_argument("--binaries", default=None, help="comma-list to restrict (default: all)")
|
||||
ap.add_argument("--max-batches", type=int, default=0, help="stop after N batches (0 = unbounded)")
|
||||
ap.add_argument("--once", action="store_true")
|
||||
a = ap.parse_args()
|
||||
only = set(a.binaries.split(",")) if a.binaries else None
|
||||
api_base = os.environ.get("API_BASE")
|
||||
model = os.environ.get("MODEL")
|
||||
if not api_base or not model:
|
||||
log("set API_BASE and MODEL (the served fine-tuned model)"); sys.exit(2)
|
||||
os.makedirs(AUTODIR, exist_ok=True)
|
||||
tried = set(json.load(open(TRIED))) if os.path.exists(TRIED) else set()
|
||||
total_banked = 0
|
||||
batch_i = 0
|
||||
propagated_since = set()
|
||||
|
||||
while True:
|
||||
if os.path.exists(STOP):
|
||||
log("STOP — exiting"); break
|
||||
did_work = False
|
||||
for b in binaries():
|
||||
if os.path.exists(STOP):
|
||||
break
|
||||
if only and b not in only:
|
||||
continue
|
||||
if a.max_batches and batch_i >= a.max_batches:
|
||||
break
|
||||
stubs = open_stubs(b, a.max_nins, tried)
|
||||
if not stubs:
|
||||
continue
|
||||
did_work = True
|
||||
batch = stubs[:a.batch]
|
||||
log("%s: drafting %d/%d open <=%d-ins stubs" % (b, len(batch), len(stubs), a.max_nins))
|
||||
draft(batch, api_base, model, a.iters)
|
||||
banked = gate_binary(b, DRAFTS)
|
||||
tried |= {t["name"] for t in batch}
|
||||
json.dump(sorted(tried), open(TRIED, "w"))
|
||||
total_banked += len(banked)
|
||||
propagated_since.add(b)
|
||||
batch_i += 1
|
||||
if banked:
|
||||
log("%s: BANKED %d (%s) [total %d]" % (b, len(banked), " ".join(banked), total_banked))
|
||||
# periodic fleet propagate sweep (the multiplier) — every N batches
|
||||
if batch_i % a.propagate_every == 0 and propagated_since:
|
||||
for pb in sorted(propagated_since):
|
||||
subprocess.run([".venv/bin/python", "tools/dedup_propagate.py", "--auto-from", pb,
|
||||
"--min-reach", "2"], cwd=REPO, capture_output=True, timeout=3600)
|
||||
propagated_since.clear()
|
||||
log("propagate sweep done")
|
||||
# heartbeat + flywheel stats
|
||||
fp = None
|
||||
try:
|
||||
rr = subprocess.run([".venv/bin/python", "tools/progress.py", "--fleet"],
|
||||
cwd=REPO, capture_output=True, text=True, timeout=120)
|
||||
mm = re.search(r'byte-identical\s+:\s+\d+\s*/\s*\d+\s*=\s*([\d.]+)%', rr.stdout)
|
||||
fp = float(mm.group(1)) if mm else None
|
||||
except Exception:
|
||||
pass
|
||||
json.dump({"ts": int(time.time()), "binary": b, "total_banked": total_banked,
|
||||
"fleet_pct": fp, "tried": len(tried)}, open(HB, "w"))
|
||||
json.dump({"total_banked": total_banked, "tried": len(tried), "fleet_pct": fp,
|
||||
"near_by_class": near_class_hist()}, open(STATS, "w"), indent=1)
|
||||
if a.once or not did_work or (a.max_batches and batch_i >= a.max_batches):
|
||||
log("stopping (--once / --max-batches / no work); banked %d total" % total_banked)
|
||||
break
|
||||
return total_banked
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user