mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-27 22:45:39 -04:00
feat(phase-29): tools/verify_map_findings.py — the codegen-map audit fabrication check (R3)
Moved out of .run/ (where the blanket ignore would have lost it) into tools/ per R3, and generalised
from regalloc to any docs/gcc-2.7.2-map/* audit.
The check an LLM audit of a source-derived document cannot do for itself: agents return
{real_file, real_line, source_quote}; this re-opens each file at each line and compares the verbatim
quote to what is actually there. Whitespace-normalised, +/-40-line search window, and it reports
NEAR (quote real, line wrong) SEPARATELY from FABRICATED (text appears nowhere) — because with the
2.8.1->2.7.2 drift reaching +611 lines in reload1.c, a wrong lookup lands INSIDE A DIFFERENT FUNCTION
and every sentence built on it still reads plausibly.
Used on the T34 regalloc.md audit: 184 findings, 0 FABRICATED.
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_map_findings.py — machine-check a codegen-map audit's findings against the bytes.
|
||||
|
||||
The regalloc.md re-derivation asks agents to classify each claim CONFIRMED / LINE-DRIFT / REFUTED /
|
||||
UNVERIFIABLE and to justify it with `real_file` + `real_line` + `source_quote`. An agent's verdict is
|
||||
a CLAIM, not evidence (R14) — and the specific failure mode this guards against is the one that makes
|
||||
a false REFUTED nearly undetectable by reading: gcc 2.8.1 -> 2.7.2 drifts by +300..+600 lines in
|
||||
reload1.c / local-alloc.c, so a naive line lookup lands inside a DIFFERENT FUNCTION and every
|
||||
subsequent sentence reads plausibly.
|
||||
|
||||
So: re-open each cited file at each cited line and compare the agent's verbatim quote to what is
|
||||
actually there. A quote that does not match its line invalidates the finding.
|
||||
|
||||
Comparison is whitespace-normalised (agents reflow leading tabs) but otherwise exact. A quote is also
|
||||
searched for in a +/-40 line window so a NEAR-MISS (right code, wrong line) is reported separately
|
||||
from a FABRICATION (text appears nowhere near) — those two need very different responses.
|
||||
|
||||
python3 tools/verify_map_findings.py <journal.jsonl|result.json>
|
||||
|
||||
Generic over docs/gcc-2.7.2-map/*: any audit whose agents return {real_file, real_line, source_quote}.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
SRC = os.path.join(REPO, "tools/reference/gcc-2.7.2")
|
||||
WINDOW = 40
|
||||
|
||||
|
||||
def norm(s):
|
||||
return re.sub(r"\s+", " ", (s or "")).strip()
|
||||
|
||||
|
||||
def load_findings(path):
|
||||
"""Accept either the workflow's return value or the raw journal."""
|
||||
txt = open(path, errors="replace").read()
|
||||
out = []
|
||||
# journal: one JSON object per line, agent results under .result
|
||||
for line in txt.splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
o = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
r = o.get("result", o)
|
||||
if isinstance(r, dict) and isinstance(r.get("findings"), list):
|
||||
for f in r["findings"]:
|
||||
f = dict(f)
|
||||
f.setdefault("_chunk", r.get("chunk", "?"))
|
||||
out.append(f)
|
||||
if out:
|
||||
return out
|
||||
# otherwise: a single result blob containing .findings
|
||||
i = txt.find('{"counts"')
|
||||
if i < 0:
|
||||
i = txt.find("{")
|
||||
try:
|
||||
blob = json.loads(txt[i:])
|
||||
except Exception:
|
||||
return []
|
||||
return blob.get("findings", [])
|
||||
|
||||
|
||||
def check(f):
|
||||
st = f.get("status")
|
||||
if st == "UNVERIFIABLE":
|
||||
return "SKIP", "declared unverifiable"
|
||||
rf, rl, q = f.get("real_file"), f.get("real_line"), f.get("source_quote")
|
||||
if not (rf and rl and q):
|
||||
return "NO-EVIDENCE", "status=%s but no file/line/quote" % st
|
||||
p = os.path.join(SRC, rf.replace("tools/reference/gcc-2.7.2/", "").lstrip("/"))
|
||||
if not os.path.exists(p):
|
||||
return "BAD-FILE", "no such file: %s" % rf
|
||||
lines = open(p, errors="replace").read().splitlines()
|
||||
if not (1 <= rl <= len(lines)):
|
||||
return "BAD-LINE", "%s has %d lines, cited %d" % (rf, len(lines), rl)
|
||||
nq = norm(q)
|
||||
if not nq:
|
||||
return "NO-EVIDENCE", "empty quote"
|
||||
if nq in norm(lines[rl - 1]) or norm(lines[rl - 1]) in nq:
|
||||
return "OK", ""
|
||||
lo, hi = max(0, rl - 1 - WINDOW), min(len(lines), rl - 1 + WINDOW)
|
||||
for j in range(lo, hi):
|
||||
if nq in norm(lines[j]) or (norm(lines[j]) and norm(lines[j]) in nq):
|
||||
return "NEAR", "quote found at %d, cited %d (off by %+d)" % (j + 1, rl, (j + 1) - rl)
|
||||
return "FABRICATED", "quote appears nowhere within +/-%d lines of %s:%d" % (WINDOW, rf, rl)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(__doc__)
|
||||
sys.exit(2)
|
||||
findings = load_findings(sys.argv[1])
|
||||
if not findings:
|
||||
print("NO FINDINGS PARSED from", sys.argv[1])
|
||||
sys.exit(1)
|
||||
tally = {}
|
||||
bad = []
|
||||
for f in findings:
|
||||
v, why = check(f)
|
||||
tally[v] = tally.get(v, 0) + 1
|
||||
if v not in ("OK", "SKIP"):
|
||||
bad.append((v, f, why))
|
||||
print("findings checked: %d" % len(findings))
|
||||
for k in sorted(tally):
|
||||
print(" %-12s %d" % (k, tally[k]))
|
||||
if bad:
|
||||
print("\n--- EVIDENCE PROBLEMS (these findings are NOT usable as-is) ---")
|
||||
for v, f, why in bad:
|
||||
print("\n[%s] %s :: %s" % (v, f.get("_chunk", "?"), f.get("status")))
|
||||
print(" claim: %s" % (f.get("claim_excerpt", "")[:150]))
|
||||
print(" cited: %s -> %s:%s" % (f.get("cited"), f.get("real_file"), f.get("real_line")))
|
||||
print(" why : %s" % why)
|
||||
st = {}
|
||||
for f in findings:
|
||||
st[f.get("status")] = st.get(f.get("status"), 0) + 1
|
||||
print("\nstatus distribution: %s" % st)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user