From 7acd0c4e2a1d991b3b03fd8d20316751d1b42576 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:07:27 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-29):=20tools/verify=5Fmap=5Ffindings?= =?UTF-8?q?.py=20=E2=80=94=20the=20codegen-map=20audit=20fabrication=20che?= =?UTF-8?q?ck=20(R3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved out of .run/ (where the blanket ignore would have lost it) into tools/ per R3, and generalised from regalloc to any docs/gcc-2.7.2-map/* audit. The check an LLM audit of a source-derived document cannot do for itself: agents return {real_file, real_line, source_quote}; this re-opens each file at each line and compares the verbatim quote to what is actually there. Whitespace-normalised, +/-40-line search window, and it reports NEAR (quote real, line wrong) SEPARATELY from FABRICATED (text appears nowhere) — because with the 2.8.1->2.7.2 drift reaching +611 lines in reload1.c, a wrong lookup lands INSIDE A DIFFERENT FUNCTION and every sentence built on it still reads plausibly. Used on the T34 regalloc.md audit: 184 findings, 0 FABRICATED. --- tools/verify_map_findings.py | 125 +++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tools/verify_map_findings.py diff --git a/tools/verify_map_findings.py b/tools/verify_map_findings.py new file mode 100644 index 000000000..d201d7dcc --- /dev/null +++ b/tools/verify_map_findings.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""verify_map_findings.py — machine-check a codegen-map audit's findings against the bytes. + +The regalloc.md re-derivation asks agents to classify each claim CONFIRMED / LINE-DRIFT / REFUTED / +UNVERIFIABLE and to justify it with `real_file` + `real_line` + `source_quote`. An agent's verdict is +a CLAIM, not evidence (R14) — and the specific failure mode this guards against is the one that makes +a false REFUTED nearly undetectable by reading: gcc 2.8.1 -> 2.7.2 drifts by +300..+600 lines in +reload1.c / local-alloc.c, so a naive line lookup lands inside a DIFFERENT FUNCTION and every +subsequent sentence reads plausibly. + +So: re-open each cited file at each cited line and compare the agent's verbatim quote to what is +actually there. A quote that does not match its line invalidates the finding. + +Comparison is whitespace-normalised (agents reflow leading tabs) but otherwise exact. A quote is also +searched for in a +/-40 line window so a NEAR-MISS (right code, wrong line) is reported separately +from a FABRICATION (text appears nowhere near) — those two need very different responses. + + python3 tools/verify_map_findings.py + +Generic over docs/gcc-2.7.2-map/*: any audit whose agents return {real_file, real_line, source_quote}. +""" +import json +import os +import re +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SRC = os.path.join(REPO, "tools/reference/gcc-2.7.2") +WINDOW = 40 + + +def norm(s): + return re.sub(r"\s+", " ", (s or "")).strip() + + +def load_findings(path): + """Accept either the workflow's return value or the raw journal.""" + txt = open(path, errors="replace").read() + out = [] + # journal: one JSON object per line, agent results under .result + for line in txt.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + o = json.loads(line) + except Exception: + continue + r = o.get("result", o) + if isinstance(r, dict) and isinstance(r.get("findings"), list): + for f in r["findings"]: + f = dict(f) + f.setdefault("_chunk", r.get("chunk", "?")) + out.append(f) + if out: + return out + # otherwise: a single result blob containing .findings + i = txt.find('{"counts"') + if i < 0: + i = txt.find("{") + try: + blob = json.loads(txt[i:]) + except Exception: + return [] + return blob.get("findings", []) + + +def check(f): + st = f.get("status") + if st == "UNVERIFIABLE": + return "SKIP", "declared unverifiable" + rf, rl, q = f.get("real_file"), f.get("real_line"), f.get("source_quote") + if not (rf and rl and q): + return "NO-EVIDENCE", "status=%s but no file/line/quote" % st + p = os.path.join(SRC, rf.replace("tools/reference/gcc-2.7.2/", "").lstrip("/")) + if not os.path.exists(p): + return "BAD-FILE", "no such file: %s" % rf + lines = open(p, errors="replace").read().splitlines() + if not (1 <= rl <= len(lines)): + return "BAD-LINE", "%s has %d lines, cited %d" % (rf, len(lines), rl) + nq = norm(q) + if not nq: + return "NO-EVIDENCE", "empty quote" + if nq in norm(lines[rl - 1]) or norm(lines[rl - 1]) in nq: + return "OK", "" + lo, hi = max(0, rl - 1 - WINDOW), min(len(lines), rl - 1 + WINDOW) + for j in range(lo, hi): + if nq in norm(lines[j]) or (norm(lines[j]) and norm(lines[j]) in nq): + return "NEAR", "quote found at %d, cited %d (off by %+d)" % (j + 1, rl, (j + 1) - rl) + return "FABRICATED", "quote appears nowhere within +/-%d lines of %s:%d" % (WINDOW, rf, rl) + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(2) + findings = load_findings(sys.argv[1]) + if not findings: + print("NO FINDINGS PARSED from", sys.argv[1]) + sys.exit(1) + tally = {} + bad = [] + for f in findings: + v, why = check(f) + tally[v] = tally.get(v, 0) + 1 + if v not in ("OK", "SKIP"): + bad.append((v, f, why)) + print("findings checked: %d" % len(findings)) + for k in sorted(tally): + print(" %-12s %d" % (k, tally[k])) + if bad: + print("\n--- EVIDENCE PROBLEMS (these findings are NOT usable as-is) ---") + for v, f, why in bad: + print("\n[%s] %s :: %s" % (v, f.get("_chunk", "?"), f.get("status"))) + print(" claim: %s" % (f.get("claim_excerpt", "")[:150])) + print(" cited: %s -> %s:%s" % (f.get("cited"), f.get("real_file"), f.get("real_line"))) + print(" why : %s" % why) + st = {} + for f in findings: + st[f.get("status")] = st.get(f.get("status"), 0) + 1 + print("\nstatus distribution: %s" % st) + + +if __name__ == "__main__": + main()