BFM-decomp/tools/mine_hindsight.py

#!/usr/bin/env python3
"""mine_hindsight.py — gather every recorded hindsight in the project into one file with file:line anchors (P33 F2).

    tools/mine_hindsight.py [--out .run/P33/hindsight.md]

Three sources, none of them re-typed (R31: cite, never backfill):
  1. docs/decision-log.md — every `Hindsight / for the wiki` bullet (plain or bold spelling, with its continuation
     lines), under the dated entry heading it belongs to;
  2. phase-ends/PhaseEnd_*.md — every "What we believed, what failed, and what we would do sooner" section (P31, P32);
  3. phase-ends/PhaseEnd_*.md — every `## Deviations` table (all phases).
Output: a Markdown digest (scratch; the retrospective cites the SOURCES by heading/path, this file is the working set).
Prints the census with denominators (R41): entries scanned, hindsight bullets found, PhaseEnds with a believed section,
deviation rows.
"""
import argparse
import pathlib
import re
import sys

REPO = pathlib.Path(__file__).resolve().parent.parent
LOG = REPO / "docs" / "decision-log.md"
HIND_RE = re.compile(r"^\s*- (?:\*\*)?Hindsight[^:]*:(?:\*\*)?\s*(.*)$")          # `- Hindsight / for the wiki: …` (plain or bold)
HIND_SEC_RE = re.compile(r"^#{2,4} .*Hindsight", re.I)                          # `### Hindsight path` sections (newer entries)


def mine_log():
    lines = LOG.read_text(encoding="utf-8", errors="replace").splitlines()
    heading, entries, out = None, 0, []
    i = 0
    while i < len(lines):
        ln = lines[i]
        if ln.startswith("## "):
            heading = (ln[3:].strip(), i + 1)
            if not ln.startswith("## [date]"):
                entries += 1
        m = HIND_RE.match(ln)
        if m and heading and not heading[0].startswith("[date]"):
            body = [m.group(1)]
            j = i + 1
            while j < len(lines) and lines[j].startswith("  ") and not lines[j].lstrip().startswith("- "):
                body.append(lines[j].strip()); j += 1
            out.append((heading[0], heading[1], i + 1, " ".join(body)))
            i = j; continue
        if HIND_SEC_RE.match(ln) and not ln.startswith("## ") and heading and not heading[0].startswith("[date]"):
            j = i + 1
            body = []
            while j < len(lines) and not lines[j].startswith("#"):
                if lines[j].strip():
                    body.append(lines[j].strip())
                j += 1
            out.append((heading[0], heading[1], i + 1, " ".join(body)))
            i = j; continue
        i += 1
    return entries, out


def mine_believed():
    out = []
    for f in sorted((REPO / "phase-ends").glob("PhaseEnd_Phase*.md"), key=lambda p: [int(x) if x.isdigit() else x for x in re.split(r"(\d+)", p.stem)]):
        text = f.read_text(encoding="utf-8", errors="replace").splitlines()
        for i, ln in enumerate(text):
            if ln.startswith("## What we believed"):
                j = i + 1
                while j < len(text) and not text[j].startswith("## "):
                    j += 1
                out.append((f.name, i + 1, "\n".join(text[i:j]).strip()))
    return out


def mine_deviations():
    out = []
    for f in sorted((REPO / "phase-ends").glob("PhaseEnd_Phase*.md"), key=lambda p: [int(x) if x.isdigit() else x for x in re.split(r"(\d+)", p.stem)]):
        text = f.read_text(encoding="utf-8", errors="replace").splitlines()
        for i, ln in enumerate(text):
            if ln.startswith("## Deviations"):
                rows = []
                j = i + 1
                while j < len(text) and not text[j].startswith("## "):
                    if text[j].startswith("|") and not text[j].startswith("|---") and "| Item |" not in text[j]:
                        rows.append((j + 1, text[j]))
                    j += 1
                out.append((f.name, i + 1, rows))
    return out


def main(argv):
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--out", default=str(REPO / ".run" / "P33" / "hindsight.md"))
    a = ap.parse_args(argv)
    entries, hind = mine_log()
    believed = mine_believed()
    devs = mine_deviations()
    nrows = sum(len(r) for _, _, r in devs)
    md = ["# Mined hindsight — the working set for docs/retrospective.md (generated by tools/mine_hindsight.py; scratch)", "",
          f"decision-log: {entries} entries, {len(hind)} Hindsight bullets · believed-sections: {len(believed)} PhaseEnds · "
          f"deviations: {nrows} rows across {len(devs)} PhaseEnds", "",
          "## 1. Decision-log hindsight bullets (docs/decision-log.md:line — entry heading)", ""]
    for head, hl, l, body in hind:
        md.append(f"- **`docs/decision-log.md:{l}`** (entry `:{hl}` — {head[:110]}): {body}")
    md += ["", "## 2. 'What we believed' sections", ""]
    for fn, l, text in believed:
        md += [f"### `phase-ends/{fn}:{l}`", "", text, ""]
    md += ["## 3. Deviations tables (Item | Plan | Actual | Reason)", ""]
    for fn, l, rows in devs:
        md.append(f"### `phase-ends/{fn}:{l}` ({len(rows)} rows)")
        for rl, row in rows:
            md.append(f"- `:{rl}` {row}")
        md.append("")
    out = pathlib.Path(a.out); out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text("\n".join(md) + "\n", encoding="utf-8")
    print(f"mine_hindsight: {entries} decision-log entries → {len(hind)} Hindsight bullets; {len(believed)} believed-sections; "
          f"{nrows} deviation rows in {len(devs)} PhaseEnds -> {out}")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))