#!/usr/bin/env python3
"""gccmap_cites.py — tag every `file.c:NNN` citation in docs/gcc-2.7.2-map/*.md with the source tree it refers to (P33 E3).

    tools/gccmap_cites.py                 # derive a tag for every cite from the reference trees and WRITE it in place (idempotent)
    tools/gccmap_cites.py --dry-run       # the same derivation and census, nothing written
    tools/gccmap_cites.py --check         # TEXTUAL: exit 0 iff every cite carries a tag and every override is still used
                                          #   (no reference tree needed — safe for CI and `make tools-health`)
    tools/gccmap_cites.py --verify        # re-derive every existing tag from the trees; exit 1 on any disagreement
    tools/gccmap_cites.py --controls      # the known-true cases (R39) — exit 0 iff every one derives as recorded

The problem this solves. The codegen map was written in Phase 23 against `tools/reference/gcc-papermario/`, which is
gcc 2.8.1 — NOT the pinned 2.7.2 — so its line numbers are correct for the wrong compiler (drift up to +611 lines, enough
to land inside a different function); the Phase-29 audit re-derived every claim against the vanilla 2.7.2 subset at
`tools/reference/gcc-2.7.2/` and found the MECHANISMS sound but left the numbers mixed. A reader following a citation
needs to know which tree it belongs to. Tags are written right after the cite (inside the same code span when the cite
is in one):

    sched.c:2469 [2.7.2]        the cited construct sits at that line in the vanilla 2.7.2 subset (checked, not assumed)
    global.c:616 [2.8.1 pm]     the cited construct sits at that line in gcc-papermario (2.8.1); the 2.7.2 line differs
    engine_core.h:24926 [repo]  a project file, not gcc
    reload1.c:779 [?]           undecided by the evidence AND no override — listed; `--check` fails on it

How a tag is derived (R33 — from the trees, never typed):
  1. EVIDENCE: the identifiers in backticked/bold spans within ±160 characters of the cite (only C-looking ones — with an
     underscore, as every real gcc macro/function cited has; prose words and bare RTL codes are dropped), nearest first. For each, the nearest occurrence to the cited line(s)
     in each tree's file, within ±60 lines (a function name sits at the header, the cite often inside the body). The tree
     whose occurrence is nearer wins; an equal distance is a TIE — the line is equally valid in both — resolved by an
     override if one exists, else the author's cue, else 2.7.2. A line
     past the end of a tree's file cannot be that tree. Files absent from the 2.7.2 subset (18 .c files + config/mips)
     can only be `[2.8.1 pm]`.
  2. CUES: the author's own words — `papermario` / `2.8.1` just before the cite, or `→2.7.2 :NNN` just after it, mean
     2.8.1; `2.7.2` just before means 2.7.2. A cue that CONTRADICTS the evidence leaves the cite undecided (R34 — a
     disagreement fails loudly, it is not averaged); a cue alone decides when there is no evidence.
  3. OVERRIDES: docs/gcc-2.7.2-map/cite_overrides.tsv (`file:line<TAB>tag<TAB>reason`) — a human judgment, kept small;
     `--check` refuses an override whose cite no longer exists (a stale override is a lie, R51).
A tag that already exists is never silently changed: a derivation that disagrees is reported and exits 1
(`--retag` accepts the derivation). Tagging, --verify and --controls need both trees and refuse loudly without them
(R43); --check needs neither.
"""
import argparse
import pathlib
import re
import sys

REPO = pathlib.Path(__file__).resolve().parent.parent
MAP_DIR = REPO / "docs" / "gcc-2.7.2-map"
OVERRIDES = MAP_DIR / "cite_overrides.tsv"
TREES = {"2.7.2": REPO / "tools" / "reference" / "gcc-2.7.2", "2.8.1 pm": REPO / "tools" / "reference" / "gcc-papermario"}
TAGS = ("2.7.2", "2.8.1 pm", "repo", "?")
# a cite, optionally already tagged:  file.c:NNN[-MMM] [tag]
CITE = re.compile(r"\b([a-z0-9_-]+\.[ch]):(\d+)(?:-(\d+))?(?: \[(2\.7\.2|2\.8\.1 pm|repo|\?)\])?")
SPAN = re.compile(r"`([^`]+)`|\*\*([^*]+)\*\*")
IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]{2,}")
DROP = {"A23", "PSX", "REG", "MEM", "SET", "USE", "RTL", "RTX", "CSE", "GTE", "LUID", "TRUE", "FALSE", "STEERABLE",
        "INTRINSIC", "DETERMINED", "NOTHING", "SAME", "FILE", "BOTH", "ACROSS", "CALLS", "ORDER", "LEVER", "TEMP", "SLOT",
        "VALUE", "DENOMINATOR", "FORWARD", "BIRTH", "WITHOUT", "FIRST", "CALL", "ALU", "EXPR", "CODE"}
NEAR, WIDE = 3, 60
SKIP = {"README.md"}
# the known-true cases the map's own text records (audited S23 / t7g); --controls derives them and compares
CONTROLS = [("sched.md", "sched.c:2469", "2.7.2"), ("regalloc.md", "global.c:616", "2.8.1 pm"),
            ("regalloc.md", "local-alloc.c:1745", "2.8.1 pm"), ("cse_expr.md", "cse.c:8621-8644", "2.8.1 pm"),
            ("loop.md", "loop.c:4819", "2.7.2"), ("t7g-giant-harvest.md", "engine_core.h:24926", "repo")]


def tree_file(tree: pathlib.Path, name: str):
    for cand in (tree / name, tree / "config" / "mips" / name):
        if cand.is_file():
            return cand
    return None


_cache = {}


def lines_of(path: pathlib.Path):
    if path not in _cache:
        _cache[path] = path.read_text(encoding="utf-8", errors="replace").split("\n")
    return _cache[path]


_spans = {}


def spans_of(text: str):
    """Every backticked / bold span of the WHOLE document, paired once (pairing inside a window starting mid-span
    inverts the pairs and silently drops the identifiers right next to a cite — an S88 defect)."""
    key = id(text)
    if key not in _spans:
        # fenced code blocks would pair their backticks with the prose's and invert everything after them:
        # blank them (same length, so every offset stays valid) before pairing
        blanked = re.sub(r"```.*?```", lambda m: " " * len(m.group(0)), text, flags=re.S)
        _spans[key] = [(m.start(), m.end(), m.group(1) or m.group(2)) for m in SPAN.finditer(blanked)]
    return _spans[key]


def identifiers_near(text: str, start: int, end: int, own_file: str):
    """C-looking identifiers from backticked/bold spans within ±160 chars, nearest to the cite first."""
    lo, hi = max(0, start - 160), min(len(text), end + 160)
    found = []
    for s, e, span_text in spans_of(text):
        if e < lo or s > hi:
            continue
        dist = min(abs(s - start), abs(e - end))
        for ident in IDENT.findall(span_text):
            if ident in DROP or ident == own_file.split(".")[0]:
                continue
            if "_" not in ident:        # every real gcc macro/function cited has one; bare ALL-CAPS words (NOT, AND, DEST) are prose
                continue
            found.append((dist, ident))
    found.sort()
    out, seen = [], set()
    for _, ident in found:
        if ident not in seen:
            seen.add(ident); out.append(ident)
    return out[:8]


def nearest(tree: pathlib.Path, name: str, a: int, b: int, ident: str):
    """Distance (in lines) from the cited range to the nearest occurrence of ident in this tree's file, within WIDE;
    None = file absent; -1 = the cited line is past the end of the file; WIDE+1 = not found within the window."""
    f = tree_file(tree, name)
    if f is None:
        return None
    L = lines_of(f)
    if a > len(L):
        return -1
    pat = re.compile(r"\b" + re.escape(ident) + (r"" if "_" in ident else r"\b"))
    best = WIDE + 1
    for i in range(max(0, a - 1 - WIDE), min(len(L), b + WIDE)):
        if pat.search(L[i]):
            d = 0 if a - 1 <= i <= b - 1 else (a - 1 - i if i < a - 1 else i - (b - 1))
            best = min(best, d)
    return best


def cue(text: str, start: int, end: int):
    """The author's own words: a prefix cue (tie-breaker) or the arrow form `→2.7.2 :NNN` (decisive — the author
    states both numbers, so the cited one is the 2.8.1 line). Returns (tag, decisive)."""
    strip = re.compile(r" \[(?:2\.7\.2|2\.8\.1 pm|repo|\?)\]")       # never read the tags this tool itself wrote (R57)
    before = strip.sub("", text[max(0, start - 40):start])[-24:]
    after = strip.sub("", text[end:end + 30])[:14]
    if re.search(r"`?→\**2\.7\.2 ?:", after):
        return "2.8.1 pm", True
    if "papermario" in before or "2.8.1" in before:
        return "2.8.1 pm", False
    if re.search(r"2\.7\.2 ?`?$", before) or before.endswith("2.7.2 "):
        return "2.7.2", False
    return None, False


DEF_HEADER = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*) \(")      # gcc-2.x style: the function name at column 0, then " ("
_extents = {}


def enclosing(tree: pathlib.Path, name: str, line: int):
    """The function whose definition header is the last one at or before `line` in this tree's file (None = unknown)."""
    f = tree_file(tree, name)
    if f is None or not name.endswith(".c"):
        return None
    if f not in _extents:
        _extents[f] = [(i + 1, m.group(1)) for i, ln in enumerate(lines_of(f)) if (m := DEF_HEADER.match(ln))]
    if line > len(lines_of(f)):
        return "<past EOF>"
    last = None
    for start, fn in _extents[f]:
        if start > line:
            break
        last = fn
    return last


def snippets_near(text: str, start: int, end: int):
    """Backticked spans within ±160 chars that look like quoted source (≥ 6 chars, not a lone identifier/cite)."""
    lo, hi = max(0, start - 160), min(len(text), end + 160)
    out = []
    for s0, e0, span_text in spans_of(text):
        if e0 < lo or s0 > hi or text[s0] != "`":
            continue
        s = " ".join(span_text.split())
        if len(s) >= 6 and not CITE.search(s) and not re.fullmatch(r"[A-Za-z0-9_.$-]+", s):
            out.append((min(abs(s0 - start), abs(e0 - end)), s))
    return [s for _, s in sorted(out)][:6]


def snippet_distance(tree: pathlib.Path, name: str, a: int, b: int, snippet: str):
    """Distance from the cited range to the nearest line containing the whitespace-normalized snippet (see nearest())."""
    f = tree_file(tree, name)
    if f is None:
        return None
    L = lines_of(f)
    if a > len(L):
        return -1
    best = WIDE + 1
    for i in range(max(0, a - 1 - WIDE), min(len(L), b + WIDE)):
        if snippet in " ".join(L[i].split()):
            d = 0 if a - 1 <= i <= b - 1 else (a - 1 - i if i < a - 1 else i - (b - 1))
            best = min(best, d)
    return best


def evidence(name, a, b, idents, snippets=()):
    """The tree the identifiers point at, or None; plus a note. Quoted source snippets first (strongest), then function
    extents (a cite inside a long body is far from the header), then the nearest-occurrence distance (macros, globals,
    headers)."""
    for s in snippets:
        d27, d28 = snippet_distance(TREES["2.7.2"], name, a, b, s), snippet_distance(TREES["2.8.1 pm"], name, a, b, s)
        ok27 = d27 is not None and 0 <= d27 <= WIDE
        ok28 = d28 is not None and 0 <= d28 <= WIDE
        if ok27 and (not ok28 or d27 <= d28):
            return "2.7.2", f"quoted `{s}` at distance {d27} (pm {d28 if d28 is not None else 'n/a'})"
        if ok28:
            return "2.8.1 pm", f"quoted `{s}` at distance {d28} (2.7.2 {'n/a' if d27 is None else ('past EOF' if d27 == -1 else d27)})"
    for ident in idents:
        if "_" in ident and name.endswith(".c"):
            e27, e28 = enclosing(TREES["2.7.2"], name, a), enclosing(TREES["2.8.1 pm"], name, a)
            in27 = e27 is not None and (e27 == ident or e27.startswith(ident))
            in28 = e28 is not None and (e28 == ident or e28.startswith(ident))
            if in27 and in28:
                return "tie", f"line {a} is inside `{ident}` in both trees"
            if in27:
                return "2.7.2", f"line {a} is inside `{e27}` (pm: `{e28}`)"
            if in28:
                return "2.8.1 pm", f"line {a} is inside `{e28}` (2.7.2: `{e27}`)"
    for ident in idents:
        d27, d28 = nearest(TREES["2.7.2"], name, a, b, ident), nearest(TREES["2.8.1 pm"], name, a, b, ident)
        ok27 = d27 is not None and 0 <= d27 <= WIDE
        ok28 = d28 is not None and 0 <= d28 <= WIDE
        if ok27 and ok28 and d27 == d28:
            return "tie", f"`{ident}` at the same distance {d27} in both trees"
        if ok27 and (not ok28 or d27 < d28):
            return "2.7.2", f"`{ident}` at distance {d27} (pm {d28 if d28 is not None else 'n/a'})"
        if ok28:
            note = f"`{ident}` at distance {d28} (2.7.2 {'n/a' if d27 is None else ('past EOF' if d27 == -1 else d27)})"
            return "2.8.1 pm", note
    return None, ("no C-looking identifier near the cite" if not idents else f"none of {idents[:4]} within ±{WIDE} lines in either tree")


def derive(text: str, m: re.Match, overrides):
    name, a = m.group(1), int(m.group(2))
    b = int(m.group(3)) if m.group(3) and int(m.group(3)) >= a else a
    key = cite_key(m)
    if (REPO / "src" / "shared" / name).exists():
        return "repo", "project file"
    c, decisive = cue(text, m.start(), m.end())
    if decisive:
        return c, "the text gives both numbers (→2.7.2 :NNN), so this is the 2.8.1 line"
    f27 = tree_file(TREES["2.7.2"], name)
    if f27 is not None and a > len(lines_of(f27)):
        f28 = tree_file(TREES["2.8.1 pm"], name)
        if f28 is not None and a <= len(lines_of(f28)):
            return "2.8.1 pm", f"line {a} is past the end of the 2.7.2 file ({len(lines_of(f27))} lines)"
    idents = identifiers_near(text, m.start(), m.end(), name)
    ev, note = evidence(name, a, b, idents, snippets_near(text, m.start(), m.end()))
    if ev == "tie":                                   # equally valid in both: the human override may resolve it, else 2.7.2
        if key in overrides:
            return overrides[key][0], f"override resolves a tie ({note}): {overrides[key][1]}"
        return c or "2.7.2", note + (f"; the text's cue [{c}] decides" if c else "; a tie is a valid 2.7.2 line")
    if ev and c and ev != c:
        return "?", f"evidence says [{ev}] ({note}) but the text's cue says [{c}] — resolve by hand (override)"
    if ev:
        return ev, note + (f"; cue agrees" if c else "")
    if c:
        return c, f"cue only ({note})"
    if key in overrides:
        tag, reason = overrides[key]
        return tag, f"override: {reason}"
    if tree_file(TREES["2.7.2"], name) is None:
        note += " (file not in the 2.7.2 subset)"
    else:
        note += f" — line {a} is inside 2.7.2 `{enclosing(TREES['2.7.2'], name, a)}` / pm `{enclosing(TREES['2.8.1 pm'], name, a)}`"
    return "?", note


def load_overrides():
    out = {}
    if OVERRIDES.exists():
        for ln in OVERRIDES.read_text(encoding="utf-8").splitlines():
            ln = ln.rstrip()
            if not ln or ln.startswith("#"):
                continue
            key, tag, reason = (ln.split("\t") + ["", ""])[:3]
            if tag not in TAGS or tag == "?":
                sys.exit(f"gccmap_cites: bad override tag {tag!r} for {key} in {OVERRIDES}")
            out[key.strip()] = (tag, reason.strip())
    return out


def map_files():
    return [p for p in sorted(MAP_DIR.glob("*.md")) if p.name not in SKIP]


def cite_key(m):
    return f"{m.group(1)}:{m.group(2)}" + (f"-{m.group(3)}" if m.group(3) else "")


def require_trees():
    for tag, path in TREES.items():
        if not path.is_dir():
            print(f"gccmap_cites: reference tree for [{tag}] missing at {path} — see docs/SETUP.md §5.6 (re-clone / fetch); "
                  f"--check needs no tree")
            return False
    return True


def run_check():
    overrides = load_overrides()
    used, untagged, n = set(), [], 0
    for p in map_files():
        text = p.read_text(encoding="utf-8")
        for m in CITE.finditer(text):
            n += 1
            key = cite_key(m)
            if key in overrides:
                used.add(key)
            if m.group(4) is None or m.group(4) == "?":
                untagged.append((p.name, key, m.group(4)))
    stale = sorted(set(overrides) - used)
    print(f"gccmap_cites --check: {n} cites in {len(map_files())} files, {len(untagged)} untagged/undecided, "
          f"{len(overrides)} overrides ({len(stale)} stale)")
    for f, k, tag in untagged:
        print(f"  UNTAGGED {f}: {k}" + (" [?]" if tag == "?" else ""))
    for k in stale:
        print(f"  STALE override {k}: no such cite any more — delete it from {OVERRIDES.name}")
    if untagged or stale:
        return 1
    print("gccmap_cites --check: OK")
    return 0


def run_controls():
    if not require_trees():
        return 2
    overrides = {}
    bad = 0
    for fname, key, want in CONTROLS:
        text = (MAP_DIR / fname).read_text(encoding="utf-8")
        got = None
        for m in CITE.finditer(text):
            if cite_key(m) == key:
                got, why = derive(text, m, overrides)
                break
        status = "ok  " if got == want else "FAIL"
        bad += got != want
        print(f"  {status} {fname} {key}: derived [{got}] (want [{want}]) — {why if got is not None else 'cite not found'}")
    print(f"gccmap_cites --controls: {len(CONTROLS)} known-true cases, {bad} failed")
    return 1 if bad else 0


def run_explain():
    """For every undecided cite: the map's own sentence and the cited lines in both trees — the input to an override."""
    if not require_trees():
        return 2
    overrides = load_overrides()
    n = 0
    for p in map_files():
        text = p.read_text(encoding="utf-8")
        for m in CITE.finditer(text):
            tag, why = derive(text, m, overrides)
            if tag != "?":
                continue
            n += 1
            name, a = m.group(1), int(m.group(2))
            ctx = " ".join(text[max(0, m.start() - 110):m.end() + 70].split())
            print(f"\n[{n}] {p.name} {cite_key(m)} — {why}\n    text: …{ctx}…")
            for tag_, tree in TREES.items():
                f = tree_file(tree, name)
                if f is None:
                    print(f"    {tag_:9s}: (file absent)"); continue
                L = lines_of(f)
                show = [f"{i + 1}: {L[i].rstrip()[:110]}" for i in range(a - 1, min(len(L), a + 1)) if i >= 0]
                print(f"    {tag_:9s}: " + (" | ".join(show) if show else f"(past EOF, {len(L)} lines)"))
    print(f"\ngccmap_cites --explain: {n} undecided cites shown")
    return 0


def run_derive(write: bool, verify: bool, retag: bool):
    if not require_trees():
        return 2
    overrides = load_overrides()
    totals = {t: 0 for t in TAGS}; disagreements = []; undecided = []; changed_files = 0; n = 0; over_used = 0
    over_keys_used, cite_keys = set(), set()
    for p in map_files():
        text = p.read_text(encoding="utf-8")
        out, pos, file_changed = [], 0, False
        for m in CITE.finditer(text):
            n += 1
            tag, why = derive(text, m, overrides)
            cite_keys.add(cite_key(m))
            if why.startswith("override"):          # "override: …" and "override resolves a tie …"
                over_used += 1; over_keys_used.add(cite_key(m))
            existing = m.group(4)
            totals[tag] += 1
            if tag == "?":
                undecided.append((p.name, cite_key(m), why))
            if existing and existing != "?" and existing != tag:
                disagreements.append((p.name, cite_key(m), existing, tag, why))
                if not retag:
                    tag = existing        # never silently change a tag
            new = f"{m.group(1)}:{m.group(2)}" + (f"-{m.group(3)}" if m.group(3) else "") + f" [{tag}]"
            if new != m.group(0):
                file_changed = True
            out.append(text[pos:m.start()]); out.append(new); pos = m.end()
        out.append(text[pos:])
        if file_changed:
            changed_files += 1
            if write:
                p.write_text("".join(out), encoding="utf-8")
    mode = "verify" if verify else ("write" if write else "dry-run")
    print(f"gccmap_cites ({mode}): {n} cites in {len(map_files())} files — [2.7.2] {totals['2.7.2']} · [2.8.1 pm] "
          f"{totals['2.8.1 pm']} · [repo] {totals['repo']} · undecided {totals['?']} · override hits {over_used} "
          f"({len(overrides)} rows) · files {'written' if write else 'that would change'}: {changed_files}")
    for f, k, why in undecided:
        print(f"  UNDECIDED {f}: {k} — {why}")
    for k in sorted(set(overrides) & cite_keys - over_keys_used):
        print(f"  REDUNDANT override {k}: the evidence decides it — delete the row from {OVERRIDES.name}")
    for f, k, old, new, why in disagreements:
        print(f"  DISAGREE {f}: {k} tagged [{old}], derived [{new}] ({why})" + ("" if retag else " — kept; --retag to accept"))
    if verify:
        return 1 if (disagreements or undecided) else 0
    return 1 if undecided else 0


def main(argv):
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    g = ap.add_mutually_exclusive_group()
    g.add_argument("--dry-run", action="store_true")
    g.add_argument("--check", action="store_true")
    g.add_argument("--verify", action="store_true")
    g.add_argument("--controls", action="store_true")
    g.add_argument("--explain", action="store_true", help="show the undecided cites with the map's sentence and both trees' lines")
    ap.add_argument("--retag", action="store_true", help="accept a derivation that disagrees with an existing tag")
    a = ap.parse_args(argv)
    if a.check:
        return run_check()
    if a.controls:
        return run_controls()
    if a.explain:
        return run_explain()
    return run_derive(write=not (a.dry_run or a.verify), verify=a.verify, retag=a.retag)


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))