BFM-decomp/tools/ghidra_roster.py

#!/usr/bin/env python3
"""ghidra_roster.py — config/ghidra/ROSTER.md from the committed Ghidra annotation files (P33 B5).

    tools/ghidra_roster.py            # writes config/ghidra/ROSTER.md
    tools/ghidra_roster.py --check    # exit 1 if ROSTER.md is stale (for tools-health)

One row per config/ghidra/<program>.jsonl: the program's kind (PS-X EXE via the PSX loader, or a raw
blob at its vram), its payload and vram as the build registers them (`make -s print-EXE / print-VRAM_BASE`
— the same source tools/ghidra_rebuild.sh imports from, R33), the memory blocks the file pins, the
hand-authored row census (everything that is not a `program`/`block`/`archive` container row), and the
last `--proof` verdict recorded by tools/ghidra_rebuild.sh in .run/ghidra_rebuild/<program>.proof
("unproven here" when that scratch file is absent — a fresh clone has not run the proof yet).
The 123 other programs of the live project carry no hand-authored rows and are regenerated on demand
(tools/prefetch_fleet.py); they are not in the roster on purpose.
"""
import collections
import datetime as dt
import json
import pathlib
import subprocess
import sys

REPO = pathlib.Path(__file__).resolve().parent.parent
CONF = REPO / "config" / "ghidra"
SCR = REPO / ".run" / "ghidra_rebuild"
CONTAINER = {"program", "block", "archive"}
# The PS-X EXE programs (the PSX loader, no vram argument) — mirrors the case table in ghidra_rebuild.sh.
EXES = {
    "SLUS_007.26": ("main", "extracted/retail/SLUS_007.26"),
    "sep8_SLUS_007.26": ("proto-sep8", "extracted/proto/sep8_SLUS_007.26 (prototype disc)"),
    "aug31_USA_DEMO.EXE": ("proto-demo", "extracted/proto/aug31_USA_DEMO.EXE (prototype disc)"),
}


def make_print(var, alias):
    r = subprocess.run(["make", "-s", f"print-{var}", f"BINARY={alias}"], cwd=REPO, capture_output=True, text=True)
    return r.stdout.strip() if r.returncode == 0 else "?"


def row_for(path):
    prog = path.stem
    kinds = collections.Counter()
    blocks = []
    image_base = fmt = "?"
    for ln in path.read_text(encoding="utf-8").splitlines():
        if not ln.strip():
            continue
        o = json.loads(ln)
        kinds[o["k"]] += 1
        if o["k"] == "program":
            image_base, fmt = o.get("image_base", "?"), o.get("format", "?")
        elif o["k"] == "block":
            blocks.append(o)
    if prog in EXES:
        kind, payload = "PS-X EXE", EXES[prog][1]
        vram = "header"
    else:
        kind, payload, vram = "raw blob", make_print("EXE", prog), make_print("VRAM_BASE", prog)
    hand = {k: v for k, v in kinds.items() if k not in CONTAINER}
    census = ", ".join(f"{k} {v}" for k, v in sorted(hand.items())) or "none (container rows only)"
    proof = SCR / f"{prog}.proof"
    verdict = proof.read_text(encoding="utf-8").strip() if proof.exists() else "unproven here"
    # the loaded (initialized) blocks by name+range; the loader's hardware/uninitialized blocks only as a count
    loaded = [f"{b['name']} {b['start']}–{b['end']}" for b in blocks if b.get("init")]
    other = len(blocks) - len(loaded)
    blk = "; ".join(loaded) + (f" (+{other} uninit/hardware)" if other else "")
    return f"| `{prog}` | {kind} ({fmt}) | `{payload}` | {vram} | {blk} | {sum(hand.values())} — {census} | {verdict} |"


def render():
    files = sorted(CONF.glob("*.jsonl"))
    lines = [
        "# config/ghidra/ROSTER.md — the Ghidra programs whose hand-authored RE work is tracked as text",
        "",
        "> Generated by `tools/ghidra_roster.py` — never edit by hand. One row per `config/ghidra/<program>.jsonl`",
        "> (the delta format of `tools/ghidra_scripts/ExportAnnotations.java`). Rebuild any row's program from the",
        "> disc + this file with `tools/ghidra_rebuild.sh <program>`; prove it with `--proof` (P33 B5). The binary",
        "> Ghidra project embeds the game's bytes and left git at P33; these files are what R20 backs up instead.",
        "",
        "| program | kind | payload | vram | blocks | hand-authored rows | last proof (this machine) |",
        "|---|---|---|---|---|---|---|",
    ]
    lines += [row_for(p) for p in files]
    lines += [
        "",
        f"{len(files)} programs tracked. The live project's other programs (the location overlays and `md_*` modules,",
        "129 on disk at P33) hold no hand-authored rows and are regenerated on demand by `tools/prefetch_fleet.py`.",
        "Proof verdicts come from `.run/ghidra_rebuild/<program>.proof` (scratch; absent on a fresh clone).",
        "",
        f"_Generated {dt.date.today().isoformat()}._",
        "",
    ]
    return "\n".join(lines)


def main(argv):
    out = CONF / "ROSTER.md"
    text = render()
    if "--check" in argv:
        cur = out.read_text(encoding="utf-8") if out.exists() else ""
        # the proof column and the date are per-machine; --check compares everything else (a fresh clone
        # has no .run/ghidra_rebuild/*.proof and must not read as stale)
        def strip(t):
            out = []
            for l in t.splitlines():
                if l.startswith("_Generated"):
                    continue
                if l.startswith("| `"):
                    l = l.rsplit("|", 2)[0]
                out.append(l)
            return "\n".join(out)
        if strip(cur) != strip(text):
            print(f"ghidra_roster: {out} is STALE — run tools/ghidra_roster.py", file=sys.stderr)
            return 1
        print(f"ghidra_roster: OK ({len(list(CONF.glob('*.jsonl')))} programs)")
        return 0
    out.write_text(text, encoding="utf-8")
    print(f"ghidra_roster: wrote {out} ({len(list(CONF.glob('*.jsonl')))} programs)")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))