diff --git a/docs/SETUP.md b/docs/SETUP.md index a9c2ef5d0..4bc314412 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -663,7 +663,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/ld_interleave.py` | Interleave linker inputs to match original section ordering. | | | `tools/split_src_region.py` | Split a `src/` region file at object boundaries. | | | `tools/rollout_whale_o0.py` | **(Phase 24 W9)** Roll out the -O0 whale `func_80144B9C` ×134: per single-file overlay, line-split `.c` at the whale, carve the yaml code subseg into before/`_o0b`(-O0)/`_after`, write a thin `_o0b.c` that `#include`s the shared `src/shared/func_80144B9C.h`. Idempotent; the `WHALE_O0B_OBJS` Makefile wildcard -O0-compiles all `_o0b.o` (cookbook §38). | -| **Reports** | `tools/progress.py` | Per-binary decomp progress (`make report`); counts dedup-shared fns as REAL via the registry (Phase 11). **`--fleet`** (Phase 15) aggregates all 136 binaries → `docs/progress.fleet.md` (deterministic, source-derived). | +| **Reports** | `tools/progress.py` | Per-binary decomp progress (`make report`); counts dedup-shared fns as REAL via the registry (Phase 11). **`--fleet`** (Phase 15) aggregates all 136 binaries → `docs/progress.fleet.md` (deterministic, source-derived). **`--weighted`** (Phase 25) prints the two BYTE/instruction-weighted metrics from `.run/sig.*.jsonl` (executable code only): **instr-weighted** (fleet per-overlay, the decomp.dev-display number) + **distinct-code** (dedup, each unique fn once, the distinct-RE number); both also fold into `--fleet` alongside the ×134-inflated function-count %. Needs `make sig-overlays` first; degrades gracefully without sigs. | | | `tools/difficulty.py` | Per-function difficulty scoring. | | | `tools/dup_report.py` | Duplicate-function report; `--cross` (Phase 11) buckets all binaries → `docs/duplicates.cross.md`. **Phase 15:** ingests each overlay once (named ∪ `sig.ov_*` glob, deduped by alias) — else onboarded overlays double-count and inflate collapsible bytes ~2×. | | **Cross-binary dedup** (Phase 11, cookbook §11) | `tools/sig_image.py` | **Ghidra-FREE** per-function signer for a flat image (overlay/resident); `h_exact` byte-matches the Ghidra dumper, self-consistent `h_norm`; linear-partition + `detect_code_end` boundaries. | diff --git a/docs/progress.fleet.md b/docs/progress.fleet.md index c867e9bb2..0a19500a7 100644 --- a/docs/progress.fleet.md +++ b/docs/progress.fleet.md @@ -3,9 +3,13 @@ # source-derived (committed src/*.c + config/dedup.us.yaml). Live byte gate: `make check-all`; # cross-binary collapsible-byte leverage: docs/duplicates.cross.md. +# THREE progress metrics (all matter — see the labels): +FLEET fn-count byte-ident: 256430 / 344306 = 74.48% (REAL+LINKED+empties; FUNCTION-count, ×134-inflated — one crack counts per overlay) +FLEET instr-weighted : 7222555 / 12707182 = 56.8% (shipped .text across resident+134 overlays; the decomp.dev-DISPLAY number) +FLEET distinct-code(uniq): 1478352 / 5410077 = 27.3% (28814/84996 unique fns; the DISTINCT-RE number; main EXE not sig'd) + FLEET REAL substantive : 254901 (of which dedup-shared 223275 via 1813 groups / 223725 instances) FLEET LINKED PsyQ objs : 959 -FLEET byte-identical : 256430 / 344306 = 74.48% (REAL+LINKED+empties) FLEET NON_MATCHING : 7 (0 in any default build — G4) FLEET INCLUDE_ASM stubs : 87869 FLEET matchable : 344306 diff --git a/tools/progress.py b/tools/progress.py index 03c7f0265..9b38bafb2 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -509,6 +509,51 @@ def report(binary, audit=False, write=True): matchable=matchable, byteident=byteident) +def weighted_metrics(): + """Instruction/byte-weighted matching % from the committed sigs (.run/sig.*.jsonl) + src stubs. + Two framings, both against EXECUTABLE CODE only (not the ISO/audio/assets/data); MIPS instrs are + all 4 bytes so instruction% == byte%: + - fleet : sum(matched nins) / sum(total nins) over resident + every overlay (shared engine + counted PER-OVERLAY) — the number a decomp.dev/frogress per-binary aggregate displays. + - dedup : each distinct h_exact class ONCE, nins-weighted, matched if ANY overlay has it non-stub + — the 'distinct reverse-engineering' number (harsh: the ~74k x1 unique monsters dominate). + Covers resident + the 134 overlays; the main EXE has no image-sig (a small separate binary, excluded). + Returns a dict, or None if no sigs (run `make sig-overlays` first — keeps --fleet working without them).""" + import os, glob, json + paths = sorted(glob.glob(str(ROOT / ".run/sig.ov_*.jsonl"))) + rp = ROOT / ".run/sig.resident.jsonl" + if rp.exists(): + paths.append(str(rp)) + if not paths: + return None + + def src_stubs(binname): + m = set() + for cf in glob.glob(str(ROOT / f"src/{binname}/*.c")): + for a in re.findall(r'INCLUDE_ASM\([^)]*,\s*func_([0-9A-Fa-f]+)\)', open(cf).read()): + m.add(int(a, 16)) + return m + + fm = ft = 0 + cls_nins, matched_cls = {}, set() + for p in paths: + b = os.path.basename(p).split("sig.")[1][:-6] + st = src_stubs(b) + for line in open(p): + r = json.loads(line) + a, n, hx = int(r["addr"], 16), r["nins"], r["h_exact"] + ft += n + cls_nins[hx] = n # h_exact-identical -> identical nins + if a not in st: # non-stub == matched (fleet is 136/136 byte-identical) + fm += n + matched_cls.add(hx) + ut = sum(cls_nins.values()) + um = sum(n for hx, n in cls_nins.items() if hx in matched_cls) + return dict(fleet_m=fm, fleet_t=ft, fleet_pct=(100 * fm / ft if ft else 0.0), + dedup_m=um, dedup_t=ut, dedup_pct=(100 * um / ut if ut else 0.0), + nbins=len(paths), dedup_fns=len(matched_cls), dedup_total_fns=len(cls_nins)) + + def fleet(): """Aggregate every binary into docs/progress.fleet.md — DETERMINISTIC (source-derived from the committed src/*.c + config/dedup.us.yaml). The live byte-identity gate is `make check-all`; @@ -531,30 +576,50 @@ def fleet(): except Exception: pass - out = ["# BFM FLEET matching progress (generated by tools/progress.py --fleet — authoritative)", - f"# {len(rows)} binaries: main + resident + {len(rows)-2} location overlays. DETERMINISTIC,", - "# source-derived (committed src/*.c + config/dedup.us.yaml). Live byte gate: `make check-all`;", - "# cross-binary collapsible-byte leverage: docs/duplicates.cross.md.", "", - f"FLEET REAL substantive : {REAL:6d} (of which dedup-shared {SHARED} via {ngroups} groups / {nmembers} instances)", - f"FLEET LINKED PsyQ objs : {LINKED:6d}", - f"FLEET byte-identical : {BYTE:6d} / {MATCH} = {100*BYTE/MATCH:.2f}% (REAL+LINKED+empties)", - f"FLEET NON_MATCHING : {NM:6d} (0 in any default build — G4)", - f"FLEET INCLUDE_ASM stubs : {STUBS:6d}", - f"FLEET matchable : {MATCH:6d}", "", - "| binary | REAL | shared | LINKED | byte-ident | matchable | byte-ident % |", - "|---|---:|---:|---:|---:|---:|---:|"] + wm = weighted_metrics() + head = ["# BFM FLEET matching progress (generated by tools/progress.py --fleet — authoritative)", + f"# {len(rows)} binaries: main + resident + {len(rows)-2} location overlays. DETERMINISTIC,", + "# source-derived (committed src/*.c + config/dedup.us.yaml). Live byte gate: `make check-all`;", + "# cross-binary collapsible-byte leverage: docs/duplicates.cross.md.", "", + "# THREE progress metrics (all matter — see the labels):", + f"FLEET fn-count byte-ident: {BYTE:6d} / {MATCH} = {100*BYTE/MATCH:.2f}% (REAL+LINKED+empties; FUNCTION-count, ×134-inflated — one crack counts per overlay)"] + if wm: + head += [ + f"FLEET instr-weighted : {wm['fleet_m']:7d} / {wm['fleet_t']} = {wm['fleet_pct']:.1f}% (shipped .text across resident+{wm['nbins']-1} overlays; the decomp.dev-DISPLAY number)", + f"FLEET distinct-code(uniq): {wm['dedup_m']:7d} / {wm['dedup_t']} = {wm['dedup_pct']:.1f}% ({wm['dedup_fns']}/{wm['dedup_total_fns']} unique fns; the DISTINCT-RE number; main EXE not sig'd)"] + else: + head += ["# (instr-weighted + distinct-code metrics need .run/sig.*.jsonl — run `make sig-overlays`)"] + head += ["", + f"FLEET REAL substantive : {REAL:6d} (of which dedup-shared {SHARED} via {ngroups} groups / {nmembers} instances)", + f"FLEET LINKED PsyQ objs : {LINKED:6d}", + f"FLEET NON_MATCHING : {NM:6d} (0 in any default build — G4)", + f"FLEET INCLUDE_ASM stubs : {STUBS:6d}", + f"FLEET matchable : {MATCH:6d}", ""] + table = ["| binary | REAL | shared | LINKED | byte-ident | matchable | byte-ident % |", + "|---|---:|---:|---:|---:|---:|---:|"] for r in rows: pct = (100 * r['byteident'] / r['matchable']) if r['matchable'] else 0.0 - out.append(f"| {r['binary']} | {r['real']} | {r['shared']} | {r['linked']} | " - f"{r['byteident']} | {r['matchable']} | {pct:.1f}% |") - text = "\n".join(out) + "\n" - print("\n".join(out[:11])) + table.append(f"| {r['binary']} | {r['real']} | {r['shared']} | {r['linked']} | " + f"{r['byteident']} | {r['matchable']} | {pct:.1f}% |") + text = "\n".join(head + table) + "\n" + print("\n".join(head[:-1])) # the summary block (drop the trailing blank) (ROOT / "docs/progress.fleet.md").write_text(text) def main(): if '--fleet' in sys.argv: fleet(); return + if '--weighted' in sys.argv: + wm = weighted_metrics() + if not wm: + sys.exit("progress.py --weighted: no .run/sig.*.jsonl — run `make sig-overlays` first.") + print("BFM byte-weighted matching progress (executable code only; resident + %d overlays; main EXE excluded):" + % (wm['nbins'] - 1)) + print(" instr-weighted (fleet, decomp.dev-display): %7d / %d = %.1f%%" + % (wm['fleet_m'], wm['fleet_t'], wm['fleet_pct'])) + print(" distinct-code (dedup, distinct-RE) : %7d / %d = %.1f%% (%d/%d unique fns)" + % (wm['dedup_m'], wm['dedup_t'], wm['dedup_pct'], wm['dedup_fns'], wm['dedup_total_fns'])) + return audit = '--audit' in sys.argv check = '--check' in sys.argv binary = next((sys.argv[i + 1] for i, x in enumerate(sys.argv)