mirror/BFM-decomp
1
0
Fork 0
mirror of https://github.com/Druthulu/BFM-decomp synced 2026-10-01 07:40:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(phase-11): T6 — sign the 134 overlays; cross-report spans 136 binaries (milestone report half)

Browse Source 
- sig_image overlay discovery: linear partition (split contiguous code at jr+delay boundaries) +
  detect_code_end (first run of invalid instrs = the code->data transition; overlay code decodes
  ~100% valid, data drops to 43-95%). BFS dead-ended — overlays dispatch via function-pointer
  tables, not jal — so call-graph BFS found ~2 fns; linear partition recovers the whole code prefix
- Makefile: sig-overlays: signed 134 overlays -> .run/sig.ov_*.jsonl signs all 134 SCxx 0.4.dec @ 0x80128158 -> .run/sig.ov_*.jsonl (27s)
- dup_report --cross: ingest the overlay sigs (sig.ov_* convention) + main/resident; condensed
  source header (overlay set-sha for reproducibility); cap top-200 per subsection + state totals;
  compact member sample + #bin column
- RESULT (docs/duplicates.cross.md): h_exact 9366 cross-binary groups / 28.5 MB collapsible; h_norm
  8957 / 38.3 MB. Top group = a 770-instruction function BYTE-IDENTICAL in all 134 overlays (one
  match credits the whole fleet) — 'one match unlocks many' quantified; the Phase-12/13 work queue
- dedup-check green; committed per-binary digests byte-stable; report deterministic (sorted glob,
  no Date/random); the EXE<->resident pair still shares nothing (overlay<->overlay dominates)
This commit is contained in:
Drew T
2026-06-16 01:26:49 -06:00
parent 81e33c7a1e
commit f672c709c1
5 changed files with 886 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/dup_report.py
+42 -18
View File
@@ -39,12 +39,19 @@ def _load_sig(rel):
return [json.loads(l) for l in raw.decode().splitlines() if l.strip()], hashlib.sha1(raw).hexdigest()
OV_GLOB = ".run/sig.ov_*.jsonl" # sig_image overlay outputs (convention: ov_<SCxx>_<nnn>)
CROSS_CAP = 200 # max groups listed per subsection (ranked by collapsible bytes)
MEMBERS_SHOWN = 4 # sample members per group row (large groups span dozens of overlays)
def cross_report(minins):
"""Phase 11: bucket all BINARIES' functions together; emit docs/duplicates.cross.md.
A group whose members span >1 binary is a cross-binary duplicate (the dedup target an
engine/overlay match auto-credits); same-binary groups are where the share machinery is
first proven. Ranked by collapsible bytes = (count-1)*nins*4 (bytes SAVED — one member
stays the source). h_exact = guaranteed byte-match; h_norm = byte-confirm-required."""
"""Phase 11: bucket EVERY binary's functions together; emit docs/duplicates.cross.md. Sources =
the named BINARIES (main, resident) + every sig_image overlay sig (.run/sig.ov_*.jsonl). A group
whose members span >1 binary is a cross-binary duplicate (the dedup target an engine/overlay match
auto-credits — the Phase-12/13 work queue); same-binary groups are where the share machinery is
first proven. Ranked by collapsible bytes = (count-1)*nins*4 (bytes SAVED — one member stays the
source). h_exact = guaranteed byte-match; h_norm = candidate (byte-confirm via dedup_integrate)."""
import glob
sources, funcs = [], []
for alias, cfg in BINARIES.items():
rows, sha = _load_sig(cfg["sig"])
@@ -56,25 +63,44 @@ def cross_report(minins):
if r['nins'] >= minins and r.get('src') != 'IMPORTED':
r = dict(r); r['binary'] = alias
funcs.append(r)
# overlay sigs (sig_image) — summarized as one source line (a set-sha makes the digest reproducible)
ov_set = hashlib.sha1(); ov_n = 0
for p in sorted(glob.glob(str(ROOT / OV_GLOB))):
alias = pathlib.Path(p).name[len("sig."):-len(".jsonl")]
rows, sha = _load_sig(str(pathlib.Path(p).relative_to(ROOT)))
if rows is None:
continue
ov_set.update(sha.encode()); ov_n += 1
for r in rows:
if r['nins'] >= minins and r.get('src') != 'IMPORTED':
r = dict(r); r['binary'] = alias
funcs.append(r)
if ov_n:
sources.append(f"# overlays {ov_n}x sig.ov_*.jsonl (sig_image, nins>={minins}) set-sha1={ov_set.hexdigest()}")
out = ["# Cross-binary duplicate function groups (generated by tools/dup_report.py --cross)",
"# sources:", *sources,
"# Collapsible bytes = (count-1)*nins*4 (bytes saved; one member stays the source).",
"# A cross-binary group (binaries>1) is the Phase-12/13 work queue; lead with h_exact",
"# (guaranteed byte-match), h_norm is candidate-only (byte-confirm via dedup_integrate).", ""]
"# (guaranteed byte-match), h_norm is candidate-only (byte-confirm via dedup_integrate).",
f"# Overlay sigs are sig_image (Ghidra-free): h_exact is exact; h_norm is self-consistent.",
f"# Each subsection lists the top {CROSS_CAP} groups by collapsible bytes; totals are stated.", ""]
def emit(groups, tier_letter, cls_letter, title):
out.append(f"### {title} — {len(groups)} groups, "
f"{sum((len(g)-1)*g[0]['nins']*4 for g in groups)} collapsible bytes")
out.append("| gid | bytes | nins | count | binaries | members (binary:addr name) |")
total_bytes = sum((len(g) - 1) * g[0]['nins'] * 4 for g in groups)
cap = f" (top {CROSS_CAP} of {len(groups)} shown)" if len(groups) > CROSS_CAP else ""
out.append(f"### {title} — {len(groups)} groups, {total_bytes} collapsible bytes{cap}")
out.append("| gid | bytes | nins | count | #bin | sample members (binary:addr) |")
out.append("|---|---|---|---|---|---|")
for i, g in enumerate(groups):
for i, g in enumerate(groups[:CROSS_CAP]):
g = sorted(g, key=lambda r: (r['binary'], r['addr']))
rep = g[0]
binset = sorted({r['binary'] for r in g})
members = "; ".join(f"{r['binary']}:{r['addr']} ({r['name']})" for r in g)
out.append(f"| {tier_letter}{cls_letter}{i} | {(len(g)-1)*rep['nins']*4} | {rep['nins']} | "
f"{len(g)} | {','.join(binset)} | {members} |")
nbin = len({r['binary'] for r in g})
shown = "; ".join(f"{r['binary']}:{r['addr']}" for r in g[:MEMBERS_SHOWN])
if len(g) > MEMBERS_SHOWN:
shown += f"; …+{len(g) - MEMBERS_SHOWN}"
out.append(f"| {tier_letter}{cls_letter}{i} | {(len(g) - 1) * rep['nins'] * 4} | {rep['nins']} | "
f"{len(g)} | {nbin} | {shown} |")
out.append("")
for tier, label, tl in (('h_exact', 'byte-identical', 'e'), ('h_norm', 'structural', 'n')):
@@ -84,9 +110,7 @@ def cross_report(minins):
groups = [g for g in idx.values() if len(g) > 1]
cross = [g for g in groups if len({r['binary'] for r in g}) > 1]
intra = [g for g in groups if len({r['binary'] for r in g}) == 1]
# rank by collapsible bytes desc, then a stable key (first member binary:addr)
keyf = lambda g: (-(len(g) - 1) * g[0]['nins'] * 4,
min((r['binary'], r['addr']) for r in g))
keyf = lambda g: (-(len(g) - 1) * g[0]['nins'] * 4, min((r['binary'], r['addr']) for r in g))
cross.sort(key=keyf); intra.sort(key=keyf)
out.append(f"## {tier} ({label})")
out.append("")
@@ -95,7 +119,7 @@ def cross_report(minins):
(ROOT / CROSS_MD).write_text("\n".join(out) + "\n")
for line in out:
if line.startswith('#'):
if line.startswith('#') or line.startswith('##'):
print(line)
tools/sig_image.py
+40 -12
View File
@@ -55,18 +55,42 @@ def read_seeds(path):
return seeds
def bootstrap_seeds(data, vram_base, lo, hi):
"""Discover entries with no Ghidra: every in-range jal target + the region start (jal-closure,
the match_protos anchor model). Misses functions reached ONLY via jump tables — a documented
coverage gap, acceptable for the dedup scan (h_exact still collapses what IS reached)."""
seeds = {lo}
for off in range(lo - vram_base, hi - vram_base, 4):
word = struct.unpack_from("<I", data, off)[0]
ins = make_insn(word, vram_base + off)
if ins.isFunctionCall():
t = ins.getInstrIndexAsVram()
if lo <= t < hi:
seeds.add(t)
def detect_code_end(data, vram_base, lo, hi, run=3):
"""Find the code->data boundary as the first run of `run` consecutive INVALID instructions.
Overlay code decodes ~100% valid (verified: the SC01/077 code prefix is 100% valid, the data
tail drops to 43-95%), so the first sustained invalid run is the transition. A single invalid
word (a rare decode quirk) does not trip it; `run` consecutive does. Returns a vram <= hi."""
bad = 0
o = lo - vram_base
end_off = hi - vram_base
while o + 4 <= end_off:
if make_insn(struct.unpack_from("<I", data, o)[0], vram_base + o).isValid():
bad = 0
else:
bad += 1
if bad >= run:
return vram_base + o - (run - 1) * 4 # back up to the start of the invalid run
o += 4
return hi
def bootstrap_seeds(data, vram_base, entry, hi):
"""Discover entries with no Ghidra by LINEAR PARTITION of the contiguous code: walk from `entry`,
each function is [pos, func_end(pos)], the next starts right after. Stop at the first block with
NO return (func_end hits the hard bound) — that is the code->data transition (data has no regular
`jr $ra` epilogue). Overlays dispatch most code via function-pointer tables (not `jal`), so a
call-graph BFS finds almost nothing; linear partition recovers the whole contiguous-code prefix.
Coverage gap (documented): functions AFTER an embedded data island / jump table, or tail-call
functions ending in `j` (no `jr`), are not reached until splat boundaries land (Phase 13). For the
dedup scan this is conservative — every function found is real; byte-identical overlays match fully."""
seeds = set()
pos = entry
while pos < hi:
end = func_end(data, vram_base, pos, hi)
if end >= hi: # no return found in [pos, hi): left the code region -> stop
break
seeds.add(pos)
pos = end
return seeds
@@ -204,6 +228,10 @@ def main():
lo = int(a.text_lo, 0) if a.text_lo else vram_base
hi = min(int(a.text_hi, 0), img_end) if a.text_hi else img_end
if a.bootstrap and not seeds_map:
# auto-bound the code region (overlays have no splat config yet): stop at the code->data
# transition so the data tail isn't mis-partitioned as functions.
if not a.text_hi:
hi = detect_code_end(data, vram_base, lo, hi)
seeds_map = {s: "" for s in bootstrap_seeds(data, vram_base, lo, hi)}
rows = sign_image(data, vram_base, seeds_map, lo, hi)