From faedd103e4958fa8018633670b9a04210a9efa73 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:05:29 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-26):=20task=202=20=E2=80=94=20tools/?= =?UTF-8?q?family=5Fhseq.py=20full-frontier=20survey=20+=20shared=20word-d?= =?UTF-8?q?iff=20classifier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - family_remap.py: shared classifier (stream_words / reloc_indices / reg_fields / classify_member) — per-instruction diff class RELOC/IMM/IMM_SA/STRUCT, register-drift aware (h_seq ignores registers, so regalloc-drift members are STRUCT-excluded, not templatable). Reused by T1/T2a/T3. - tools/family_hseq.py: cluster ALL unmatched overlay instances by h_seq; per family classify every member vs exemplar (PURE/IMM/MIXED), tag per-location/cross-address/scattered, matched-sibling count, has_mid_jr (§8), exemplar pick (matched-ov077 > matched > draft-ov077 > modal), size band. -> .run/family_hseq.json + docs/family-hseq.md (committed digest). - VERIFIED: fleet 74.8/58.2/30.3 (= PhaseEnd_25 + progress.py exact); tail cross-check 663 families / 186 substantial / 1.847M ins (exact); classification vs Plan-agent PURE-same 62 & IMM 8 exact; 890x134/562x134 PURE per-location + 952x113 #addr21 IMM confirmed. Full frontier: 581 substantial families / 3.22M templatable ins; 345 matched-sibling PURE/IMM families / 1.14M ins = V2/V3 corpus. --- docs/family-hseq.md | 65 +++++++++++ phase-ends/CURRENT_PHASE.md | 5 +- tools/family_hseq.py | 207 ++++++++++++++++++++++++++++++++++++ tools/family_remap.py | 103 ++++++++++++++++++ 4 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 docs/family-hseq.md create mode 100644 tools/family_hseq.py diff --git a/docs/family-hseq.md b/docs/family-hseq.md new file mode 100644 index 000000000..4eae268da --- /dev/null +++ b/docs/family-hseq.md @@ -0,0 +1,65 @@ +# Phase 26 — h_seq family survey (T1) + +> Generated by `tools/family_hseq.py` from the 134 overlay sigs + per-overlay src stubs. Ranked by TEMPLATABLE byte-weight (PURE+IMM members × nins × 4). The byte-gate is the arbiter. + +**Fleet (overlays):** 74.8% fn / 58.2% instr / 30.3% distinct-code matched. Unmatched: 86,178 instances / 5,314,133 ins (55,654 distinct classes). + +**Tail cross-check (Phase-25 close):** 65,833 tail fns / 2,798,915 ins → 663 h_seq families ≥2, **186 substantial (nins≥80) / 1,847,597 ins**. + +**Full frontier (all unmatched by h_seq):** 2897 target families (≥2 members or a matched sibling) + 3553 singletons (Step-D residue). Substantial: **581 families / 3,222,713 templatable ins**, 29 with a matched sibling (zero-crack). Substantial member classes: 18,317 PURE · 39 IMM · 6 STRUCT-excluded. + + +## Top substantial families (by templatable byte-weight) + +| # | nins | members (P/I/S) | #addr/#ov | tag | class | matched | jr | exemplar | templ. ins | +|--:|--:|--|--|--|--|--:|:-:|--|--:| +| 1 | 890 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80178d40 draft-ov077 | 119,260 | +| 2 | 952 | 113 (107/6/0) | 21/113 | scattered | IMM | 0 | Y | 0x8017bebc modal | 107,576 | +| 3 | 562 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8015ae2c draft-ov077 | 75,308 | +| 4 | 536 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8017a4ac draft-ov077 | 71,824 | +| 5 | 493 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8015a3c8 draft-ov077 | 66,062 | +| 6 | 490 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013f350 draft-ov077 | 65,660 | +| 7 | 438 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x801380e0 draft-ov077 | 58,692 | +| 8 | 424 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80131340 draft-ov077 | 56,816 | +| 9 | 371 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80176734 draft-ov077 | 49,714 | +| 10 | 363 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8015444c draft-ov077 | 48,642 | +| 11 | 337 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80159c84 draft-ov077 | 45,158 | +| 12 | 329 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013c414 draft-ov077 | 44,086 | +| 13 | 327 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80176218 draft-ov077 | 43,818 | +| 14 | 312 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8013faf8 draft-ov077 | 41,808 | +| 15 | 304 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8014d820 draft-ov077 | 40,736 | +| 16 | 289 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80135eb0 draft-ov077 | 38,726 | +| 17 | 279 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x801670e4 draft-ov077 | 37,386 | +| 18 | 272 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013b83c draft-ov077 | 36,448 | +| 19 | 271 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8015b950 draft-ov077 | 36,314 | +| 20 | 260 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80140958 draft-ov077 | 34,840 | +| 21 | 249 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8013cb84 draft-ov077 | 33,366 | +| 22 | 240 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8013d53c draft-ov077 | 32,160 | +| 23 | 231 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80175da8 draft-ov077 | 30,954 | +| 24 | 226 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8012956c draft-ov077 | 30,284 | +| 25 | 222 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8015c32c draft-ov077 | 29,748 | +| 26 | 108 | 267 (267/0/0) | 2/134 | cross-address | PURE | 1 | · | 0x8014dd8c matched-ov077 | 28,836 | +| 27 | 213 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013ffd8 draft-ov077 | 28,542 | +| 28 | 209 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8016cbc0 draft-ov077 | 28,006 | +| 29 | 204 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8015ea3c draft-ov077 | 27,336 | +| 30 | 201 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8015d738 draft-ov077 | 26,934 | +| 31 | 198 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013bd74 draft-ov077 | 26,532 | +| 32 | 198 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x801412a8 draft-ov077 | 26,532 | +| 33 | 188 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8015f448 draft-ov077 | 25,192 | +| 34 | 188 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8016ab6c draft-ov077 | 25,192 | +| 35 | 188 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80175ab8 draft-ov077 | 25,192 | +| 36 | 93 | 268 (268/0/0) | 2/134 | cross-address | PURE | 0 | · | 0x8014ffdc draft-ov077 | 24,924 | +| 37 | 183 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8014032c draft-ov077 | 24,522 | +| 38 | 181 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80135a4c draft-ov077 | 24,254 | +| 39 | 174 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8017ae2c draft-ov077 | 23,316 | +| 40 | 173 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x80154c24 draft-ov077 | 23,182 | +| 41 | 165 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80178004 draft-ov077 | 22,110 | +| 42 | 162 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80169bbc draft-ov077 | 21,708 | +| 43 | 161 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x80160534 draft-ov077 | 21,574 | +| 44 | 161 | 133 (133/0/0) | 1/133 | per-location | PURE | 1 | · | 0x8016dc20 matched-ov077 | 21,413 | +| 45 | 158 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x801299c8 draft-ov077 | 21,172 | +| 46 | 156 | 133 (133/0/0) | 1/133 | per-location | PURE | 1 | · | 0x8016df5c matched-ov077 | 20,748 | +| 47 | 154 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | Y | 0x8013c0f8 draft-ov077 | 20,636 | +| 48 | 155 | 133 (133/0/0) | 1/133 | per-location | PURE | 1 | · | 0x801365b8 matched-ov077 | 20,615 | +| 49 | 154 | 133 (133/0/0) | 1/133 | per-location | PURE | 1 | · | 0x80144090 matched-ov077 | 20,482 | +| 50 | 148 | 134 (134/0/0) | 1/134 | per-location | PURE | 0 | · | 0x8016d1d8 draft-ov077 | 19,832 | diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 8a038a8a1..3b0684194 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -12,7 +12,7 @@ The Phase-25 h_seq reframe: the "unique tail" is really per-location families - [x] **Task 0 — Bootstrap** `[xHigh]` — this file + harness task list (R28). *(completes with this commit)* - [x] **Task 1 — Remap core: extended reloc tracker + single-pass substitution** `[xHigh]` — `family_remap.py`: hi propagates through add/addu index adds; `symbol_map`/`remap` gained backward-compatible `to_addr=None` (cross-address) + `imm_map` hook; single-pass simultaneous substitution + self-rename. norm_stream/h_norm UNTOUCHED. Discovery → cookbook §40b (R30) + decision-log (R31). **V0 PASS** (`.run/v0_reloc.py`): 22/22 regression (`func_80141100` NEW==OLD), 15/15 fix (`func_801407F4` vs splat .s, recovers `D_80187B88/90/B0`), cross-addr symbol_map clean. **V1 PASS** (`.run/v1_regression.py`): 160 real h_norm pairs, 96 SAME, **0 lost**, differences are strict indexed-reloc improvements. *(committed)* -- [ ] **Task 2 — `tools/family_hseq.py` + committed manifest** `[xHigh]` — full-frontier survey (tail + h_norm reach≥2); per family: members/nins/byte-weight/#addr/#ovs, tag {per-location|cross-address|scattered}, diff_class {PURE|IMM(+positions)|STRUCT-EXCLUDED} via shared classifier, matched-sibling count, `has_mid_jr` (§8 risk), exemplar pick (matched ▸ ov077 ▸ modal-addr), size band → `.run/family_hseq.json` + `docs/family-hseq.md`. Verify: 663/186/1.85M ±, 3 named families, classification totals, `progress.py --weighted` cross-check. +- [x] **Task 2 — `tools/family_hseq.py` + committed manifest** `[xHigh]` — full-frontier survey; shared word-diff classifier added to `family_remap.py` (`stream_words`/`reloc_indices`/`reg_fields`/`classify_member` — PURE/IMM/STRUCT, register-drift aware). → `.run/family_hseq.json` + `docs/family-hseq.md`. **VERIFIED:** fleet 74.8/58.2/30.3 (= PhaseEnd_25 & progress.py to the decimal); tail cross-check **663 families / 186 substantial / 1.847M ins** (exact); classification vs Plan-agent table **PURE-same 62, IMM 8 exact, PURE-cross 114≈103**; 890×134/562×134 PURE per-location + 952×113 #addr21 IMM confirmed; full frontier = 581 substantial families / 3.22M templatable ins, **345 matched-sibling PURE/IMM families / 1.14M ins = the V2/V3 zero-crack corpus**. *(committed)* - [ ] **Task 3 — Imm engine (T2a, 3 tiers) + cross-address delta (T2b)** `[xHigh]` — Tier 1 simultaneous value-replace (asm-side ambiguity check; C tokenizer; signed imm16/sa/shift-vs-multiply spellings); Tier 2 targeted probe (~1–6 values/family; match_one-pipeline recompile; nins+h_seq guards; FRAGILE); Tier 3 member-fail → agent queue. T2b: h_seq sibling discovery at any addr; self-rename; distinct "link-undef" logging (fallback: `config/symbols.ov_*.txt` append + re-extract, second pass). Verify: 25 matched-matched imm pairs A→B regression. - [ ] **Task 4 — `family_sweep --hseq` mode** `[xHigh]` — manifest-driven; member word-diff pre-filter (STRUCT skip); stage `.run/sweep//`; existing two-phase stage→`harvest_verify` gate; `--only/--reconcile/--no-preclassify` carry over; 30-sec sibling-TU compile probe before ×N claims (pin-free, §42e). - [ ] **Task 5 — Zero-crack validation harvests (V2/V3) — GO/NO-GO** `[xHigh; UC if fan-out — R26 prompt]` — V2: the 63-fam/0.31M-ins tracker-miss corpus (incl. `0x801407f4`/`0x8015d5e8`/`0x8015f118` ×118) → real banks + measured success rate + failure taxonomy. V3: 12 imm families with matched exemplars (`0x8016b448` ×134, `0x8017dde8` ×108). Scale NOTHING until rates are on the table; poor → fix tooling or assisted-template fallback. @@ -24,7 +24,7 @@ The Phase-25 h_seq reframe: the "unique tail" is really per-location families - [ ] **Task 11 — Step-D residue map** `[xHigh]` — true singletons (~0.27M ins) + 5 behemoths → Phase-27 input doc. NO execution. - [ ] **Task 12 — PhaseEnd** `[Max — Tier 1; R27 prompt]` — P7 walk, milestone demo, gate 2, `PhaseEnd_Phase26.md`, worklog → `logs/Phase26.md` (R19), in-file recap (R25), decision-log current (R31). -## ▶ CURRENT TASK: Task 2 — `tools/family_hseq.py` + committed manifest (Opus/xHigh) +## ▶ CURRENT TASK: Task 3 — the imm engine (T2a, 3 tiers) + cross-address delta (T2b) (Opus/xHigh) ## Milestone (gate 2 — structural completion, per Drew) @@ -47,5 +47,6 @@ On approval → `/model opus` + `/effort xHigh` (Tasks 0–4; ALL Fable5 via `Ag ## Log +- **2026-07-11 (session 1, Task 2):** Built `tools/family_hseq.py` (full-frontier h_seq survey) + the shared word-diff classifier in `family_remap.py` (PURE/IMM/STRUCT, register-drift aware). Reproduces the fleet metrics + the 663/186/1.85M tail cross-check exactly; classification matches the Plan-agent table; 581 substantial target families / 3.22M templatable ins, 345 matched-sibling families / 1.14M ins = the zero-crack corpus. `docs/family-hseq.md` committed. Committed. - **2026-07-11 (session 1, Task 1):** Extended `reloc_targets` for the add/addu indexed-global idiom (the "reach-1 tail" was largely this tracker blind spot, not unique code — decision-log + cookbook §40b). `symbol_map`/`remap` gained backward-compatible `to_addr` (cross-address T2b) + `imm_map` hook; sequential→single-pass substitution (fixes the latent chained-rename bug). norm_stream/h_norm untouched. V0 (22/22 regression, 15/15 fix vs splat .s) + V1 (160 pairs, 0 regressions) both green. Committed. - **2026-07-11 (session 1, planning):** Phase Start (Tier 1, Fable5+Max+plan-mode). Megaplan analyzed; survey reproduced from sigs+src (R14) — fleet metrics match PhaseEnd_25 exactly; 186-not-986 substantial-family correction; #2 family is cross-address; 93 matched-sibling families found (0.51M ins). Tooling recon (Explore) + design pressure-test (Plan agent) → **the reloc-tracker blind-spot discovery** (addu-preserves-hi; 890×121 family is PURE reloc; 63-fam zero-crack corpus; sequential-substitution latent bug). Drew's gate-1 decisions: carried queue → end of phase; structural milestone. Plan approved; task list built (R28). Task 0 complete with this commit. diff --git a/tools/family_hseq.py b/tools/family_hseq.py new file mode 100644 index 000000000..3f929f5d6 --- /dev/null +++ b/tools/family_hseq.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Phase-26 T1: the ranked h_seq FAMILY survey — the finish-the-decomp target map. + +The Phase-25 close (R14): the "unique tail" is a strict-`h_norm` artifact. Re-cluster the unmatched frontier +by the looser `h_seq` (mnemonic skeleton — same instruction sequence, ignoring registers/immediates/relocs) +and ~90% collapses into per-location FAMILIES: the same engine fn recurring across ~120 overlays, differing +only in per-overlay symbols (RELOC, §40) + a few immediates (IMM, §46/T2a). This groups ALL unmatched overlay +instances by h_seq and, per family, classifies every member against an exemplar via the shared word-diff +classifier (family_remap.classify_member) into PURE (reloc-only, mechanically bankable now) / IMM (needs the +T2a immediate engine) / STRUCT (register-alloc drift or an h_seq collision — NOT templatable, excluded). + +Matched-vs-unmatched = per-overlay INCLUDE_ASM stubs in that overlay's own src (ground truth; reproduces +progress.py's 74.8% fn / 58.2% instr / 30.3% distinct exactly). Exemplar pick, most-useful first: a MATCHED +member (its C exists → template for ~0 tokens; prefer ov_SC01_077) ▸ an unmatched ov_SC01_077 member (cached +Ghidra-C for drafting) ▸ the member at the family's modal address. The whole-binary byte-gate stays the sole +arbiter (G3/P9): this survey RANKS and CLASSES; it never asserts a match. + + .venv/bin/python tools/family_hseq.py + -> .run/family_hseq.json (full, ranked by templatable byte-weight) + docs/family-hseq.md (digest, committed) + +Ground truth = the sigs (`make sig-overlays`) + src stubs. Companion to tools/family_manifest.py (h_norm). +""" +import json, glob, re, collections, sys +sys.path.insert(0, "tools") +import family_remap as FR + +SUBSTANTIAL = 80 # nins >= this is the campaign band (below = mid/tiny, collision-prone) +TINY = 16 # nins < this is the coincidental-h_seq-collision band (report, don't campaign) +EX_OV = "ov_SC01_077" # the canonical drafting overlay (cached Ghidra-C) + + +def load(): + """-> instances[(ov, addr, nins, h_exact, h_norm, h_seq, matched)], and nins/matched maps.""" + stubs = {} + for ovdir in sorted(glob.glob("src/ov_*")): + ov = ovdir.split("/")[-1] + txt = "".join(open(f).read() for f in glob.glob(f"{ovdir}/*.c")) + stubs[ov] = set(int(m, 16) for m in re.findall(r'INCLUDE_ASM\([^)]*,\s*func_([0-9A-Fa-f]+)\)', txt)) + inst = [] + for p in sorted(glob.glob(".run/sig.ov_*.jsonl")): + ov = f"ov_{p.split('sig.ov_')[1][:-6]}" + st = stubs.get(ov) + if st is None: + continue + for line in open(p): + line = line.strip() + if not line: + continue + d = json.loads(line) + a = int(d["addr"], 16) + inst.append((ov, a, d["nins"], d["h_exact"], d["h_norm"], d["h_seq"], a not in st)) + return inst + + +def has_mid_jr(words): + """a `jr` on a non-$ra register = a jump-table dispatch (rodata jtbl → §8 workflow risk).""" + for w in words: + if (w >> 26) == 0 and (w & 0x3F) == 0x08 and ((w >> 21) & 0x1F) != 31: + return True + return False + + +def pick_exemplar(members, matched): + """(ov, addr, kind) — matched(ov077) ▸ matched(any) ▸ unmatched ov077 ▸ modal-addr member.""" + m077 = [m for m in matched if m[0] == EX_OV] + if m077: + return (m077[0][0], m077[0][1], "matched-ov077") + if matched: + m = min(matched, key=lambda x: (x[0], x[1])) + return (m[0], m[1], "matched") + u077 = [m for m in members if m[0] == EX_OV] + if u077: + return (u077[0][0], u077[0][1], "draft-ov077") + modal = collections.Counter(a for _, a in members).most_common(1)[0][0] + cand = sorted(m for m in members if m[1] == modal) + return (cand[0][0], cand[0][1], "modal") + + +def main(): + inst = load() + nins_of = {(o, a): n for o, a, n, *_ in inst} + + # ---- honest fleet metrics (cross-check vs progress.py --weighted) ---- + tot, tot_i = len(inst), sum(x[2] for x in inst) + un = [x for x in inst if x[6] is False] + un_i = sum(x[2] for x in un) + ue = collections.defaultdict(list) + for x in inst: + ue[x[3]].append(x) + mat_hex = {h for h, v in ue.items() if any(z[6] for z in v)} + di_tot = sum(v[0][2] for v in ue.values()) + di_mat = sum(v[0][2] for h, v in ue.items() if h in mat_hex) + metrics = { + "fn_count_matched_pct": round(100 * (tot - len(un)) / tot, 1), + "instr_weighted_matched_pct": round(100 * (tot_i - un_i) / tot_i, 1), + "distinct_matched_pct": round(100 * di_mat / di_tot, 1), + "unmatched_instances": len(un), "unmatched_ins": un_i, + "distinct_unmatched_classes": len(ue) - len(mat_hex), + } + + # ---- tail cross-check (reproduce the Phase-25 close: 663 families / 186 substantial / 1.85M ins) ---- + uncls = {h: v for h, v in ue.items() if h not in mat_hex} + reach_hn = collections.Counter(v[0][4] for v in uncls.values()) + tail = {h: v for h, v in uncls.items() if reach_hn[v[0][4]] == 1} + tfam = collections.defaultdict(list) + for h, v in tail.items(): + tfam[v[0][5]].append(v[0]) + tmulti = {k: v for k, v in tfam.items() if len(v) >= 2} + tsub = {k: v for k, v in tmulti.items() if v[0][2] >= SUBSTANTIAL} + tailcheck = { + "tail_fns": sum(len(v) for v in tail.values()), "tail_ins": sum(v[0][2] for v in tail.values()), + "hseq_families_ge2": len(tmulti), "substantial_families": len(tsub), + "substantial_ins": sum(x[2] for v in tsub.values() for x in v), + } + + # ---- primary survey: ALL unmatched instances grouped by h_seq (the full frontier) ---- + by_seq = collections.defaultdict(lambda: {"members": [], "matched": []}) + for o, a, n, hx, hn, hs, matched in inst: + (by_seq[hs]["matched"] if matched else by_seq[hs]["members"]).append((o, a)) + + families = [] + for hs, g in by_seq.items(): + members, matched = g["members"], g["matched"] + if not members: # fully matched -> done + continue + nins = nins_of[members[0]] + ex_ov, ex_addr, ex_kind = pick_exemplar(members, matched) + ex_words = FR.stream_words(ex_ov, ex_addr, nins) + cnt = {"PURE": 0, "IMM": 0, "STRUCT": 0, "LEN": 0} + for (o, a) in members: + if (o, a) == (ex_ov, ex_addr): + cnt["PURE"] += 1 # the exemplar vs itself is trivially pure + continue + cls, _ = FR.classify_member(ex_words, FR.stream_words(o, a, nins)) + cnt[cls] = cnt.get(cls, 0) + 1 + n_templatable = cnt["PURE"] + cnt["IMM"] + diff_class = ("PURE" if cnt["IMM"] == 0 and cnt["STRUCT"] == 0 and cnt["LEN"] == 0 + else "IMM" if cnt["STRUCT"] == 0 and cnt["LEN"] == 0 else "MIXED") + addrs = {a for _, a in members} + ovs = {o for o, _ in members} + tag = ("per-location" if len(addrs) == 1 + else "cross-address" if len(addrs) <= 8 else "scattered") + band = ("substantial" if nins >= SUBSTANTIAL else "tiny" if nins < TINY else "mid") + families.append({ + "h_seq": hs, "nins": nins, "band": band, + "n_members": len(members), "n_matched": len(matched), "n_templatable": n_templatable, + "n_addrs": len(addrs), "n_ovs": len(ovs), "addr_tag": tag, + "diff_class": diff_class, "cls_counts": cnt, "has_mid_jr": has_mid_jr(ex_words or []), + "byte_weight_templatable": n_templatable * nins * 4, + "byte_weight_all": len(members) * nins * 4, + "exemplar": {"ov": ex_ov, "addr": f"0x{ex_addr:08x}", "kind": ex_kind}, + "members": [[o, f"0x{a:08x}"] for o, a in sorted(members)], + "matched_members": [[o, f"0x{a:08x}"] for o, a in sorted(matched)], + }) + families.sort(key=lambda f: -f["byte_weight_templatable"]) + + out = {"metrics": metrics, "tailcheck": tailcheck, "families": families} + json.dump(out, open(".run/family_hseq.json", "w")) + + # ---- digest ---- + multi = [f for f in families if f["n_members"] >= 2 or f["n_matched"] >= 1] + singles = [f for f in families if f not in multi] + subst = [f for f in multi if f["band"] == "substantial"] + with_matched = [f for f in subst if f["n_matched"] >= 1] + templ_ins_subst = sum(f["byte_weight_templatable"] for f in subst) // 4 + L = [] + L.append("# Phase 26 — h_seq family survey (T1)\n") + L.append(f"> Generated by `tools/family_hseq.py` from the 134 overlay sigs + per-overlay src stubs. " + f"Ranked by TEMPLATABLE byte-weight (PURE+IMM members × nins × 4). The byte-gate is the arbiter.\n") + L.append(f"**Fleet (overlays):** {metrics['fn_count_matched_pct']}% fn / " + f"{metrics['instr_weighted_matched_pct']}% instr / {metrics['distinct_matched_pct']}% distinct-code " + f"matched. Unmatched: {metrics['unmatched_instances']:,} instances / {metrics['unmatched_ins']:,} ins " + f"({metrics['distinct_unmatched_classes']:,} distinct classes).\n") + L.append(f"**Tail cross-check (Phase-25 close):** {tailcheck['tail_fns']:,} tail fns / " + f"{tailcheck['tail_ins']:,} ins → {tailcheck['hseq_families_ge2']} h_seq families ≥2, " + f"**{tailcheck['substantial_families']} substantial (nins≥{SUBSTANTIAL}) / " + f"{tailcheck['substantial_ins']:,} ins**.\n") + tw = sum(f['cls_counts']['PURE'] for f in subst) + ti = sum(f['cls_counts']['IMM'] for f in subst) + ts = sum(f['cls_counts']['STRUCT'] for f in subst) + L.append(f"**Full frontier (all unmatched by h_seq):** {len(multi)} target families " + f"(≥2 members or a matched sibling) + {len(singles)} singletons (Step-D residue). " + f"Substantial: **{len(subst)} families / {templ_ins_subst:,} templatable ins**, " + f"{len(with_matched)} with a matched sibling (zero-crack). " + f"Substantial member classes: {tw:,} PURE · {ti:,} IMM · {ts:,} STRUCT-excluded.\n") + L.append("\n## Top substantial families (by templatable byte-weight)\n") + L.append("| # | nins | members (P/I/S) | #addr/#ov | tag | class | matched | jr | exemplar | templ. ins |") + L.append("|--:|--:|--|--|--|--|--:|:-:|--|--:|") + for i, f in enumerate(subst[:50], 1): + c = f["cls_counts"] + L.append(f"| {i} | {f['nins']} | {f['n_members']} ({c['PURE']}/{c['IMM']}/{c['STRUCT']}) | " + f"{f['n_addrs']}/{f['n_ovs']} | {f['addr_tag']} | {f['diff_class']} | {f['n_matched']} | " + f"{'Y' if f['has_mid_jr'] else '·'} | {f['exemplar']['addr']} {f['exemplar']['kind']} | " + f"{f['byte_weight_templatable']//4:,} |") + open("docs/family-hseq.md", "w").write("\n".join(L) + "\n") + + print(f"fleet: {metrics['fn_count_matched_pct']}% fn / {metrics['instr_weighted_matched_pct']}% instr / " + f"{metrics['distinct_matched_pct']}% distinct") + print(f"tailcheck: {tailcheck['hseq_families_ge2']} families ≥2, {tailcheck['substantial_families']} " + f"substantial / {tailcheck['substantial_ins']:,} ins (expect ~663 / ~186 / ~1.85M)") + print(f"frontier: {len(multi)} target families ({len(subst)} substantial, {len(with_matched)} w/ matched sib) " + f"+ {len(singles)} singletons") + print(f"-> .run/family_hseq.json + docs/family-hseq.md") + + +if __name__ == "__main__": + main() diff --git a/tools/family_remap.py b/tools/family_remap.py index 552dd8fb3..fb007b629 100644 --- a/tools/family_remap.py +++ b/tools/family_remap.py @@ -88,6 +88,109 @@ def reloc_targets(ov, addr): return out +# ---- Phase-26 shared word-diff classifier (family_hseq survey T1, imm engine T2a, sweep pre-filter T3) ---- +# Two members of an h_seq family have IDENTICAL mnemonic sequences but may differ in reloc fields +# (symbol-remappable), in true immediates / shift amounts (imm-substitutable, T2a), OR in REGISTER +# allocation (NOT templatable — h_seq ignores registers). Classify each differing instruction so a +# family/member is bucketed PURE (reloc-only) / IMM / STRUCT-EXCLUDED (regalloc drift or collision). + +_IMG_CACHE = {} + + +def _img(ov): + if ov not in _IMG_CACHE: + p = img_path(ov) + _IMG_CACHE[ov] = open(p, "rb").read() if p else None + return _IMG_CACHE[ov] + + +def stream_words(ov, addr, nins): + """the nins raw instruction words of a function from its overlay image (cached).""" + data = _img(ov) + if data is None: + return None + off = addr - VRAM + return [struct.unpack_from(" originating lui index + for k, w in enumerate(words): + op = w >> 26 + if op in (2, 3): + jal.add(k) + elif op == 0x0F: + pend[(w >> 16) & 0x1F] = k + elif op in LO_OPS: + rs = (w >> 21) & 0x1F + if rs in pend: + lo.add(k) + lui_addr.add(pend[rs]) + del pend[rs] + pend.pop((w >> 16) & 0x1F, None) + elif op == 0: + funct = w & 0x3F + rd = (w >> 11) & 0x1F + if funct in (0x20, 0x21): + rs, rt = (w >> 21) & 0x1F, (w >> 16) & 0x1F + if rs in pend: + pend[rd] = pend[rs] + elif rt in pend: + pend[rd] = pend[rt] + else: + pend.pop(rd, None) + else: + pend.pop(rd, None) + return jal | lui_addr | lo + + +def reg_fields(w): + """the register + opcode-identity fields (everything EXCEPT the imm / jump-target / shift-amount). + Equal reg_fields with unequal words ⟹ the diff is purely immediate/target/sa (not regalloc).""" + op = w >> 26 + if op == 0: # R-type: rs, rt, rd, funct (sa excluded) + return (0, (w >> 21) & 0x1F, (w >> 16) & 0x1F, (w >> 11) & 0x1F, w & 0x3F) + if op in (2, 3): # j / jal: target excluded + return (op,) + if op == 0x0F: # lui: rt only (hi excluded) + return (op, (w >> 16) & 0x1F) + return (op, (w >> 21) & 0x1F, (w >> 16) & 0x1F) # I-type: rs, rt (imm excluded) + + +_SHIFT_FUNCTS = (0x00, 0x02, 0x03) # sll srl sra (sa is the immediate); sllv/srlv/srav excluded + + +def classify_member(words_ex, words_sib): + """returns (cls, positions): cls in {PURE, IMM, STRUCT, LEN}; positions = [(idx, kind)] for each + DIFFERING instruction, kind in {RELOC, IMM, IMM_SA, STRUCT}. Reuses the extended reloc tracker.""" + if words_ex is None or words_sib is None: + return "LEN", [] + if len(words_ex) != len(words_sib): + return "LEN", [] + rel = reloc_indices(words_ex) + positions = [] + for k, (a, b) in enumerate(zip(words_ex, words_sib)): + if a == b: + continue + if reg_fields(a) != reg_fields(b): # opcode/register/funct drift -> not templatable + positions.append((k, "STRUCT")) + elif k in rel: + positions.append((k, "RELOC")) + elif (a >> 26) == 0: + positions.append((k, "IMM_SA" if (a & 0x3F) in _SHIFT_FUNCTS else "STRUCT")) + else: + positions.append((k, "IMM")) # I-type non-address immediate (incl. const-lui hi) + if any(c == "STRUCT" for _, c in positions): + cls = "STRUCT" + elif any(c in ("IMM", "IMM_SA") for _, c in positions): + cls = "IMM" + else: + cls = "PURE" + return cls, positions + + def symbol_map(addr, from_ov, to_ov, to_addr=None): """{exemplar_name: sibling_name} for the per-overlay symbols (positional zip). None,err if not clean. to_addr defaults to addr (same-address h_norm sibling); pass it for a cross-address (T2b) sibling."""