diff --git a/docs/family-hseq.md b/docs/family-hseq.md index d4f7a6cac..83a78d9b7 100644 --- a/docs/family-hseq.md +++ b/docs/family-hseq.md @@ -1,67 +1,67 @@ # Phase 26 — h_seq family survey (T1) -> Generated by `tools/family_hseq.py` from the 138 overlay sigs + per-overlay src stubs. Ranked by TEMPLATABLE byte-weight (PURE+IMM members × nins × 4). The byte-gate is the arbiter. +> Generated by `tools/family_hseq.py` from the 139 overlay sigs + per-overlay src stubs. Ranked by TEMPLATABLE byte-weight (PURE+IMM members × nins × 4). The byte-gate is the arbiter. > -> **Scope: the 138 OVERLAYS only** (no main, no resident) · generated at HEAD `commit:1466` · stub set derived from `corpus.stubs` — the same oracle `progress.py` counts, so same-tree overlay totals agree by construction; compare digests only at the same HEAD. +> **Scope: the 139 OVERLAYS only** (no main, no resident) · generated at HEAD `commit:1497` · stub set derived from `corpus.stubs` — the same oracle `progress.py` counts, so same-tree overlay totals agree by construction; compare digests only at the same HEAD. -**Fleet (overlays):** 96.9% fn / 95.2% instr / 90.5% distinct-code matched. Unmatched: 11,031 instances / 624,451 ins (9,371 distinct classes). +**Fleet (overlays):** 96.9% fn / 95.6% instr / 90.7% distinct-code matched. Unmatched: 10,878 instances / 579,368 ins (9,358 distinct classes). -**Tail cross-check (Phase-25 close):** 7,070 tail fns / 370,683 ins → 417 h_seq families ≥2, **99 substantial (nins≥80) / 66,568 ins**. +**Tail cross-check (Phase-25 close):** 7,059 tail fns / 356,882 ins → 414 h_seq families ≥2, **97 substantial (nins≥80) / 54,055 ins**. -**Full frontier (all unmatched by h_seq):** 1964 target families (≥2 members or a matched sibling) + 3766 singletons (Step-D residue). Substantial: **330 families / 184,225 templatable ins**, 34 with a matched sibling (zero-crack). Substantial member classes: 1,142 PURE · 23 IMM · 6 STRUCT-excluded. +**Full frontier (all unmatched by h_seq):** 1962 target families (≥2 members or a matched sibling) + 3772 singletons (Step-D residue). Substantial: **326 families / 139,821 templatable ins**, 32 with a matched sibling (zero-crack). Substantial member classes: 982 PURE · 23 IMM · 6 STRUCT-excluded. ## Top substantial families (by templatable byte-weight) | # | nins | members (P/I/S) | #addr/#ov | tag | class | matched | jr | exemplar | templ. ins | |--:|--:|--|--|--|--|--:|:-:|--|--:| -| 1 | 198 | 137 (137/0/0) | 1/137 | per-location | PURE | 1 | · | 0x801412a8 matched-ov077 | 27,126 | -| 2 | 947 | 16 (16/0/0) | 6/16 | cross-address | PURE | 0 | Y | 0x8017c974 draft-ov077 | 15,152 | -| 3 | 246 | 16 (16/0/0) | 6/16 | cross-address | PURE | 0 | · | 0x8017c294 draft-ov077 | 3,936 | -| 4 | 611 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | · | 0x80186e24 modal | 3,055 | -| 5 | 551 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80189540 modal | 2,755 | -| 6 | 125 | 21 (21/0/0) | 21/21 | scattered | PURE | 1 | Y | 0x8018a564 matched | 2,625 | -| 7 | 557 | 4 (4/0/0) | 3/4 | cross-address | PURE | 1 | Y | 0x80189030 matched | 2,228 | -| 8 | 137 | 14 (14/0/0) | 14/14 | scattered | PURE | 1 | Y | 0x801884d8 matched | 1,918 | -| 9 | 281 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80186270 modal | 1,686 | -| 10 | 470 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | Y | 0x80191320 modal | 1,410 | -| 11 | 279 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017f2d4 modal | 1,395 | -| 12 | 673 | 2 (2/0/0) | 2/2 | cross-address | PURE | 1 | Y | 0x8019059c matched | 1,346 | -| 13 | 263 | 5 (5/0/0) | 5/5 | cross-address | PURE | 6 | · | 0x80182fd4 matched | 1,315 | -| 14 | 254 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80188e1c modal | 1,270 | -| 15 | 251 | 5 (5/0/0) | 5/5 | cross-address | PURE | 1 | Y | 0x80180b04 matched | 1,255 | -| 16 | 198 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80185f58 modal | 1,188 | -| 17 | 385 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | Y | 0x8018ff98 modal | 1,155 | -| 18 | 185 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80187aec modal | 1,110 | -| 19 | 324 | 3 (3/0/0) | 2/3 | cross-address | PURE | 0 | Y | 0x80183324 draft-ov077 | 972 | -| 20 | 95 | 10 (10/0/0) | 9/10 | scattered | PURE | 0 | Y | 0x80185d70 modal | 950 | -| 21 | 231 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x8017f83c modal | 924 | -| 22 | 288 | 3 (3/0/0) | 3/3 | cross-address | PURE | 3 | Y | 0x80184c74 matched | 864 | -| 23 | 166 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80188b84 modal | 830 | -| 24 | 165 | 5 (5/0/0) | 5/5 | cross-address | PURE | 0 | Y | 0x801818f0 modal | 825 | -| 25 | 406 | 2 (2/0/0) | 2/2 | cross-address | PURE | 0 | Y | 0x80184938 modal | 812 | -| 26 | 196 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x8017ffd0 modal | 784 | -| 27 | 257 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x80192b60 modal | 771 | -| 28 | 255 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x8018e9bc modal | 765 | -| 29 | 254 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x80192768 modal | 762 | -| 30 | 188 | 4 (4/0/0) | 1/4 | per-location | PURE | 134 | Y | 0x8016ab6c matched-ov077 | 752 | -| 31 | 187 | 4 (4/0/0) | 1/4 | per-location | PURE | 134 | · | 0x8013dd68 matched-ov077 | 748 | -| 32 | 147 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017ed5c modal | 735 | -| 33 | 181 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | · | 0x8017d7c0 modal | 724 | -| 34 | 710 | 1 (1/0/0) | 1/1 | per-location | PURE | 2 | Y | 0x80191c50 matched | 710 | -| 35 | 341 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80184bd8 modal | 682 | -| 36 | 134 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017eb44 modal | 670 | -| 37 | 93 | 7 (7/0/0) | 2/7 | cross-address | PURE | 0 | · | 0x8017c6a0 modal | 651 | -| 38 | 108 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x8017eeec modal | 648 | -| 39 | 320 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80181b00 modal | 640 | -| 40 | 317 | 2 (2/0/0) | 2/1 | cross-address | PURE | 0 | · | 0x8017fdf8 modal | 634 | -| 41 | 211 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x8018e8a0 modal | 633 | -| 42 | 207 | 3 (3/0/0) | 3/3 | cross-address | PURE | 2 | · | 0x801880ec matched | 621 | -| 43 | 124 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | · | 0x8017defc modal | 620 | -| 44 | 154 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x801844c8 modal | 616 | -| 45 | 203 | 3 (0/3/0) | 2/3 | cross-address | IMM | 7 | · | 0x8017e804 matched | 609 | -| 46 | 304 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80185ef8 modal | 608 | -| 47 | 101 | 6 (3/3/0) | 6/3 | cross-address | IMM | 0 | · | 0x8018de60 modal | 606 | -| 48 | 201 | 3 (3/0/0) | 2/3 | cross-address | PURE | 0 | Y | 0x80181be4 draft-ov077 | 603 | -| 49 | 99 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | · | 0x80187960 modal | 594 | -| 50 | 148 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | · | 0x80181a30 modal | 592 | +| 1 | 246 | 16 (16/0/0) | 6/16 | cross-address | PURE | 0 | · | 0x8017c294 draft-ov077 | 3,936 | +| 2 | 611 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | · | 0x80186e24 modal | 3,055 | +| 3 | 551 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80189540 modal | 2,755 | +| 4 | 125 | 21 (21/0/0) | 21/21 | scattered | PURE | 1 | Y | 0x8018a564 matched | 2,625 | +| 5 | 557 | 4 (4/0/0) | 3/4 | cross-address | PURE | 1 | Y | 0x80189030 matched | 2,228 | +| 6 | 137 | 14 (14/0/0) | 14/14 | scattered | PURE | 1 | Y | 0x801884d8 matched | 1,918 | +| 7 | 281 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80186270 modal | 1,686 | +| 8 | 470 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | Y | 0x80191320 modal | 1,410 | +| 9 | 279 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017f2d4 modal | 1,395 | +| 10 | 673 | 2 (2/0/0) | 2/2 | cross-address | PURE | 1 | Y | 0x8019059c matched | 1,346 | +| 11 | 254 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80188e1c modal | 1,270 | +| 12 | 251 | 5 (5/0/0) | 5/5 | cross-address | PURE | 1 | Y | 0x80180b04 matched | 1,255 | +| 13 | 198 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80185f58 modal | 1,188 | +| 14 | 385 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | Y | 0x8018ff98 modal | 1,155 | +| 15 | 185 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x80187aec modal | 1,110 | +| 16 | 324 | 3 (3/0/0) | 2/3 | cross-address | PURE | 0 | Y | 0x80183324 draft-ov077 | 972 | +| 17 | 95 | 10 (10/0/0) | 9/10 | scattered | PURE | 0 | Y | 0x80185d70 modal | 950 | +| 18 | 231 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x8017f83c modal | 924 | +| 19 | 288 | 3 (3/0/0) | 3/3 | cross-address | PURE | 3 | Y | 0x80184c74 matched | 864 | +| 20 | 166 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x80188b84 modal | 830 | +| 21 | 165 | 5 (5/0/0) | 5/5 | cross-address | PURE | 0 | Y | 0x801818f0 modal | 825 | +| 22 | 406 | 2 (2/0/0) | 2/2 | cross-address | PURE | 0 | Y | 0x80184938 modal | 812 | +| 23 | 196 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x8017ffd0 modal | 784 | +| 24 | 257 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x80192b60 modal | 771 | +| 25 | 255 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x8018e9bc modal | 765 | +| 26 | 254 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x80192768 modal | 762 | +| 27 | 188 | 4 (4/0/0) | 1/4 | per-location | PURE | 134 | Y | 0x8016ab6c matched-ov077 | 752 | +| 28 | 187 | 4 (4/0/0) | 1/4 | per-location | PURE | 134 | · | 0x8013dd68 matched-ov077 | 748 | +| 29 | 147 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017ed5c modal | 735 | +| 30 | 181 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | · | 0x8017d7c0 modal | 724 | +| 31 | 341 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80184bd8 modal | 682 | +| 32 | 134 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | Y | 0x8017eb44 modal | 670 | +| 33 | 93 | 7 (7/0/0) | 2/7 | cross-address | PURE | 0 | · | 0x8017c6a0 modal | 651 | +| 34 | 108 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | Y | 0x8017eeec modal | 648 | +| 35 | 320 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80181b00 modal | 640 | +| 36 | 317 | 2 (2/0/0) | 2/1 | cross-address | PURE | 0 | · | 0x8017fdf8 modal | 634 | +| 37 | 211 | 3 (3/0/0) | 3/3 | cross-address | PURE | 0 | · | 0x8018e8a0 modal | 633 | +| 38 | 207 | 3 (3/0/0) | 3/3 | cross-address | PURE | 2 | · | 0x801880ec matched | 621 | +| 39 | 124 | 5 (5/0/0) | 4/5 | cross-address | PURE | 0 | · | 0x8017defc modal | 620 | +| 40 | 154 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | Y | 0x801844c8 modal | 616 | +| 41 | 203 | 3 (0/3/0) | 2/3 | cross-address | IMM | 7 | · | 0x8017e804 matched | 609 | +| 42 | 304 | 2 (2/0/0) | 1/2 | per-location | PURE | 0 | · | 0x80185ef8 modal | 608 | +| 43 | 101 | 6 (3/3/0) | 6/3 | cross-address | IMM | 0 | · | 0x8018de60 modal | 606 | +| 44 | 201 | 3 (3/0/0) | 2/3 | cross-address | PURE | 0 | Y | 0x80181be4 draft-ov077 | 603 | +| 45 | 99 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | · | 0x80187960 modal | 594 | +| 46 | 148 | 4 (4/0/0) | 4/4 | cross-address | PURE | 0 | · | 0x80181a30 modal | 592 | +| 47 | 98 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | · | 0x80183f50 modal | 588 | +| 48 | 117 | 5 (5/0/0) | 5/5 | cross-address | PURE | 0 | · | 0x8018171c modal | 585 | +| 49 | 144 | 4 (4/0/0) | 3/4 | cross-address | PURE | 1 | · | 0x8018b238 matched | 576 | +| 50 | 96 | 6 (6/0/0) | 6/6 | cross-address | PURE | 0 | · | 0x80185dd8 modal | 576 | diff --git a/tools/audit_frontier.py b/tools/audit_frontier.py index 7e5b60c56..7ddf688ff 100644 --- a/tools/audit_frontier.py +++ b/tools/audit_frontier.py @@ -55,6 +55,7 @@ def reference(): """THE reference view: open stubs per binary, from the coverage-asserted corpus oracle.""" per, missing = {}, [] for p in sorted(glob.glob(os.path.join(REPO, ".run/sig.ov_*.jsonl"))) + \ + sorted(glob.glob(os.path.join(REPO, ".run/sig.md_*.jsonl"))) + \ [os.path.join(REPO, ".run/sig.resident.jsonl"), os.path.join(REPO, ".run/sig.SLUS_007.26.jsonl")]: if not os.path.exists(p): diff --git a/tools/backlog.py b/tools/backlog.py index a4348fde0..5610a7685 100644 --- a/tools/backlog.py +++ b/tools/backlog.py @@ -134,7 +134,7 @@ def _open_stubs(binary): return _STUB_CACHE[binary] -_BINARY_IN_PATH_RE = re.compile(r"\b(ov_[A-Z0-9]+_\d+|resident|main)\b") +_BINARY_IN_PATH_RE = re.compile(r"\b(ov_[A-Z0-9]+_\d+|md_[A-Z0-9]+_\d+|resident|main)\b") def _binary_of(rec): diff --git a/tools/corpus.py b/tools/corpus.py index c967b95cb..50776bf9b 100644 --- a/tools/corpus.py +++ b/tools/corpus.py @@ -333,7 +333,7 @@ def sig_is_independent(binary): Ghidra sig there would resurrect the 'measuring Ghidra's limits' artefact). MAIN stays excluded — sig_image cannot yet sign the EXE (0x800 header offset, interleaved data islands, one text range); that second oracle is scoped-and-deferred in docs/second-oracle.md.""" - return binary.startswith("ov_") or binary == "resident" + return binary.startswith(("ov_", "md_")) or binary == "resident" def audit(binary): diff --git a/tools/dedup_propagate.py b/tools/dedup_propagate.py index 9b9156898..8e5935fbd 100644 --- a/tools/dedup_propagate.py +++ b/tools/dedup_propagate.py @@ -41,9 +41,17 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent # ---------------------------------------------------------------- fleet / sig helpers def onboarded_overlays(): - mk = (ROOT / "config/overlays.mk").read_text() - m = re.search(r"^OVERLAY_BINARIES\s*:=\s*(.*)$", mk, re.M) - return m.group(1).split() if m else [] + """Overlay aliases + (S44) module aliases — every non-main/resident binary a shared body can + propagate into. Modules carry engine functions too; excluding them re-creates the SC07 + invisible-work bug one class over.""" + out = [] + for f, var in (("config/overlays.mk", "OVERLAY_BINARIES"), ("config/modules.mk", "MODULE_BINARIES")): + fp = ROOT / f + if fp.exists(): + m = re.search(rf"^{var}\s*:=\s*(.*)$", fp.read_text(), re.M) + if m: + out += m.group(1).split() + return out def sig_path(ov): diff --git a/tools/family_hseq.py b/tools/family_hseq.py index 484241070..eae6664da 100644 --- a/tools/family_hseq.py +++ b/tools/family_hseq.py @@ -39,11 +39,19 @@ def load(): # regex only matched `func_` symbols — so the 100 CURATED-name stubs (listCdBuffer) were # invisible, and family_hseq therefore labelled 3 still-stubbed functions as MATCHED exemplars. # A phantom exemplar is re-nominated by every sweep, produces nothing, and books a silent skip. - stubs = {ov: set(corpus.stubs(ov)) for ov in - (d.split("/")[-1] for d in sorted(glob.glob("src/ov_*")))} + # P30 S44: widened from src/ov_* + sig.ov_* to EVERY non-main binary — the resident and the + # md_* modules carry shareable engine functions too, and their absence here is why the R36 + # gate's family-map check (CHECK 4) could never see them. main stays excluded (structurally + # barren — zero h_exact overlap, checked twice, S39). + stubs = {b: set(corpus.stubs(b)) for b in + (d.split("/")[-1] for d in sorted(glob.glob("src/ov_*")) + sorted(glob.glob("src/md_*")))} + if os.path.isdir("src/resident"): + stubs["resident"] = set(corpus.stubs("resident")) inst = [] - for p in sorted(glob.glob(".run/sig.ov_*.jsonl")): - ov = f"ov_{p.split('sig.ov_')[1][:-6]}" + for p in sorted(glob.glob(".run/sig.ov_*.jsonl")) + sorted(glob.glob(".run/sig.md_*.jsonl")) \ + + sorted(glob.glob(".run/sig.resident.jsonl")): + base = os.path.basename(p)[len("sig."):-len(".jsonl")] + ov = base st = stubs.get(ov) if st is None: continue @@ -173,7 +181,8 @@ def main(): # R32/R36: report the scope we ACTUALLY scanned, never a hardcoded count. The literal "134" sat # here while the tool (correctly, via the src/ov_* + sig.ov_* globs) scanned 138 — a generated # doc that misreports its own scope reads exactly like a tool that missed 4 binaries. - n_sigs = len(glob.glob(".run/sig.ov_*.jsonl")) # the SAME glob load() scans — cannot drift from it + n_sigs = len(glob.glob(".run/sig.ov_*.jsonl")) + len(glob.glob(".run/sig.md_*.jsonl")) \ + + len(glob.glob(".run/sig.resident.jsonl")) # the SAME globs load() scans — cannot drift # P30 T0c: stamp scope + tree state. The "family_hseq 29,961 vs progress 28,296" carried defect # was a CROSS-DATE, CROSS-SCOPE misread of two digests (the 07-29 map @ SESSION-25-open vs the # 07-30 fleet digest; the delta was exactly the 2,713 banked between). The tools share one diff --git a/tools/prefetch_fleet.py b/tools/prefetch_fleet.py index cdc1eaf89..78a28faee 100644 --- a/tools/prefetch_fleet.py +++ b/tools/prefetch_fleet.py @@ -30,7 +30,7 @@ overlay is logged and the batch CONTINUES; final exit is non-zero if anything ha tools/prefetch_fleet.py [--limit-programs N] [--dry-run] """ -import argparse, glob, json, os, re, subprocess, sys, time +import glob, argparse, glob, json, os, re, subprocess, sys, time REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "tools")) @@ -64,7 +64,9 @@ def gather_targets(): if fname(addr) not in have: prog.setdefault(program, set()).add(addr) - for b in ("main", "resident"): + _mds = tuple(os.path.basename(x)[len("splat."):-len(".yaml")] + for x in sorted(glob.glob(os.path.join(REPO, "config/splat.md_*.yaml")))) # S44 + for b in ("main", "resident") + _mds: for a in corpus.stubs(b): add(PROGRAM_NAME[b], a) fams = json.load(open(os.path.join(REPO, ".run", "family_hseq.json")))["families"] diff --git a/tools/progress.py b/tools/progress.py index bbfdcb458..3b4110281 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -644,7 +644,8 @@ def weighted_metrics(): import os, glob, json sys.path.insert(0, str(ROOT / "tools")) import corpus - paths = sorted(glob.glob(str(ROOT / ".run/sig.ov_*.jsonl"))) + paths = sorted(glob.glob(str(ROOT / ".run/sig.ov_*.jsonl")))\ + + sorted(glob.glob(str(ROOT / ".run/sig.md_*.jsonl"))) # S44: module class rp = ROOT / ".run/sig.resident.jsonl" if rp.exists(): paths.append(str(rp))