From 234788dfc4274572f1821fcd57ec201de5678f2a Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:00:26 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-26):=20=C2=A78b=20overlay-src=20pars?= =?UTF-8?q?er=20(404/404)=20+=20jr=20isolation=20tool=20+=20the=20gcc-scop?= =?UTF-8?q?ing=20wall=20finding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tools/overlay_src_split.py: overlay-.c-aware partition (header = includes + Phase-17 canonical-sig layer; per-address items = preamble + body; robust def/decl/K&R/DEFINE_func/ SETTER/RETCONST classification). Fleet-validated 404/404 overlay .c, 341,902 items — round-trip exact / 0 unresolved / 0 non-monotonic. The Stage-2 isolation unblock. - tools/jr_isolate_all.py: multi-cut jr resegment (config split at jr boundaries, source repartition + INCLUDE_ASM path repoint, banked-jr carve repoint, -O0 skip, ambient decl carry). SINGLE-cut isolation byte-identical (func_8013FFD8 -> d19c9580, R22). - FINDING (decision-log 2026-07-13): full 54-jr isolation of the dense _after object hits gcc-2.7.2 block-scope-extern TU-persistence (func_801734BC/D_80126B3E declared only in engine_core.h DEFINE_func macros); mechanical TU-split breaks it. Fix = declaration- completion from a global symbol->type map (Drew-approved next step; lazy per-core). - baseline intact (ov_SC01_077 rebuilds d19c9580); no config/src/binary change committed. CURRENT_PHASE session-5 checkpoint + decision-log R31. db.*.gbf = R23 noise, not staged. --- docs/decision-log.md | 47 +++++ phase-ends/CURRENT_PHASE.md | 19 +- tools/jr_isolate_all.py | 337 ++++++++++++++++++++++++++++++ tools/overlay_src_split.py | 400 ++++++++++++++++++++++++++++++++++++ 4 files changed, 802 insertions(+), 1 deletion(-) create mode 100644 tools/jr_isolate_all.py create mode 100644 tools/overlay_src_split.py diff --git a/docs/decision-log.md b/docs/decision-log.md index f3812a1375..25c18599f7 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -420,3 +420,50 @@ INCLUDE_ASM rodata) cannot arbitrate a match whose difference lives in that regi light tail because it "feels productive." Wiki lesson: when a newly-built mechanism has an un-built sub-case that the expensive targets will hit, force that sub-case out on the cheap targets first — de-risking and building-the-missing-piece are the same move. + +### 2026-07-13 — the jr-core ISOLATION wall: mechanical TU-splitting breaks gcc-2.7.2's lenient scoping + +- **Context + belief:** Stage 2 of the multi-jtbl campaign (heavy jr cores → template ×134) needs each + matched jr-function ALONE in its own code subseg so its jtbl carves without a same-subseg collision. + Drew's steer: build the **scalable "isolate-ALL-jr-per-sibling" upfront resegment** (one-shot multi-cut + per overlay) so every Stage-2 core bank is a trivial fill during the closing Fable5 window. Belief going + in: this is mechanical source-splitting — partition the overlay `.c` at jr boundaries, repoint config + + carves, rebuild byte-identical. +- **What was built + PROVEN:** `tools/overlay_src_split.py` — an overlay-`.c`-aware partition (header = + includes + Phase-17 canonical-sig layer; each addressed item = its preamble + body; robust + definition/declaration/K&R/`DEFINE_func`/`SETTER`/`RETCONST` classification). **Fleet-validated 404/404 + overlay `.c`, 341,902 items, round-trip exact / 0 unresolved / 0 non-monotonic.** `tools/jr_isolate_all.py` + — multi-cut resegment (config split at jr boundaries, source repartition + INCLUDE_ASM path repoint, + banked-jr carve repoint, -O0-object skip). **SINGLE-cut isolation byte-identical** (isolate func_8013FFD8 + in the simple `main` object → clean `make build` = `d19c9580`, R22). +- **What FAILED (byte-verified):** the FULL 54-jr isolation on ov_SC01_077 hits a **long tail of C-scoping + edge cases**, culminating in the decisive one: **`func_801734BC` uses `D_80126B3E` with no local decl; + `D_80126B3E` is declared `extern s16` ONLY inside `DEFINE_func` macros in `engine_core.h`.** The original + `_after.c` compiles because **gcc-2.7.2 lets a block-scope `extern` (from an earlier `DEFINE_func` macro + expansion) persist to file scope for the rest of the TU** — splitting `_after` separates the core from the + earlier macro that declares the symbol → `undeclared`. Earlier tail members (all fixed incrementally, in + order): block-scope externs must not be hoisted (per-fn type shadows — `D_80115118` is `unsigned short` + in most funcs but the struct `S115118` in one); file-scope decl ORDERING across a cut (`D_80115110` used + above its in-region decl); **ambient decl context** (a region needs the file-scope decls that lived in + earlier regions of the object — solved: prepend, original order, shadow-safe because a file-scope-declared + symbol can't carry a *different*-typed block shadow or the original wouldn't compile); file-local-typed + externs (`extern Vec8 D_…;`) can't hoist above their typedef. +- **The why (root):** these overlay TUs are hand-matched against a compiler that treats a block-scope + `extern` as declaring the symbol for the WHOLE TU. Mechanical splitting into per-jr TUs breaks that + invisible cross-function dependency, and the dependency is carried through **`DEFINE_func`/`SETTER` macro + expansions in `engine_core.h`**, not just visible col-0 decls — so no amount of *col-0* ambient-carry + fixes it. +- **The candidate fix (not yet built):** **declaration-completion** — build a global symbol→type map from + `engine_core.h`'s macro `extern`s + all overlay col-0 decls, and for each region emit a file-scope + `extern ;` for every `D_`/`func_` symbol the region USES, EXCLUDING type-inconsistent symbols + (the `D_80115118` shadow set, kept block-scope in bodies). This makes every region self-contained + regardless of where the original declared the symbol. Est. ~40–60 LOC on top of the proven parser; the + whole-binary byte-gate arbitrates. **Owner decision pending (Drew): invest in declaration-completion vs + a different Stage-2 approach** — surfaced this session before sinking more time (P5a: repeated failures, + distinct root cause each). +- **Hindsight / for the wiki:** "mechanical source split" of matching-decomp overlay code is NOT mechanical + — the C is written against a specific compiler's lenient scoping (block-scope-extern TU persistence, + macro-injected decls, per-function type shadows). Splitting a TU means REBUILDING each fragment's full + declaration environment from a global symbol map, not relocating text. The parser (structure) was the + easy 20%; the declaration environment (semantics) is the 80%. Prove the mechanism on the SIMPLE object + first (it passed) but budget for the dense object's scoping tail before committing to upfront-×134. diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index f597953693..9e1066d376 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -24,7 +24,23 @@ The Phase-25 h_seq reframe: the "unique tail" is really per-location families - [ ] **Task 11 — Step-D residue map** `[xHigh]` — true singletons (~0.27M ins) + 5 behemoths → Phase-27 input doc. NO execution. - [ ] **Task 12 — PhaseEnd** `[Max — Tier 1; R27 prompt]` — P7 walk, milestone demo, gate 2, `PhaseEnd_Phase26.md`, worklog → `logs/Phase26.md` (R19), in-file recap (R25), decision-log current (R31). -## ▶ SESSION-4 CHECKPOINT (2026-07-12 later) — RESUME FROM HERE (fresh session) +## ▶ SESSION-5 CHECKPOINT (2026-07-13) — RESUME FROM HERE (fresh session) + +**Stage-2 isolation build — parser DONE + single-cut PROVEN; full isolate-all hit the gcc-2.7.2 scoping wall. Effort: Max. Drew's decision (2026-07-13, Fable5 window extended to 7/19 so no time pressure): BUILD declaration-completion + apply isolation LAZILY per-core (not upfront-×134 — avoids the ~7,200-file cost; same tool, isolate only cores that crack). Tools committed this session.** + +Built this session (baseline reverted + intact — ov_SC01_077 rebuilds `d19c9580`; only the 2 new tools are untracked, NOT yet committed): +1. **`tools/overlay_src_split.py`** — overlay-`.c`-aware partition (header = includes + Phase-17 canonical-sig layer; per-address items = preamble + body; robust def/decl/K&R/`DEFINE_func`/`SETTER`/`RETCONST` classification). **VALIDATED fleet-wide: 404/404 overlay `.c`, 341,902 items — round-trip exact / 0 unresolved / 0 non-monotonic.** This is solid, reusable, commit-worthy. +2. **`tools/jr_isolate_all.py`** — multi-cut resegment (config split at every jr boundary, source repartition + INCLUDE_ASM path repoint, banked-jr carve repoint via config + `overlays.mk --order`, -O0-object skip, ambient file-scope decl carry). **SINGLE-cut isolation BYTE-IDENTICAL** (func_8013FFD8 in `main` → clean `make build` = `d19c9580`, R22). Full 54-jr isolation on the dense `_after` object hit the wall. + +**THE WALL (byte-verified, decision-log 2026-07-13):** mechanical TU-splitting breaks gcc-2.7.2's **block-scope-extern TU-persistence** — `func_801734BC` uses `D_80126B3E`, declared `extern s16` ONLY inside `DEFINE_func` macros in `engine_core.h`; the original `_after.c` compiles because an earlier macro's block-scope `extern` persists to file scope for the whole TU. Splitting separates the core from that macro → `undeclared`. (7 distinct C-scoping edge cases surfaced + fixed incrementally; this last one is the deep one — col-0 ambient-carry can't reach macro-injected decls.) + +**THE FIX TO BUILD (Drew-approved, fresh session):** **declaration-completion** — build a global symbol→type map from `engine_core.h` macro `extern`s (the `DEFINE_func`/`SETTER`/`RETCONST` bodies) + all overlay col-0 decls; then for each isolated region emit a file-scope `extern ;` for every `D_`/`func_` symbol the region USES, EXCLUDING the type-shadowed set (symbols with inconsistent decls across the codebase, e.g. `D_80115118` = `unsigned short` vs struct `S115118` — kept block-scope in bodies). This makes every region self-contained regardless of where the original declared the symbol (col-0, block-scope body, or macro). Est. ~40–60 LOC on the proven `overlay_src_split.py`; the whole-binary byte-gate (G3/P9/R22) arbitrates. Wire it into `jr_isolate_all._render_region` REPLACING the col-0-only ambient carry (which can't reach macro-injected decls). + +**NEXT (fresh session):** (1) build declaration-completion (above); (2) re-prove full isolate-all byte-identical on ov_SC01_077; (3) switch isolation to LAZY per-core — integrate a single-core isolate into the bank flow (`jtbl_family_bank`), isolating a heavy jr core only when banking it (fewer files than upfront-×134); (4) then the heavy-jr-core crack waves — Fable5 (window now to 7/19) → §8 carve + template ×134 → R22. Committed tools this session: `tools/overlay_src_split.py`, `tools/jr_isolate_all.py`. Ghidra MCP NOT needed (matching uses cached asm). **R27 boundary: prompt Drew to re-confirm effort/Fable5 before the crack waves.** + +--- + +## ▶ SESSION-4 CHECKPOINT (2026-07-12 later) — superseded by SESSION-5 above **Stage 1 in progress — multi-jtbl `--order` BUILT + proven cross-subseg at fleet scale.** Effort: Max (this session, Tier-1 tooling build). @@ -118,6 +134,7 @@ On approval → `/model opus` + `/effort xHigh` (Tasks 0–4; ALL Fable5 via `Ag ## Log +- **2026-07-13 (session 5, Stage-2 isolation build — parser DONE, single-cut PROVEN, full isolate-all hit the gcc-2.7.2 scoping wall; Max):** Drew re-approved Stage 2 + set `/effort max`; steer = scalable isolate-ALL-jr-per-sibling upfront resegment, then Fable5 on the heavy cores. Built + fleet-validated **`tools/overlay_src_split.py`** (overlay-`.c`-aware partition; **404/404 overlay `.c`, 341,902 items, round-trip exact / 0 unresolved / 0 non-monotonic** — fixed 5 parser edge cases: definition-vs-declaration by brace/`;` scan, K&R defs, comment/`#`-directive peeling, `def_name` from the signature not preamble, `SETTER`/`RETCONST` macro anchors, trailing-footer preservation). Built **`tools/jr_isolate_all.py`** (multi-cut resegment: config split at jr boundaries, source repartition + INCLUDE_ASM path repoint, banked-jr carve repoint, -O0 skip, ambient file-scope decl carry). **SINGLE-cut isolation BYTE-IDENTICAL** (func_8013FFD8 in `main`, clean `make build` `d19c9580`, R22) — Task 2 proven. **Full 54-jr isolation FAILED on the dense `_after` object** after fixing 7 C-scoping edge cases in sequence, on the deep one (byte-verified): **gcc-2.7.2 block-scope-extern TU-persistence** — `func_801734BC` uses `D_80126B3E` declared `extern s16` only inside `engine_core.h` `DEFINE_func` macros; splitting the TU separates the core from the earlier macro expansion that declares it → `undeclared`. Root: mechanical TU-splitting rebuilds the decl ENVIRONMENT (semantics), not just text — col-0 ambient-carry can't reach macro-injected decls. **Candidate fix (not built): declaration-completion** from a global `engine_core.h`+overlay symbol→type map (decision-log 2026-07-13, R31). **Baseline reverted + intact; 2 tools untracked, NOT committed** — surfaced the fork to Drew (P5a) before sinking more time. NEXT: Drew's steer (build declaration-completion vs pivot Stage-2), then commit the tools. - **2026-07-12 (session 4, multi-jtbl `--order` + family-1 — the Stage-1 de-risk build; Max):** Recon proved the Stage-1 core risk is precise: `jtbl_family_bank.revert()` restores each sibling to committed = has func_8012ACE0's carve, so banking ANY 2nd jr family makes the overlay multi-jtbl. Built `ld_interleave.py --order` (address-ordered N-piece sandwich; legacy `--front/--tail` byte-untouched → main + 133 single-carve siblings unaffected) + rewrote `jtbl_carve.py` additive/regenerate-from-config (parses tail data-region + existing carves, splits the containing data piece for the new jtbl, re-emits `--order`; same-subseg → fails loud). Fixed `jtbl_family_bank.bank()` to `make extract` BEFORE the carve (asm must match the reverted config; the old error-string retry was fragile — 3 early siblings carve-failed). **Family-1 `func_801734BC` (34-ins PURE jr, ov_SC01_077_after) matched in ov077 (shared-tail switch idiom `case N: t=-N; break; default: goto after;`) + banked 133/133 siblings — CROSS-subseg multi-jtbl (func_8012ACE0 `_a` + func_801734BC `_after`), the primary Stage-1 goal. R22 clean-fleet 136/136 byte-identical (~52s).** Wrote `tools/jr_isolate.py` (same-subseg non-contiguous case — split a fn into its own subseg via `split_src_region`, `jtbl_carve` re-derives the carve subseg; ascending-order discipline avoids carve-renames) — not yet integrated/committed. §8a finding: the still-INCLUDE_ASM carve build fails (migrated rodata not emitted through INCLUDE_ASM) — OFF the banking critical path (bank swaps C before building). Committed (Drew pushes — R6). NEXT: prove isolation on a higher fn → integrate → pivot to Stage 2. - **2026-07-12 (session 3, R22 parallelization — Drew asked "why so slow, profile+optimize"):** Profiled the clean-fleet R22 (`make clean` + extract-all + check-all). **Phase split: extract-all ~6m11s (the BIGGER phase, 136 serial `splat split` runs) + check-all ~2m58s (136 serial builds; most overlays ~1.3s, ov077 slow — maspsx on its 15K-line `.c`) = ~9 min.** Both phases were serial on a 32-core box (the old Makefile comment called cross-binary `-j` "racy"). **Verified the only shared write is the 4 generated `include/*.inc` macros at extract time (identical content); everything else is per-binary-disjoint and `include/` is read-only during a build → parallel is safe.** Added `JOBS ?= 16` + **`make extract-all`** (seed `main` serial for the macros, then parallel-extract the rest) + **parallel `make check-all`** (`xargs -P$(JOBS)`), correctness-gated. **Result: `make clean && make extract-all && make check-all` = ~50s, 136/136 byte-identical — a ~10.5× speedup (9m→~50s).** Committed the Makefile change (own commit). This compounds across the endgame (R22 runs every commit). - **2026-07-12 (session 3, §8 ×134 AUTOMATION — func_8012ACE0 banked fleet-wide):** Built the jr-function ×134 harvest pipeline and proved it end-to-end. `tools/jtbl_carve.py` = per-overlay §8 carve generator (finding: the same fn is at the same vram across siblings but its jtbl is at a DIFFERENT address in each — ov_SC01_000 jtbl_801A331C / ov_SC02_000 jtbl_801E587C — so the carve is recomputed per sibling; rewrites the config data-tail subsegs + sets `_JTBL_INTERLEAVE`). `tools/jtbl_family_bank.py` = the sibling sweep driver (per sibling: revert→carve→extract→remap_hseq+reconcile→whole-binary gate; idempotent, revert-on-fail). **Propagation-cap fix:** `family_remap.extract_unit` now carries single-line `typedef`s (jr-function bodies define local `typedef struct{} Foo_;` that must template with the body, else the sibling sees `Foo undeclared` — additive, byte-gate-protected). **func_8012ACE0 family: 133/133 siblings BANKED, 0 failures; R22 clean-fleet 136/136 byte-identical.** Metrics: distinct-code 39.1% (50,698 unique fns), instr 63.0%. **Opportunity map (`has_mid_jr` families):** 237 total (5,805 members / 5.68M ins) — 46 small mid/tiny (771 members, mechanical, same pipeline) + 191 substantial = the Fable5 cores (Task 7, paused). Committed. **NEXT: (a) R22 profiling/parallelization (Drew asked — check-all builds 136 binaries serially on a 32-core box); (b) the other 45 small jr families.** diff --git a/tools/jr_isolate_all.py b/tools/jr_isolate_all.py new file mode 100644 index 0000000000..cc54bf6cd5 --- /dev/null +++ b/tools/jr_isolate_all.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""Phase-26 §8b: isolate every jr (switch) function in an overlay into its OWN code +subseg — the one-shot multi-cut resegment that unblocks the Stage-2 heavy-jr-core +harvest (each isolated jr carves its jtbl independently, so banking any core is a +clean fill with no same-subseg collision; cookbook §8/§8b, the whale `_o0b` precedent +generalized). + +For each -O2 code object that contains jr-functions, the object is cut right BEFORE +each jr vram: [gap0][jr1 + trailing non-jr][jr2 + ...] ... . The leading gap keeps +the object's name; each jr-led region becomes `_jr_`. Source is repartitioned +(overlay_src_split, H5) and INCLUDE_ASM stub paths repointed to the new subseg. The 2 +already-banked jr (their real-C is PRESERVED) have their `.rodata` carve repointed to +their new `_jr_` subseg (config piece + overlays.mk --order) — no un-banking, no +metric churn. -O0 objects (`*_o0`, `*_o0b`) are skipped (their new subsegs would lose +the Makefile -O0 flag; the heavy Stage-2 cores are all -O2). + +Byte-neutral by construction: the split only reorganizes .text into more objects placed +contiguously in address order (the existing -O0 multi-object precedent), and the carve +bytes are unchanged (only the owning object's NAME changes). `make build` (SHA1) is the +sole arbiter (G3/P9/R22). + + jr_isolate_all.py [--only func_X,func_Y] [--dry-run] +""" +import argparse +import glob +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import overlay_src_split as oss + +REPO = oss.REPO +O0_SUFFIX = ("_o0", "_o0b") + + +def sh(cmd): + return subprocess.run(cmd, shell=True, cwd=REPO, capture_output=True, text=True) + + +def oss_vram(ov): + txt = open(os.path.join(REPO, f"config/splat.{ov}.yaml")).read() + m = re.search(r"vram:\s*(0x[0-9A-Fa-f]+)", txt) + if not m: + sys.exit(f"jr_isolate_all: no vram in config for {ov}") + return int(m.group(1), 16) + + +def code_objects(cfg_lines): + """[(line_idx, indent, off, name)] for every `- [off, c, name]` code piece, in order.""" + objs = [] + for i, ln in enumerate(cfg_lines): + m = re.match(r'^(\s*)- \[(0x[0-9A-Fa-f]+),\s*c,\s*(\w+)\]', ln) + if m: + objs.append((i, m.group(1), int(m.group(2), 16), m.group(3))) + return objs + + +def rodata_carves(cfg_lines): + """[(line_idx, off, subseg)] for every `.rodata` carve piece.""" + out = [] + for i, ln in enumerate(cfg_lines): + m = re.match(r'^\s*- \[(0x[0-9A-Fa-f]+),\s*\.rodata,\s*(\w+)\]', ln) + if m: + out.append((i, int(m.group(1), 16), m.group(2))) + return out + + +def jr_inventory(ov): + """Return (all_jr:{vram:src_kind}, banked:{vram:func_name}). src_kind in + {'asm','banked'}. jr = INCLUDE_ASM funcs whose .s references a jtbl_ + the + already-banked jr (real-C `def`/`define` items whose name is recorded in a + .run/banked_func_*.json — the exemplar is NOT in its own sibling list, so we + confirm presence by parsing the source, not by the sibling roster).""" + asm_jr = {} + for p in glob.glob(os.path.join(REPO, f"asm/{ov}/nonmatchings/*/*.s")): + if re.search(r'jtbl_[0-9A-Fa-f]{8}', open(p).read()): + fn = os.path.basename(p)[:-2] + if re.fullmatch(r'func_[0-9A-Fa-f]{8}', fn): + asm_jr[int(fn[5:], 16)] = fn + # candidate banked-jr names (global roster) -> confirm each is a real-C def here + cand = set() + for bj in glob.glob(os.path.join(REPO, ".run/banked_func_*.json")): + cand.add(os.path.basename(bj)[len("banked_"):-len(".json")]) + realc = {} # addr -> name for def/define items + syms = oss.load_ov_syms(ov) + for cf in glob.glob(os.path.join(REPO, f"src/{ov}/*.c")): + _, items = oss.parse_overlay_c(open(cf).read(), syms) + for addr, name, kind, _ in items: + if kind in ("def", "define") and name and addr is not None: + realc[addr] = name + banked = {a: nm for a, nm in realc.items() if nm in cand} + alljr = dict(asm_jr) + alljr.update({a: "banked" for a in banked}) # marker; name in `banked` + return alljr, banked + + +def plan(ov, only=None): + """Compute the resegment plan without touching disk. Returns a dict.""" + base = oss_vram(ov) + cfg_lines = open(os.path.join(REPO, f"config/splat.{ov}.yaml")).read().splitlines() + objs = code_objects(cfg_lines) + obj_ranges = [] # (start_vram, end_vram, name, line_idx, indent) + for k, (li, ind, off, nm) in enumerate(objs): + end = objs[k + 1][2] if k + 1 < len(objs) else None + obj_ranges.append((base + off, (base + end) if end is not None else None, nm, li, ind)) + + def obj_of(vram): + for s, e, nm, li, ind in obj_ranges: + if s <= vram and (e is None or vram < e): + return nm + return None + + alljr, banked = jr_inventory(ov) + if only: + only_addrs = {int(x[5:], 16) for x in only if re.fullmatch(r'func_[0-9A-Fa-f]{8}', x)} + alljr = {a: v for a, v in alljr.items() if a in only_addrs} + banked = {a: nm for a, nm in banked.items() if a in only_addrs} + + # group jr by their -O2 object (skip -O0 objects + objects with no jr) + skipped_o0 = [] + per_obj = {} + for a in sorted(alljr): + nm = obj_of(a) + if nm is None: + continue + if nm.endswith(O0_SUFFIX): + skipped_o0.append(a) + continue + per_obj.setdefault(nm, []).append(a) + + # banked jr -> its object (for carve repoint) + banked_obj = {banked[a]: obj_of(a) for a in banked} + return { + "base": base, "cfg_lines": cfg_lines, "obj_ranges": obj_ranges, + "per_obj": per_obj, "banked": banked, "banked_obj": banked_obj, + "skipped_o0": skipped_o0, + } + + +def subseg_name(ov, vram): + return f"{ov}_jr_{vram:08X}" # uppercase hex, matching the func_XXXXXXXX convention + + +def build_new_config(ov, p): + """Return (new_cfg_lines, new_files:{path:content}, carve_renames:{old_sub:new_sub}).""" + base = p["base"] + cfg_lines = list(p["cfg_lines"]) + syms = oss.load_ov_syms(ov) + + # 1) source repartition + the config code-region replacement (per object, bottom-up so + # line indices stay valid). + new_files = {} + replacements = [] # (line_idx, [new config lines]) + carve_renames = {} # old code-subseg name -> banked jr's new _jr_ subseg + banked_by_obj = {} + for fn, obj in p["banked_obj"].items(): + banked_by_obj.setdefault(obj, []).append(fn) + + for s, e, nm, li, ind in p["obj_ranges"]: + if nm not in p["per_obj"]: + continue + cuts = p["per_obj"][nm] # jr vrams in this object + srcpath = os.path.join(REPO, f"src/{ov}/{nm}.c") + header, regions = _partition(srcpath, cuts, syms) + # region 0 (lo=None) keeps the object name; each jr-led region -> _jr_. Regions are + # processed in address order, accumulating this object's file-scope decls as `ambient` so + # each region carries the decl context it had in the original single object. + cfg_block = [] + ambient = [] + for (lo, hi, items) in regions: + sub = nm if lo is None else subseg_name(ov, lo) + off = (s if lo is None else lo) - base + cfg_block.append(f"{ind}- [{hex(off)}, c, {sub}]") + body = _render_region(header, items, old_sub=nm, new_sub=sub, ambient=ambient) + new_files[os.path.join(REPO, f"src/{ov}/{sub}.c")] = body + ambient = ambient + _file_scope_decls(items) # context for later regions + # a banked jr leading this region -> its carve must repoint to `sub` + if lo is not None: + for fn in banked_by_obj.get(nm, []): + if int(fn[5:], 16) == lo: + carve_renames[nm] = sub + replacements.append((li, cfg_block)) + + # apply config code-region replacements bottom-up + for li, block in sorted(replacements, reverse=True): + cfg_lines[li:li + 1] = block + + # 2) repoint the .rodata carve pieces to the banked jr's new subseg + for i, ln in enumerate(cfg_lines): + m = re.match(r'^(\s*- \[0x[0-9A-Fa-f]+,\s*\.rodata,\s*)(\w+)(\].*)$', ln) + if m and m.group(2) in carve_renames: + cfg_lines[i] = m.group(1) + carve_renames[m.group(2)] + m.group(3) + + return cfg_lines, new_files, carve_renames + + +def _partition(srcpath, cuts, syms): + """overlay_src_split.partition but taking a preloaded syms dict.""" + header, items = oss.parse_overlay_c(open(srcpath).read(), syms) + if any(it[0] is None and it[2] == "tail" for it in items): + sys.exit(f"jr_isolate_all: unaddressable content in {srcpath}") + footer = [it for it in items if it[2] == "footer"] + addressed = [it for it in items if it[0] is not None] + cuts = sorted(set(cuts)) + bounds = [None] + cuts + [None] + regions = [] + for lo, hi in zip(bounds[:-1], bounds[1:]): + sel = sorted((it for it in addressed + if (lo is None or it[0] >= lo) and (hi is None or it[0] < hi)), + key=lambda it: it[0]) + regions.append((lo, hi, sel)) + if footer: + lo, hi, sel = regions[-1] + regions[-1] = (lo, hi, sel + footer) + return header, regions + + +# a hoistable declaration line: an `extern` decl, or a func/data prototype ending in `;` +# with no `{` body. These are legal to REPEAT in C (unlike typedef/struct/enum), so we hoist a +# deduped copy to the region top — every symbol is then declared before any body uses it (a cut can +# strand a use above its in-region decl; the source redundantly re-declares externs per fn-group). +_HOIST_RE = re.compile( + r'^\s*(?:extern\b.*;' + r'|[A-Za-z_][\w\*\s]*\b(?:func_[0-9A-Fa-f]{8}|D_[0-9A-Fa-f]{8})\b[^{]*;)\s*(?:/\*.*\*/)?\s*$') + + +_SYM_RE = re.compile(r'\b((?:D_|func_)[0-9A-Fa-f]{8})\b') +# a decl whose base type is a BUILTIN / include-provided type is safe to hoist to the region +# top; one that names a FILE-LOCAL type (`Vec8`, `struct BigCopy`, ...) must stay where the type +# is defined (hoisting it above the typedef is a parse error), so it is NOT carried as ambient. +_SAFE_TYPE = re.compile( + r'^\s*(?:extern\s+)?(?:const\s+)?(?:(?:un)?signed\s+)?' + r'(?:void|char|short|int|long|float|double' + r'|[su](?:8|16|32|64)|M2C_UNK|MNC_UNK)\b') + + +def _file_scope_decls(items): + """[(line, [syms])] for each COL-0 (file-scope) extern/proto decl with a builtin base type, + in item order (file-local-typed decls are left in place — see _SAFE_TYPE).""" + out = [] + for _, _, _, text in items: + for line in text.split("\n"): + if not line or line[0].isspace(): # col-0 only (block-scope stays put) + continue + if "{" in line or "}" in line: + continue + if _HOIST_RE.match(line) and _SAFE_TYPE.match(line): + out.append((line.rstrip(), _SYM_RE.findall(line))) + return out + + +def _render_region(header, items, old_sub, new_sub, ambient): + """Region .c = header + AMBIENT file-scope decls (from earlier regions of this object, in + original order, deduped by symbol) + the region's items unchanged. Ambient preserves the + exact decl visibility each function had in the original single object (a cut otherwise + strands a use above the decl that lived in an earlier region) — byte-neutral (decls emit + nothing; order preserved). Prepending is always safe: a symbol that has a file-scope decl + cannot also carry a *different*-typed block-scope shadow (the original would not compile), + so a same-typed block-scope redeclaration below is compatible.""" + if new_sub != old_sub: + items = [(a, n, k, oss.rewrite_asm_subseg(t, old_sub, new_sub)) for a, n, k, t in items] + lines, seen = [], set() + for line, syms in ambient: + key = tuple(sorted(syms)) if syms else (line,) + if key in seen: + continue + seen.add(key) + lines.append(line) + parts = [header] + if lines: + parts.append("/* Phase-26 §8b jr_isolate_all.py: ambient file-scope decls carried from " + "earlier code regions of this object (original order, shadow-excluded => " + "byte-neutral). */\n" + "\n".join(lines)) + parts.extend(t for _, _, _, t in items) + return "\n".join(parts) + "\n" + + +def repoint_overlays_mk(carve_renames, dry): + mk = os.path.join(REPO, "config/overlays.mk") + txt = open(mk).read() + changed = [] + for old_sub, new_sub in carve_renames.items(): + pat = rf'(--order[^#\n]*?){re.escape(old_sub)}\.o' + if re.search(pat, txt): + txt = re.sub(pat, lambda m: m.group(1) + new_sub + ".o", txt, count=1) + changed.append(f"{old_sub}.o -> {new_sub}.o") + if not dry: + open(mk, "w").write(txt) + return changed + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("ov") + ap.add_argument("--only", help="comma-separated func_XXXX to isolate (default: all -O2 jr)") + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args() + only = set(a.only.split(",")) if a.only else None + + p = plan(a.ov, only) + n_jr = sum(len(v) for v in p["per_obj"].values()) + print(f"jr_isolate_all {a.ov}: {n_jr} jr in {len(p['per_obj'])} -O2 objects " + f"{ {k: len(v) for k, v in p['per_obj'].items()} }") + if p["skipped_o0"]: + print(f" SKIPPED {len(p['skipped_o0'])} jr in -O0 objects: {[hex(x) for x in p['skipped_o0']]}") + if not p["per_obj"]: + print(" nothing to isolate.") + return + + cfg_lines, new_files, carve_renames = build_new_config(a.ov, p) + mk_changes = repoint_overlays_mk(carve_renames, dry=True) + print(f" -> {len(new_files)} region .c files; carve repoints: {carve_renames or '(none)'}") + for c in mk_changes: + print(f" overlays.mk --order: {c}") + + if a.dry_run: + print(" [dry-run] no files written.") + return + + # write config, source region files, overlays.mk + cfg_path = os.path.join(REPO, f"config/splat.{a.ov}.yaml") + open(cfg_path, "w").write("\n".join(cfg_lines) + "\n") + # remove the original per-object .c files that were replaced (region 0 rewrites them; + # extra regions are new — but a stale original with the OLD single-object content would + # shadow nothing since we overwrite region 0 to the same path). Write all region files: + for path, content in new_files.items(): + open(path, "w").write(content) + repoint_overlays_mk(carve_renames, dry=False) + print(f" wrote config + {len(new_files)} region files + overlays.mk. Run `make extract " + f"BINARY={a.ov} && make build BINARY={a.ov}` to byte-gate (R22).") + + +if __name__ == "__main__": + main() diff --git a/tools/overlay_src_split.py b/tools/overlay_src_split.py new file mode 100644 index 0000000000..27d5342139 --- /dev/null +++ b/tools/overlay_src_split.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Overlay-`.c`-aware source partition (Phase-26 §8b — jr isolation). + +The overlay `src//*.c` files carry a far richer top-level structure than the +library-region `split_src_region.py` was built for, so its "header = leading +#includes; every following item resolves to one address" model chokes (~922 +unresolved). This module parses an overlay `.c` into (header, [Item]) correctly: + + header = the leading `#include`s + the banner-delimited Phase-17 canonical-sig + extern layer, preserved verbatim (a file-scope decl layer every split + piece needs). + Item = a leading PREAMBLE (blank lines, `// @class`/comment annotations, and + *declarations* — `extern` protos, `typedef`/`struct`/`union`/`enum`, + `#define` — anything that is not itself an addressed anchor) attached + to the following ADDRESSED ANCHOR, which is one of: + - a function DEFINITION (` func_XXXX(...) { ... }`) + - an `INCLUDE_ASM("asm/...", func_XXXX);` stub + - a `DEFINE_func_XXXXXXXX()` dedup macro (shared engine-core body) + - a `#ifdef NON_MATCHING ... #endif` block + The anchor's address is its `func_XXXX` / `DEFINE_func_XXXX` hex, else a + `config/symbols..txt` lookup. + +The definition-vs-declaration discriminator is a forward brace/semicolon scan (NOT +a regex on one line): a *definition* has a `{` body not terminated by `;`; a +*declaration* ends at a top-level `;` before any `{`, or is a braced type ending +`};`. This is what distinguishes `void func_X(void) { ... }` (anchor) from +`s32 func_X(...); /* extern */` (preamble — the `extern` keyword is omitted, the +line ends in a trailing comment, so an "ends-with-;" test misfires). + +Primary consumer: `jr_isolate_all.py` (multi-cut resegment), via `partition()`. +The whole-binary SHA1 byte-gate remains the sole correctness arbiter (G3/P9). +""" +import os +import re +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +CANON_OPEN = re.compile(r'/\*\s*=+\s*Phase-\d+ canonical-sig layer') +CANON_CLOSE = re.compile(r'end canonical-sig layer') +INCLUDE_ASM = re.compile(r'^\s*INCLUDE_ASM\("[^"]*",\s*(\w+)\)') +DEFINE_FUNC = re.compile(r'^DEFINE_func_([0-9A-Fa-f]{8})\s*\(') +# a dedup macro whose address is its FIRST arg: SETTER(func_X, ..), RETCONST(func_X, ..) +MACRO_ARG_ANCHOR = re.compile(r'^[A-Za-z_]\w*\(\s*(func_[0-9A-Fa-f]{8})\b') +NONMATCH = re.compile(r'^#if(?:def)?\s+.*NON_MATCHING') + + +def load_syms(path): + s = {} + if not os.path.exists(path): + return s + for ln in open(path): + m = re.match(r"(\w+)\s*=\s*0x([0-9A-Fa-f]+)", ln) + if m: + s[m.group(1)] = int(m.group(2), 16) + return s + + +def addr_of(name, syms): + if not name: + return None + m = re.match(r'func_([0-9A-Fa-f]{8})$', name) + if m: + return int(m.group(1), 16) + return syms.get(name) + + +def item_func_name(text): + """The addressed name for an INCLUDE_ASM / NON_MATCHING item (name is in the + INCLUDE_ASM macro).""" + m = re.search(r'INCLUDE_ASM\("[^"]*",\s*(\w+)\)', text) + return m.group(1) if m else None + + +def def_name(construct_lines): + """The defined function's name = the identifier immediately before the first + params `(` in the (comment/string-stripped) definition signature. Robust to a + leading banner comment and to multi-line prototypes in a shared preamble (those + never reach here — scan_construct hands us the definition construct only).""" + code = [] + in_block = False + for line in construct_lines: + c = line + if in_block: + if "*/" in c: + c = c.split("*/", 1)[1] + in_block = False + else: + continue + c = re.sub(r'/\*.*?\*/', '', c) + if "/*" in c: + c = c.split("/*", 1)[0] + in_block = True + c = re.sub(r'//.*$', '', c) + code.append(c) + if "(" in "".join(code): + break + joined = "".join(code) + m = re.search(r'([A-Za-z_]\w*)\s*\(', joined) + return m.group(1) if m else None + + +def _strip(line, in_block): + """Blank out // and /* */ comments (block state carried) + string/char literals, + for brace/paren/semicolon token counting. Returns (code, in_block).""" + c = line + if in_block: + if "*/" in c: + c = c.split("*/", 1)[1] + in_block = False + else: + return "", True + c = re.sub(r'/\*.*?\*/', '', c) + if "/*" in c: + c = c.split("/*", 1)[0] + in_block = True + c = re.sub(r'//.*$', '', c) + c = re.sub(r'"(?:\\.|[^"\\])*"', '""', c) + c = re.sub(r"'(?:\\.|[^'\\])*'", "''", c) + return c, in_block + + +_DECL_KW = ("extern", "typedef", "struct", "union", "enum") + + +def split_header(lines): + """Return the number of leading lines that form the header (includes + the + banner-delimited canonical-sig layer, if present).""" + n = len(lines) + i = 0 + while i < n and (lines[i].startswith("#include") or lines[i].strip() == ""): + i += 1 + if i < n and CANON_OPEN.search(lines[i]): + while i < n and not CANON_CLOSE.search(lines[i]): + i += 1 + if i < n: + i += 1 # consume the close-marker line + return i + + +def scan_construct(lines, i): + """From top-level line `i`, return (end_exclusive, is_definition). + + Handles, brace/paren/comment/string aware: + - ANSI function def ` name() { ... }` -> definition + - K&R function def ` name(a, b) {` -> definition + - prototype ` name(...);` -> declaration + - extern / data decl `extern ...;` ` x;` -> declaration + - braced type `typedef struct { ... } T;` `struct S {...};` -> decl + A function header is `name(...)` closed at depth 0; a `;` immediately after its + `)` is a prototype, a `;` after intervening K&R param decls is not the end, and a + `{` after the header opens the body. Declarations starting with a decl keyword + (extern/typedef/struct/union/enum) never become definitions (guards fn-ptr + typedefs and struct-returning shapes absent from this codebase).""" + n = len(lines) + first = lines[i].lstrip() + force_decl = any(first == k or first.startswith(k + " ") or first.startswith(k + "\t") + for k in _DECL_KW) + paren = brace = 0 + seen_header = False # closed a depth-0 (...) that is a function header + body_open = False + since_header = [] # code chars since the header ) (empty => prototype at ;) + in_block = False + j = i + while j < n: + code, in_block = _strip(lines[j], in_block) + for ch in code: + if ch == '(': + paren += 1 + elif ch == ')': + if paren > 0: + paren -= 1 + if paren == 0 and brace == 0 and not seen_header and not force_decl: + seen_header = True + since_header = [] + continue + elif ch == '{': + brace += 1 + if brace == 1: + body_open = True + elif ch == '}': + if brace > 0: + brace -= 1 + if brace == 0 and body_open: + if seen_header and not force_decl: + return j + 1, True # function body closed + body_open = False # braced type -> await terminating ; + elif ch == ';' and paren == 0 and brace == 0: + if seen_header and not body_open and not force_decl: + if "".join(since_header).strip() == "": + return j + 1, False # prototype: ) ; + # else: a K&R param decl -> keep scanning for the body { + else: + return j + 1, False # plain / braced declaration end + if seen_header and not body_open: + since_header.append(ch) + j += 1 + return n, (seen_header and not force_decl) or body_open + + +def parse_overlay_c(src, syms): + """Return (header, items) where items = [(addr, name, kind, text)] in file order. + kind in {asm, define, def, nonmatch, tail}. `tail` = trailing content with no + following anchor (addr None) — normally absent in a well-formed file.""" + lines = src.split("\n") + hdr_end = split_header(lines) + header = "\n".join(lines[:hdr_end]) + n = len(lines) + items = [] + i = hdr_end + pre_start = i + while i < n: + raw = lines[i] + s = raw.strip() + # ---- peel non-anchor prefix lines into the accumulating preamble ---- + if s == "": + i += 1 + continue + if s.startswith("//"): + i += 1 + continue + if s.startswith("/*"): # block comment (1+ lines) + while i < n and "*/" not in lines[i]: + i += 1 + i = min(i + 1, n) + continue + if s.startswith("#") and not NONMATCH.match(raw): # #define/#undef/#include... + while i < n and lines[i].rstrip().endswith("\\"): + i += 1 + i += 1 + continue + # ---- now `raw` is a real construct: an anchor or a def/decl ---- + m = INCLUDE_ASM.match(raw) + if m: + items.append((addr_of(m.group(1), syms), m.group(1), "asm", + "\n".join(lines[pre_start:i + 1]))) + i += 1 + pre_start = i + continue + m = DEFINE_FUNC.match(raw) + if m: + items.append((int(m.group(1), 16), "func_" + m.group(1), "define", + "\n".join(lines[pre_start:i + 1]))) + i += 1 + pre_start = i + continue + m = MACRO_ARG_ANCHOR.match(raw) + if m: # SETTER/RETCONST(func_X, ...) dedup macro (1 line) + items.append((int(m.group(1)[5:], 16), m.group(1), "define", + "\n".join(lines[pre_start:i + 1]))) + i += 1 + pre_start = i + continue + if NONMATCH.match(raw): + j = i + while j < n and not lines[j].startswith("#endif"): + j += 1 + j = min(j + 1, n) + name = item_func_name("\n".join(lines[i:j])) + items.append((addr_of(name, syms), name, "nonmatch", + "\n".join(lines[pre_start:j]))) + i = j + pre_start = i + continue + # a definition-or-declaration; classify by forward scan + j, is_def = scan_construct(lines, i) + if is_def: + name = def_name(lines[i:j]) + items.append((addr_of(name, syms), name, "def", + "\n".join(lines[pre_start:j]))) + i = j + pre_start = i + else: + i = j # declaration => stays in the preamble + if pre_start < n: + # preserve the exact trailing chunk (incl. blank lines) so round-trip stays + # exact; a comment/blank-only chunk is a harmless footer, anything else is a + # genuine parser gap (kind=tail -> partition() fails loud). + trailing = "\n".join(lines[pre_start:]) + kind = "footer" if _comment_blank_only(trailing) else "tail" + items.append((None, None, kind, trailing)) + return header, items + + +def _comment_blank_only(text): + """True if `text` is only // / /* */ comments and whitespace.""" + stripped = re.sub(r'/\*.*?\*/', '', text, flags=re.S) + stripped = re.sub(r'//.*', '', stripped) + return stripped.strip() == "" + + +def load_ov_syms(ov): + """Merge every symbol file the overlay's splat config actually links + (config's symbol_addrs_path) — us + resident + per-overlay.""" + cfg = os.path.join(REPO, f"config/splat.{ov}.yaml") + syms = {} + in_list = False + for ln in open(cfg): + if re.match(r'\s*symbol_addrs_path:', ln): + in_list = True + continue + if in_list: + m = re.match(r'\s*-\s*(\S+)', ln) + if m and m.group(1).endswith(".txt"): + syms.update(load_syms(os.path.join(REPO, m.group(1)))) + else: + break + if not syms: # fallback if the key wasn't found + syms = load_syms(os.path.join(REPO, "config/symbols.us.txt")) + return syms + + +REAL_KINDS = ("def", "define", "nonmatch") # items splat will NOT regenerate + + +def rewrite_asm_subseg(text, old_sub, new_sub): + """Repoint an INCLUDE_ASM stub's path component to the new subseg name.""" + return text.replace(f'/nonmatchings/{old_sub}"', f'/nonmatchings/{new_sub}"') + + +def partition(srcpath, cuts, syms_path, verbose=True): + """Split `srcpath` at the sorted vram `cuts` (each = the low bound of a new + region). Returns [(lo, hi, [item...]), ...] region descriptors WITHOUT writing + (the caller assigns subseg names + output paths). lo/hi are vram bounds; the + first region is [start, cuts[0]) and the last is [cuts[-1], +inf).""" + header, items = parse_overlay_c(open(srcpath).read(), load_syms(syms_path)) + tail = [it for it in items if it[0] is None and it[2] == "tail"] + if tail: + sys.exit(f"partition: {srcpath} has unaddressable trailing content:\n" + f"{tail[0][3][:160]}") + footer = [it for it in items if it[2] == "footer"] + addressed = [it for it in items if it[0] is not None] + cuts = sorted(set(cuts)) + bounds = [None] + cuts + [None] # (start,cut0),(cut0,cut1),...,(cutN,end) + regions = [] + for lo, hi in zip(bounds[:-1], bounds[1:]): + sel = sorted((it for it in addressed + if (lo is None or it[0] >= lo) and (hi is None or it[0] < hi)), + key=lambda it: it[0]) + regions.append((lo, hi, sel)) + if footer: # documentation footer -> stays with the last region + lo, hi, sel = regions[-1] + regions[-1] = (lo, hi, sel + footer) + if verbose: + for lo, hi, sel in regions: + rc = sum(1 for it in sel if it[2] in REAL_KINDS) + print(f" region [{'.' if lo is None else hex(lo)}," + f"{'.' if hi is None else hex(hi)}): {len(sel)} items ({rc} real-C)") + return header, regions + + +# --------------------------------------------------------------------------- CLI +def selftest(srcpath, syms): + """Round-trip + anchor-order + kind census — proves the parser on a real file.""" + src = open(srcpath).read() + header, items = parse_overlay_c(src, syms) + rebuilt = header + "\n" + "\n".join(it[3] for it in items) + # exact round-trip (allow a single trailing-newline difference) + ok = rebuilt == src or rebuilt + "\n" == src or rebuilt == src + "\n" + from collections import Counter + kinds = Counter(it[2] for it in items) + addrs = [it[0] for it in items if it[0] is not None] + unres = [it for it in items if it[0] is None and it[2] != "footer"] + mono = sum(1 for a, b in zip(addrs, addrs[1:]) if b < a) + print(f"{os.path.basename(srcpath)}: {len(items)} items {dict(kinds)}") + print(f" round-trip exact: {ok}") + print(f" addressed: {len(addrs)} unresolved(non-footer): {len(unres)} " + f"non-monotonic transitions: {mono}") + if unres: + for it in unres[:5]: + print(f" UNRESOLVED [{it[2]}]: {it[3].strip()[:90]}") + if not ok: + # locate first divergence for debugging + a, b = src.split("\n"), rebuilt.split("\n") + for idx in range(min(len(a), len(b))): + if a[idx] != b[idx]: + print(f" first diff at line {idx+1}:\n orig: {a[idx][:80]!r}\n" + f" got : {b[idx][:80]!r}") + break + print(f" line counts: orig={len(a)} rebuilt={len(b)}") + return ok and not unres and mono == 0 + + +def _ov_from_path(p): + m = re.search(r'src/(ov_\w+?)/', p) + return m.group(1) if m else None + + +def main(): + args = sys.argv[1:] + if len(args) == 2 and args[0] == "selftest": + ov = _ov_from_path(args[1]) + syms = load_ov_syms(ov) if ov else load_syms(os.path.join(REPO, "config/symbols.us.txt")) + ok = selftest(args[1], syms) + sys.exit(0 if ok else 1) + sys.exit(__doc__) + + +if __name__ == "__main__": + main()