mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
c52190ca86
- _type_names returned the TAG for `typedef struct Rec801806C8_s Rec801806C8;`, so the typedef block and the tag's own packed struct definition collided under one key with different bodies and the R43 "CONFLICTING bodies — a rename is needed" refusal fired on legal C. Now keyed by the alias (_TYPEDEF_TAG_ALIAS); the `carried` set learns the alias; `typedef struct X X;` (alias == tag) keeps the old key so a second one still dedupes/refuses. Unit control on 7 block shapes PASS; ov_SC02_017 --only func_80186C64 --dry-run: 2 region files, no carve repoints. cookbook §497; SETUP row.
1092 lines
63 KiB
Python
1092 lines
63 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase-26 §8b: isolate every jr (switch) function in an overlay into its OWN code
|
|
subseg — the one-shot multi-cut resegment that unblocks the Stage-2 heavy-jr-core
|
|
harvest (each isolated jr carves its jtbl independently, so banking any core is a
|
|
clean fill with no same-subseg collision; cookbook §8/§8b, the whale `_o0b` precedent
|
|
generalized).
|
|
|
|
For each -O2 code object that contains jr-functions, the object is cut right BEFORE
|
|
each jr vram: [gap0][jr1 + trailing non-jr][jr2 + ...] ... . The leading gap keeps
|
|
the object's name; each jr-led region becomes `<ov>_jr_<addr>`. Source is repartitioned
|
|
(overlay_src_split, H5) and INCLUDE_ASM stub paths repointed to the new subseg. The 2
|
|
already-banked jr (their real-C is PRESERVED) have their `.rodata` carve repointed to
|
|
their new `_jr_<addr>` subseg (config piece + overlays.mk --order) — no un-banking, no
|
|
metric churn. -O0 objects (`*_o0`, `*_o0b`) are skipped (their new subsegs would lose
|
|
the Makefile -O0 flag; the heavy Stage-2 cores are all -O2).
|
|
|
|
Byte-neutral by construction: the split only reorganizes .text into more objects placed
|
|
contiguously in address order (the existing -O0 multi-object precedent), and the carve
|
|
bytes are unchanged (only the owning object's NAME changes). `make build` (SHA1) is the
|
|
sole arbiter (G3/P9/R22).
|
|
|
|
jr_isolate_all.py <ov> [--only func_X,func_Y] [--dry-run]
|
|
"""
|
|
import collections
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import overlay_src_split as oss
|
|
|
|
|
|
import mk_write as MKW # atomic, collapse-refusing overlays.mk writer (P31 S60)
|
|
|
|
REPO = oss.REPO
|
|
O0_SUFFIX = ("_o0", "_o0b")
|
|
|
|
|
|
_SCAN = {"attempted": 0, "raised": 0}
|
|
|
|
|
|
def _reloc_targets_or_die(family_remap, ov, addr, data=None):
|
|
"""`family_remap.reloc_targets`, with the ENVIRONMENTAL failure separated from the per-function one.
|
|
|
|
Both callers used to wrap this in `try/except Exception: continue`. That is right for a single
|
|
function whose relocations cannot be walked, and catastrophically wrong for a missing
|
|
`.run/sig.<bin>.jsonl` — because then EVERY call raises, the scan finds zero owners, and the R32
|
|
"every committed carve resolves to exactly one owner" assertion downstream fires and reports
|
|
carve CORRUPTION. Measured P31 S74 inside a provisioned worktree: 2,603 of 2,603 functions
|
|
raised, 0 owners found, run aborted with a confident verdict about damage that did not exist.
|
|
|
|
A missing sig index is not a fact about the carves; it is a fact about the checkout (R40 —
|
|
exonerate the instrument). So it aborts here, at the cause, naming the fix — rather than being
|
|
re-interpreted 100 lines later as evidence about the subject (R54).
|
|
|
|
`_SCAN` counts attempted vs raised: a scan in which EVERYTHING raised measured nothing, whatever
|
|
the exception was, and may not be reported as a result (R32 — assert your coverage)."""
|
|
_SCAN["attempted"] += 1
|
|
try:
|
|
return family_remap.reloc_targets(ov, addr, data=data) if data is not None \
|
|
else family_remap.reloc_targets(ov, addr)
|
|
except FileNotFoundError as e:
|
|
sys.exit(f"jr_isolate_all({ov}): cannot walk relocations — {e}. This is a MISSING INDEX, "
|
|
f"not a carve defect: run `make sig-all` (or, in a worktree, provision "
|
|
f".run/sig.*.jsonl — tools/verify_worktree.py does this). Refusing to report "
|
|
f"ownership from a scan that could not read the image index (R43).")
|
|
except Exception:
|
|
_SCAN["raised"] += 1
|
|
return None
|
|
|
|
|
|
def _assert_scan_covered(ov, what):
|
|
"""Fail loud if the ownership scan raised on EVERYTHING it attempted (R32)."""
|
|
a, r = _SCAN["attempted"], _SCAN["raised"]
|
|
if a and r == a:
|
|
sys.exit(f"jr_isolate_all({ov}): {what} attempted {a} function(s) and EVERY ONE raised — "
|
|
f"the scan measured nothing, so its 0 owners are an artifact, not a finding (R32).")
|
|
|
|
|
|
def sh(cmd):
|
|
return subprocess.run(cmd, shell=True, cwd=REPO, capture_output=True, text=True)
|
|
|
|
|
|
def oss_vram(ov):
|
|
txt = open(os.path.join(REPO, f"config/splat.{ov}.yaml")).read()
|
|
m = re.search(r"vram:\s*(0x[0-9A-Fa-f]+)", txt)
|
|
if not m:
|
|
sys.exit(f"jr_isolate_all: no vram in config for {ov}")
|
|
return int(m.group(1), 16)
|
|
|
|
|
|
def code_objects(cfg_lines):
|
|
"""[(line_idx, indent, off, name)] for every `- [off, c, name]` code piece, in order."""
|
|
objs = []
|
|
for i, ln in enumerate(cfg_lines):
|
|
m = re.match(r'^(\s*)- \[(0x[0-9A-Fa-f]+),\s*c,\s*(\w+)\]', ln)
|
|
if m:
|
|
objs.append((i, m.group(1), int(m.group(2), 16), m.group(3)))
|
|
return objs
|
|
|
|
|
|
def rodata_carves(cfg_lines, ov=None):
|
|
"""[(line_idx, off, subseg)] for every `.rodata` CARVE piece.
|
|
|
|
A carve is a jump table jtbl_carve.py lifted out of the data tail so its owning object can
|
|
emit it. The §154-A LEADING ISLAND — `- [0x0, .rodata, <alias>]`, the module's own rodata blob
|
|
of INCLUDE_RODATA data plus every stub's still-migrated table — is NOT one: it has no single
|
|
owner, so jr_inventory's "every carve resolves to exactly one banked function" check (R32)
|
|
aborted on it and md_* could not be isolated at all.
|
|
|
|
The discriminator is structural: a carve is a table LIFTED OUT OF THE DATA TAIL, so it can
|
|
never sit at the segment's own offset 0 — offset 0 is where the module-id header (md_*) or the
|
|
first function (ov_*) lives. Verified across all 213 splat configs: every `.rodata` piece at
|
|
offset 0 is an md_* leading island; ov_*/main configs have none, so passing `ov` is a no-op
|
|
for them. The discriminator used to ALSO require the subseg to be the binary's own alias, but
|
|
that conjunct broke the moment a leading island was legitimately RENAMED to the code subseg
|
|
where its emitters live (S68 md_MAIN_003 → md_MAIN_003_jr_800D12D0: spimdisasm rodata-migration
|
|
is same-subseg-only, cookbook §371), which made jr_inventory read the island as an UNOWNED
|
|
carve and R32-abort every further carve on the binary. Offset 0 alone is the honest key;
|
|
negative-controlled over all 184 configs with `.rodata` pieces (only md_MAIN_003's verdict
|
|
moved, abort → OK)."""
|
|
out = []
|
|
for i, ln in enumerate(cfg_lines):
|
|
m = re.match(r'^\s*- \[(0x[0-9A-Fa-f]+),\s*\.rodata,\s*(\w+)\]', ln)
|
|
if m:
|
|
off, sub = int(m.group(1), 16), m.group(2)
|
|
if ov is not None and off == 0:
|
|
continue # the leading island (possibly renamed), not a carve
|
|
out.append((i, off, sub))
|
|
return out
|
|
|
|
|
|
def jr_inventory(ov):
|
|
"""Return (all_jr:{vram:src_kind}, banked:{vram:func_name}). src_kind in {'asm','banked'}.
|
|
|
|
jr = still-unmatched switch functions (INCLUDE_ASM `.s` referencing a jtbl_) + the
|
|
already-banked jr (whose jtbl became a committed `.rodata` carve).
|
|
|
|
`banked` is DERIVED FROM THE IMAGE — never from a roster (R33). The old code filtered
|
|
real-C defs by an EPHEMERAL, gitignored `.run/banked_func_*.json` set: a `rm -rf .run`
|
|
/ fresh clone made all banked jr invisible at once, and a cross-address sibling (whose
|
|
roster file is named after the exemplar) was structurally missing. Two proven invariants
|
|
answer it instead: (1) the committed splat config lists every `.rodata` carve; (2) a
|
|
real-C function OWNS a carve iff it references that carve's address — `family_remap.
|
|
reloc_targets` reads the extracted image and says so. So a real-C def/define fn is a
|
|
banked jr iff it references a committed carve offset. Cross-address- and
|
|
curated-name-immune, and it finds NON-LEADER banked jr (carve in the object's own
|
|
subseg, not a `_jr_` leader) that a subseg-name model would miss. Every carve MUST
|
|
resolve to exactly one owner or the run aborts (R32) — a stranded/duplicated carve is
|
|
the func_801734BC incident (§8b) and must never be silent.
|
|
|
|
Cost: ~6s -> ~0.1s by reading the overlay image ONCE and passing it to reloc_targets."""
|
|
import family_remap
|
|
base = oss_vram(ov)
|
|
syms = oss.load_ov_syms(ov)
|
|
|
|
# still-unmatched jr: an INCLUDE_ASM fn whose .s references a jtbl_. Resolve the .s
|
|
# basename through the symbol table (addr_of) so a CURATED name (e.g. listCdBuffer) is
|
|
# not dropped by a func_-shape fullmatch (§26-A LOW finding).
|
|
asm_jr = {}
|
|
for p in glob.glob(os.path.join(REPO, f"asm/{ov}/nonmatchings/*/*.s")):
|
|
if re.search(r'jtbl_[0-9A-Fa-f]{8}', open(p).read()):
|
|
nm = os.path.basename(p)[:-2]
|
|
a = oss.addr_of(nm, syms)
|
|
if a is not None:
|
|
asm_jr[a] = nm
|
|
|
|
# already-banked jr: every real-C def/define fn that references a committed carve
|
|
# offset in the IMAGE (read once, passed to reloc_targets).
|
|
cfg_lines = open(os.path.join(REPO, f"config/splat.{ov}.yaml")).read().splitlines()
|
|
carve_offs = {off for _li, off, _sub in rodata_carves(cfg_lines, ov)}
|
|
img = open(family_remap.img_path(ov), "rb").read()
|
|
banked, owners = {}, {} # owners: carve_off -> [names]
|
|
for cf in glob.glob(os.path.join(REPO, f"src/{ov}/*.c")):
|
|
_, items = oss.parse_overlay_c(open(cf).read(), syms)
|
|
for addr, name, kind, _ in items:
|
|
if kind not in ("def", "define") or not name or addr is None:
|
|
continue
|
|
targets = _reloc_targets_or_die(family_remap, ov, addr, data=img)
|
|
if targets is None:
|
|
continue
|
|
hits = {t - base for k, t in targets if k == "data" and (t - base) in carve_offs}
|
|
if hits:
|
|
banked[addr] = name
|
|
for off in hits:
|
|
owners.setdefault(off, []).append(name)
|
|
|
|
_assert_scan_covered(ov, "jr_inventory ownership scan")
|
|
# R32: every committed carve resolves to EXACTLY ONE owner, or abort loud.
|
|
#
|
|
# OWNERSHIP HAS TWO MORE SOURCES THAN THE RELOC SCAN ABOVE (P31 S70). The scan finds an owner
|
|
# only among the overlay's OWN real-C definitions, and it was aborting on 36 carves across 8
|
|
# binaries — every one of which is BYTE-GREEN (R22 213/213), i.e. the config is right and the
|
|
# MODEL was blind (R34). The two blind spots, measured:
|
|
#
|
|
# 1. THE SUBSEG NAME IS THE OWNERSHIP RECORD (32 of 36 = 89%). The isolate convention writes
|
|
# the owner into the name: a carve in `<ov>_jr_<ADDR>` belongs to `func_<ADDR>`. Several of
|
|
# those owners are RESIDENT-range functions (0x80135D20, 0x8015C32C …) instantiated in the
|
|
# overlay through a shared macro, so they are not overlay-local definitions and
|
|
# `parse_overlay_c` cannot see them at all. Reading the name is R33 — derive from the
|
|
# invariant instead of re-deriving it by scanning relocations.
|
|
# 2. A CARVE FOR A STILL-STUBBED FUNCTION IS PENDING, NOT STRANDED (the remaining 4). The
|
|
# carve is committed and the function has simply not banked yet; `func_8016AB6C` in
|
|
# ov_SC07_010 references its carve at 0x801A6460 from an INCLUDE_ASM stub.
|
|
#
|
|
# A carve with NONE of the three is still a hard abort — that is the real corruption this
|
|
# assertion exists to catch (§8b func_801734BC class).
|
|
_named = re.compile(r"^%s_jr_([0-9A-Fa-f]{8})$" % re.escape(ov))
|
|
named_owner = {off for _li, off, sub in rodata_carves(cfg_lines, ov)
|
|
if _named.match(str(sub or ""))}
|
|
pending = set()
|
|
try:
|
|
import corpus as _corpus
|
|
_stub_addrs = list(_corpus.stubs(ov))
|
|
except Exception:
|
|
_stub_addrs = [] # the stub oracle refusing is not this assertion's business
|
|
for _a in _stub_addrs:
|
|
try:
|
|
_t = family_remap.reloc_targets(ov, int(_a), data=img)
|
|
except Exception:
|
|
continue
|
|
pending |= {x - base for k, x in _t if k == "data" and (x - base) in carve_offs}
|
|
accounted = set(owners) | named_owner | pending
|
|
problems = [("UNOWNED", hex(base + o)) for o in sorted(carve_offs - accounted)]
|
|
problems += [("MULTI", hex(base + o), owners[o]) for o in sorted(owners) if len(owners[o]) > 1]
|
|
if problems:
|
|
sys.exit(f"jr_inventory({ov}): committed .rodata carve ownership is not 1:1 (R32/R33) — "
|
|
f"a stranded/duplicated carve (§8b func_801734BC class): {problems}")
|
|
|
|
alljr = dict(asm_jr)
|
|
alljr.update({a: "banked" for a in banked}) # marker; name in `banked`
|
|
return alljr, banked
|
|
|
|
|
|
def plan(ov, only=None):
|
|
"""Compute the resegment plan without touching disk. Returns a dict."""
|
|
base = oss_vram(ov)
|
|
cfg_lines = open(os.path.join(REPO, f"config/splat.{ov}.yaml")).read().splitlines()
|
|
objs = code_objects(cfg_lines)
|
|
obj_ranges = [] # (start_vram, end_vram, name, line_idx, indent)
|
|
for k, (li, ind, off, nm) in enumerate(objs):
|
|
end = objs[k + 1][2] if k + 1 < len(objs) else None
|
|
obj_ranges.append((base + off, (base + end) if end is not None else None, nm, li, ind))
|
|
|
|
def obj_of(vram):
|
|
for s, e, nm, li, ind in obj_ranges:
|
|
if s <= vram and (e is None or vram < e):
|
|
return nm
|
|
return None
|
|
|
|
alljr, banked = jr_inventory(ov)
|
|
if only:
|
|
only_addrs = {int(x[5:], 16) for x in only if re.fullmatch(r'func_[0-9A-Fa-f]{8}', x)}
|
|
# A region may host AT MOST ONE `.rodata` carve, because an object's `.rodata` is a single
|
|
# CONTIGUOUS section. So every ALREADY-BANKED jr in an object we are cutting must be cut too:
|
|
# otherwise it shares a region with the new core, and that one object has to emit both jump
|
|
# tables — which sit far apart in the island — into one `.rodata`. Byte-proven: isolating
|
|
# func_8015AE2C (jtbl 0x801D8B54) alone left the banked func_801734BC (jtbl 0x801D8C68) inside
|
|
# its region, and the object emitted a 0x34 `.rodata` spanning BOTH tables (image +33 B).
|
|
# Cutting at each banked jr gives every one its own region → exactly one carve per object.
|
|
# (This is what cookbook §8b's "bank same-subseg families ASCENDING" note was warning about;
|
|
# it is now enforced by construction rather than left to discipline.)
|
|
touched = {obj_of(a) for a in only_addrs}
|
|
only_addrs |= {a for a in banked if obj_of(a) in touched}
|
|
alljr = {a: v for a, v in alljr.items() if a in only_addrs}
|
|
# NB `banked` itself is deliberately NOT filtered — every banked carve must stay trackable.
|
|
|
|
# group jr by their -O2 object (skip -O0 objects + objects with no jr)
|
|
skipped_o0 = []
|
|
per_obj = {}
|
|
for a in sorted(alljr):
|
|
nm = obj_of(a)
|
|
if nm is None:
|
|
continue
|
|
if nm.endswith(O0_SUFFIX):
|
|
skipped_o0.append(a)
|
|
continue
|
|
per_obj.setdefault(nm, []).append(a)
|
|
|
|
# banked jr -> its object (for carve repoint)
|
|
banked_obj = {banked[a]: obj_of(a) for a in banked}
|
|
return {
|
|
"base": base, "cfg_lines": cfg_lines, "obj_ranges": obj_ranges,
|
|
"per_obj": per_obj, "banked": banked, "banked_obj": banked_obj,
|
|
"skipped_o0": skipped_o0,
|
|
}
|
|
|
|
|
|
def subseg_name(ov, vram):
|
|
return f"{ov}_jr_{vram:08X}" # uppercase hex, matching the func_XXXXXXXX convention
|
|
|
|
|
|
def carve_owners(ov, banked, base, carve_offs):
|
|
"""{carve_offset: func_name} — which already-banked jr owns each existing `.rodata` carve.
|
|
|
|
Resolved from the EXTRACTED IMAGE (`family_remap.reloc_targets` reads each function's lui/%lo
|
|
address operands), NOT from splat `.s`: splat emits **no `.s` for a MATCHED function** (its `.c`
|
|
carries real C), so an asm scan finds nothing and every banked carve silently goes untracked —
|
|
which is precisely how func_801734BC's carve got stranded. A banked jr owns a carve iff it
|
|
references that carve's address."""
|
|
import family_remap
|
|
owners = {}
|
|
for addr, fn in banked.items():
|
|
targets = _reloc_targets_or_die(family_remap, ov, addr)
|
|
if targets is None:
|
|
continue
|
|
for kind, t in targets:
|
|
if kind == "data" and (t - base) in carve_offs:
|
|
owners[t - base] = fn
|
|
_assert_scan_covered(ov, "carve_owners")
|
|
return owners
|
|
|
|
|
|
# .globl / .ent inside a §265 verbatim `__asm__` body — the separator can be a literal two-char
|
|
# escape (`\t`, `\n`) or a quote boundary, since the directives live inside C string literals.
|
|
_EMIT_SYM_RE = re.compile(r'\.(?:globl|ent)(?:\s|\\[nt]|")+([A-Za-z_]\w*)')
|
|
|
|
|
|
def _region_emit_start(items, syms, obj_lo, obj_hi):
|
|
"""The lowest vram a region's TEXT actually EMITS — which is not always its cut vram.
|
|
|
|
P31 S74. `overlay_src_split.parse_overlay_c` recognises four ADDRESSED ANCHOR forms; a §265
|
|
verbatim `__asm__(".text\n.globl func_X\n…")` body is none of them, so it lands in the
|
|
PREAMBLE of the next anchor. Preamble is assumed byte-neutral (comments + decls) — and for a
|
|
verbatim asm body that assumption is false: it emits its bytes. Cutting md_MAIN_003 at
|
|
func_800D0268 therefore carried the verbatim bodies of func_800D0100/0174/0204 into the new
|
|
region while the yaml said the region starts at 0x800D0268 — a config 0x168 bytes above where
|
|
the object's bytes actually begin. (Only the TRAILING case was handled, in `_partition`.)
|
|
|
|
So derive the boundary from the CONTENT: the min of the region's item addresses and of every
|
|
`.globl`/`.ent` symbol its text names that resolves INSIDE this object. Returns None when
|
|
nothing resolves. Where no verbatim asm is in play this equals the cut, so every existing
|
|
isolate is unchanged — the whole-binary byte-gate (G3/P9) remains the arbiter."""
|
|
best = None
|
|
for it in items:
|
|
if it[0] is not None:
|
|
best = it[0] if best is None else min(best, it[0])
|
|
for nm in _EMIT_SYM_RE.findall(it[3] or ''):
|
|
a = oss.addr_of(nm, syms)
|
|
if a is None or a < obj_lo or (obj_hi is not None and a >= obj_hi):
|
|
continue
|
|
best = a if best is None else min(best, a)
|
|
return best
|
|
|
|
|
|
def build_new_config(ov, p):
|
|
"""Return (new_cfg_lines, new_files:{path:content}, carve_renames:{old_sub:new_sub})."""
|
|
base = p["base"]
|
|
cfg_lines = list(p["cfg_lines"])
|
|
syms = oss.load_ov_syms(ov)
|
|
|
|
# 1) source repartition + the config code-region replacement (per object, bottom-up so
|
|
# line indices stay valid).
|
|
new_files = {}
|
|
replacements = [] # (line_idx, [new config lines])
|
|
carve_moves = {} # carve OFFSET (jtbl vram - base) -> the subseg that now hosts its fn
|
|
carve_renames = {} # old code-subseg -> new subseg (derived; for the overlays.mk --order)
|
|
carve_offs = {int(m.group(1), 16) for m in
|
|
(re.match(r'^\s*- \[(0x[0-9A-Fa-f]+),\s*\.rodata,\s*\w+\]', ln) for ln in cfg_lines)
|
|
if m}
|
|
owners = carve_owners(ov, p["banked"], base, carve_offs) # {carve_off: fn}
|
|
fn_carves = {}
|
|
for _off, _fn in owners.items():
|
|
fn_carves.setdefault(_fn, []).append(_off)
|
|
banked_by_obj = {}
|
|
for fn, obj in p["banked_obj"].items():
|
|
banked_by_obj.setdefault(obj, []).append(fn)
|
|
|
|
for s, e, nm, li, ind in p["obj_ranges"]:
|
|
if nm not in p["per_obj"]:
|
|
continue
|
|
cuts = p["per_obj"][nm] # jr vrams in this object
|
|
# A jr function that IS the object's first function (the object is already named after it,
|
|
# e.g. ov_SC06_029_jr_8017C954) yields a cut at the object start -> an empty region 0 at the
|
|
# same offset -> "code subseg … out of order" at config time (P31 S62). Such a cut is a no-op.
|
|
cuts = [c for c in cuts if c != s]
|
|
if not cuts:
|
|
continue
|
|
srcpath = os.path.join(REPO, f"src/{ov}/{nm}.c")
|
|
header, regions = _partition(srcpath, cuts, syms)
|
|
provided = _provided_types(header) # the types THIS TU's includes supply (P32 T1a)
|
|
# region 0 (lo=None) keeps the object name; each jr-led region -> _jr_<lo>. Regions are
|
|
# processed in address order, accumulating this object's file-scope decls as `ambient` so
|
|
# each region carries the decl context it had in the original single object.
|
|
cfg_block = []
|
|
ambient = []
|
|
for (lo, hi, items) in regions:
|
|
# EMPTY region 0: the object's first item IS the first cut (an already-isolated region
|
|
# whose leading jr is being cut again, e.g. cutting func_80178D40 out of
|
|
# ov_SC01_000_jr_801734BC — the leader 0x801734BC is a cut too, per the banked-jr rule).
|
|
# Emitting it would duplicate region 1's line exactly (same offset, and subseg_name(lo)
|
|
# == nm when the object is already named _jr_<leader>) → splat "segments out of order".
|
|
# ...and an EMPTY CUT region likewise (P31 S74): a region with no items emits no
|
|
# bytes, so its config line is a zero-length subseg AT the next boundary's offset —
|
|
# `- [0x1f74, c, md_MAIN_003_o0e]` immediately above the existing
|
|
# `- [0x1f74, c, md_MAIN_003_o0c]`, which the ascending/unique validator rejects as
|
|
# "out of order". o0_subsplit closes every run with a cut at the next anchor (or --hi);
|
|
# when that lands exactly on an existing subseg boundary the closing region is empty by
|
|
# construction and must simply not be emitted.
|
|
if not items:
|
|
continue
|
|
sub = nm if lo is None else subseg_name(ov, lo)
|
|
if lo is None:
|
|
off = s - base
|
|
else:
|
|
_emit = _region_emit_start(items, syms, s, e)
|
|
off = (lo if _emit is None else min(lo, _emit)) - base
|
|
cfg_block.append(f"{ind}- [{hex(off)}, c, {sub}]")
|
|
body = _render_region(header, items, old_sub=nm, new_sub=sub, ambient=ambient,
|
|
syms=syms, obj_start=s)
|
|
new_files[os.path.join(REPO, f"src/{ov}/{sub}.c")] = body
|
|
ambient = ambient + _file_scope_decls(items, provided) # context for later regions
|
|
# EVERY already-banked jr that now falls in this region must have its `.rodata` carve
|
|
# repointed to `sub` — not just one that LEADS it. A cut placed BELOW an already-banked jr
|
|
# MOVES that jr into the new region, so its C-emitted jump table is linked into the new
|
|
# object while the config still names the old subseg → the carve piece under-fills and every
|
|
# later symbol shifts (byte-proven: isolating func_8015AE2C at 0x8015AE2C moved the banked
|
|
# func_801734BC @0x801734BC, whose 20-B table then landed in the new object's .rodata,
|
|
# bloating it 0x1C→0x34 and lengthening the image). This stayed hidden because both earlier
|
|
# single-core isolations cut ABOVE func_801734BC, and the full isolate-all gave every jr its
|
|
# own leading region. Carves are keyed by OFFSET (the jtbl vram), since two banked jr of one
|
|
# object can now land in DIFFERENT regions. (Cookbook §8b's "bank ASCENDING" note is exactly
|
|
# this hazard — now handled instead of merely warned about.)
|
|
for fn in banked_by_obj.get(nm, []):
|
|
a = int(fn[len("func_"):], 16)
|
|
if (lo is None or a >= lo) and (hi is None or a < hi):
|
|
for _o in fn_carves.get(fn, []):
|
|
carve_moves[_o] = sub
|
|
replacements.append((li, cfg_block))
|
|
|
|
# apply config code-region replacements bottom-up
|
|
for li, block in sorted(replacements, reverse=True):
|
|
cfg_lines[li:li + 1] = block
|
|
|
|
# 2) repoint each .rodata carve piece — matched by OFFSET, not by subseg name, because two banked
|
|
# jr of one object can now land in DIFFERENT regions.
|
|
for i, ln in enumerate(cfg_lines):
|
|
m = re.match(r'^(\s*- \[)(0x[0-9A-Fa-f]+)(,\s*\.rodata,\s*)(\w+)(\].*)$', ln)
|
|
if not m:
|
|
continue
|
|
off, cur = int(m.group(2), 16), m.group(4)
|
|
new = carve_moves.get(off)
|
|
if new and new != cur:
|
|
carve_renames[cur] = new # for the overlays.mk --order (jtbl_carve re-emits it anyway)
|
|
cfg_lines[i] = m.group(1) + m.group(2) + m.group(3) + new + m.group(5)
|
|
|
|
return cfg_lines, new_files, carve_renames
|
|
|
|
|
|
def _partition(srcpath, cuts, syms):
|
|
"""overlay_src_split.partition but taking a preloaded syms dict."""
|
|
header, items = oss.parse_overlay_c(open(srcpath).read(), syms)
|
|
# A trailing chunk with real content (kind="tail") is unaddressable only POSITIONALLY —
|
|
# no anchor follows it for the parser to attach it to. Its content still HAS addresses:
|
|
# a §265 verbatim `__asm__` body names its symbol in `.globl`/`.ent`, and
|
|
# INCLUDE_ASM/INCLUDE_RODATA name theirs. When every symbol the chunk DEFINES resolves
|
|
# at/after the LAST cut, the chunk belongs — in file order and address order alike — at
|
|
# the end of the LAST region, exactly where the split leaves it; attaching it there
|
|
# preserves per-region file order and is byte-neutral by the same argument as the split
|
|
# itself. Anything else keeps the hard refusal (R32 spirit: never rewrite a TU minus
|
|
# content you cannot place). First hit: md_MAIN_003's trailing verbatim-asm pair
|
|
# func_800D3204/func_800D3234 blocking the 0x800D0D6C -O0 carve (P31 S68).
|
|
tail = [it for it in items if it[0] is None and it[2] == "tail"]
|
|
if tail:
|
|
last_cut = max(cuts) if cuts else None
|
|
for it in tail:
|
|
# .globl/.ent live inside C string literals, so the separator can be a literal
|
|
# two-char escape (`\t`, `\n`) or a quote boundary, not just whitespace.
|
|
defined = set(re.findall(
|
|
r'\.(?:globl|ent)(?:\s|\\[nt]|")+([A-Za-z_]\w*)', it[3])) | \
|
|
set(re.findall(r'INCLUDE_(?:ASM|RODATA)\("[^"]*",\s*(\w+)\)', it[3]))
|
|
addrs = {nm: oss.addr_of(nm, syms) for nm in defined}
|
|
bad = sorted(nm for nm, a in addrs.items()
|
|
if a is None or last_cut is None or a < last_cut)
|
|
if not defined or bad:
|
|
sys.exit(f"jr_isolate_all: unaddressable content in {srcpath} — trailing "
|
|
f"chunk defines {bad if defined else '(nothing resolvable)'} "
|
|
f"which does not resolve at/after the last cut "
|
|
f"({hex(last_cut) if last_cut is not None else None}); refusing "
|
|
f"to attach it to the last region (R32)")
|
|
footer = [it for it in items if it[2] == "footer"]
|
|
# R32 COVERAGE — the same guard as overlay_src_split.partition, and for the same reason:
|
|
# `addressed` silently discards any construct whose vram did not resolve, so this function
|
|
# rewrote the TU WITHOUT it. Measured P30 S38: one carve of ov_SC02_028 deleted the two
|
|
# definition-side asm-label-alias definitions emitting func_80183AF8 and func_80184268 (their
|
|
# C identifiers are aF*, which matched neither `func_<hex>` nor `syms`), and the overlay then
|
|
# failed to link. Six wave-6 drafts were written off against that. Fail loud instead.
|
|
lost = [it for it in items if it[0] is None and it[2] not in ("tail", "footer")]
|
|
# FILE-LOCAL `static` DEFINITIONS HAVE NO ADDRESS BY CONSTRUCTION, AND THAT IS NOT A DEFECT
|
|
# (P31 S71). A `static inline` helper (§82.1) emits NOTHING of its own — it exists to shape the
|
|
# code of the function that calls it — so it carries no symbol and `addr_of` cannot resolve it.
|
|
# The R32 guard above is right to refuse an UNPLACEABLE construct, but it was refusing these
|
|
# too, which blocked the isolate on 4 of the 6 overlays whose CARVE-REFUSED functions the
|
|
# isolate is the named remedy for (`bandsetup` on ov_SC03_010/013/092, `setup_80188D90` on
|
|
# ov_SC06_029). Place them instead: a file-local definition belongs with the region that USES
|
|
# it, and if two regions use it we REFUSE rather than duplicate (two copies of a used static
|
|
# are two different objects' code — a byte change, R43).
|
|
local_defs, unplaceable = [], []
|
|
for it in lost:
|
|
if (it[2] == "def" and it[1]
|
|
and re.search(r'^\s*static\b[^;{]*\b%s\b' % re.escape(it[1]),
|
|
it[3] or '', re.M)):
|
|
local_defs.append(it)
|
|
else:
|
|
unplaceable.append(it)
|
|
if unplaceable:
|
|
sys.exit(f"jr_isolate_all: {srcpath} has {len(unplaceable)} construct(s) with no resolvable "
|
|
f"address — refusing to rewrite the file without them (R32):\n" +
|
|
"\n".join(f" kind={it[2]} name={it[1]} :: {it[3].strip()[:110]}"
|
|
for it in unplaceable[:6]))
|
|
addressed = [it for it in items if it[0] is not None]
|
|
cuts = sorted(set(cuts))
|
|
bounds = [None] + cuts + [None]
|
|
regions = []
|
|
for lo, hi in zip(bounds[:-1], bounds[1:]):
|
|
sel = sorted((it for it in addressed
|
|
if (lo is None or it[0] >= lo) and (hi is None or it[0] < hi)),
|
|
key=lambda it: it[0])
|
|
regions.append((lo, hi, sel))
|
|
if tail: # tail = guarded last-region content (see above): it HAS
|
|
lo, hi, sel = regions[-1] # addresses, validated at/after the last cut, so it belongs
|
|
regions[-1] = (lo, hi, sel + tail)
|
|
if footer:
|
|
# footer = comment/blank-only trailing chunk: it emits NOTHING, so it must not be what
|
|
# makes an otherwise-empty last region look non-empty (P31 S74 — that is what kept
|
|
# build_new_config's empty-region skip from firing on md_MAIN_003, whose closing cut at
|
|
# 0x800D0D6C lands exactly on the existing md_MAIN_003_o0c boundary). Attach it to the last
|
|
# region that actually has content, so the text is preserved and no empty subseg is emitted.
|
|
idx = max((i for i, (_l, _h, sel) in enumerate(regions) if sel), default=len(regions) - 1)
|
|
lo, hi, sel = regions[idx]
|
|
regions[idx] = (lo, hi, sel + footer)
|
|
# Place each file-local `static` definition with the ONE region that references it.
|
|
for it in local_defs:
|
|
name = it[1]
|
|
users = [i for i, (lo, hi, sel) in enumerate(regions)
|
|
if any(re.search(r'\b%s\b' % re.escape(name), o[3] or '') for o in sel
|
|
if o is not it)]
|
|
if len(users) > 1:
|
|
sys.exit(f"jr_isolate_all: file-local {name!r} is used by {len(users)} of the new "
|
|
f"regions; carrying it into each would emit two copies of the same static "
|
|
f"(a byte change). Isolate a different jr-function, or hoist {name!r} to a "
|
|
f"shared header first (R43).")
|
|
idx = users[0] if users else 0
|
|
lo, hi, sel = regions[idx]
|
|
# the definition must precede its callers inside the region (C89 needs the declaration)
|
|
regions[idx] = (lo, hi, [it] + sel)
|
|
return header, regions
|
|
|
|
|
|
# a hoistable declaration line: an `extern` decl, or a func/data prototype ending in `;`
|
|
# with no `{` body. These are legal to REPEAT in C (unlike typedef/struct/enum), so we hoist a
|
|
# deduped copy to the region top — every symbol is then declared before any body uses it (a cut can
|
|
# strand a use above its in-region decl; the source redundantly re-declares externs per fn-group).
|
|
_HOIST_RE = re.compile(
|
|
r'^\s*(?:extern\b.*;'
|
|
r'|[A-Za-z_][\w\*\s]*\b(?:func_[0-9A-Fa-f]{8}|D_[0-9A-Fa-f]{8})\b[^{]*;)\s*(?:/\*.*\*/)?\s*$')
|
|
|
|
|
|
# a col-0 decl whose base type is a BUILTIN / include-provided type is safe to hoist as-is; one
|
|
# naming a FILE-LOCAL type is only safe once that type is carried too — which `file_scope_types()`
|
|
# now does, so such decls ride along after their typedef (types are emitted before decls).
|
|
#
|
|
# ^ THAT COMMENT DESCRIBED A FIX THAT WAS NEVER APPLIED TO THE CODE (Phase 26-A audit, HIGH).
|
|
# The predicate only ever whitelisted builtins, so a decl naming a carried file-local type was matched
|
|
# by _HOIST_RE and then SILENTLY DROPPED. Measured: 4,040 dropped col-0 decls — 3,357 DATA externs and
|
|
# **683 function PROTOTYPES**. The data drops are loud (undeclared identifier -> compile error, someone
|
|
# notices). The prototype drops are NOT: in C89 an undeclared function is implicitly `int f()`, so the
|
|
# TU still COMPILES — with the wrong return type and lost pointer-ness. And this project has BYTE-PROVEN
|
|
# that the return type drives codegen (cookbook: "schedule — delay-slot fill via void return type";
|
|
# ov_SC01_077_after.c carries an `extern int` -> `extern void` flip described as byte-neutral precisely
|
|
# because the return type moves the delay slot). So a dropped prototype is a SILENT BYTE-CHANGER, armed
|
|
# to fire on the NEXT carve. Today's split is green only because the source redundantly re-declares
|
|
# externs per fn-group, so most items carry their own decl. That is luck, not design.
|
|
#
|
|
# Two of the dropped base types are not even file-local: `uint` (139 drops) and `code_fn` (21) are
|
|
# DEFINED IN src/shared/engine_types.h, which engine_core.h pulls into every region — the predicate was
|
|
# rejecting INCLUDE-PROVIDED types it had no reason to reject. And `volatile` (3 drops) fell off because
|
|
# the qualifier group has `const` but not `volatile`.
|
|
_SAFE_TYPE = re.compile(
|
|
r'^\s*(?:extern\s+)?(?:(?:const|volatile)\s+)*(?:(?:un)?signed\s+)?'
|
|
r'(?:void|char|short|int|long|float|double'
|
|
r'|[su](?:8|16|32|64)|M2C_UNK|MNC_UNK)\b')
|
|
|
|
# the base type of a col-0 decl (after extern/qualifiers/struct-union-enum), for the carried-type test
|
|
_BASE_TYPE = re.compile(
|
|
r'^\s*(?:extern\s+)?(?:(?:const|volatile)\s+)*(?:struct\s+|union\s+|enum\s+)?([A-Za-z_]\w*)')
|
|
|
|
_ENGINE_TYPES = None
|
|
|
|
|
|
def _header_types(paths):
|
|
"""Every type name the given header files define — the same recognisers the carried-type test has
|
|
always used (typedef struct {...} X; attribute-between-brace-and-name; plain typedefs; bare tags;
|
|
body-defined tags; fn-ptr typedefs)."""
|
|
names = set()
|
|
for h in paths:
|
|
p = os.path.join(REPO, h)
|
|
if not os.path.exists(p):
|
|
continue
|
|
t = open(p, errors="replace").read()
|
|
names |= set(re.findall(r'\}\s*([A-Za-z_]\w*)\s*;', t)) # typedef struct {...} X;
|
|
# `} __attribute__((packed, aligned(1))) X;` — the lifted Block4/Blk4_E960 shape (P31 S62): the
|
|
# attribute sits between the brace and the name, so the pattern above never saw them.
|
|
names |= set(re.findall(r'\}\s*__attribute__\s*\(\(.*?\)\)\s*([A-Za-z_]\w*)\s*;', t))
|
|
names |= set(re.findall(r'^\s*typedef\s+[^;{}]*?\b([A-Za-z_]\w*)\s*;', t, re.M))
|
|
names |= set(re.findall(r'^\s*(?:struct|union|enum)\s+([A-Za-z_]\w*)\s*;', t, re.M))
|
|
# ...and the same TAGS defined WITH A BODY (`struct PW8017E6D8 { int w; };`). The
|
|
# forward-decl pattern above only catches `struct X;`, and the `}\s*X;` pattern above
|
|
# catches `typedef struct {...} X;` — a plain tagged definition matches NEITHER, so its
|
|
# tag was absent from _ENGINE_TYPES and any `extern struct X D_…;` failed the
|
|
# carried-type test. Phase 29 SESSION-19: that is what blocked the func_8017C954 carve
|
|
# (`extern struct PW8017E6D8 D_801E1EC4;`, and PW8017E6D8 sits at engine_types.h:658).
|
|
# Measured blast radius: 77 such tags in engine_types.h were invisible to this check.
|
|
names |= set(re.findall(r'^\s*(?:struct|union|enum)\s+([A-Za-z_]\w*)\s*\{', t, re.M))
|
|
# fn-ptr typedefs — the name sits INSIDE the parens (`typedef void (*ActorFn)(void);`), so
|
|
# every name-before-';' pattern above misses it. Measured: exactly the 5 residual drops
|
|
# (ActorFn, FuncPtr, DispatchFn, VoidFn, code_fn). Without this the coverage assertion below
|
|
# would fire on legitimate input.
|
|
names |= set(re.findall(r'typedef\s+[^;{}]*?\(\s*\*\s*([A-Za-z_]\w*)\s*\)\s*\([^;]*\)\s*;', t))
|
|
return names
|
|
|
|
|
|
_PROVIDED_CACHE = {}
|
|
|
|
|
|
def _provided_types(header):
|
|
"""The type names THIS TU's own `#include` lines actually supply (P32 T1a fix, R33).
|
|
|
|
The carried-type test used to consult `_engine_types()` — engine_types.h + common.h — for EVERY
|
|
TU, on the assumption that every region `#include`s engine_core.h -> engine_types.h. Overlays do;
|
|
the RESIDENT, the md_* modules and main's TUs include only `common.h`. So a resident file-local
|
|
typedef whose NAME engine_types.h also defines (`CdFileLoc`, lifted from an overlay long ago) was
|
|
silently NOT carried ("the shared headers already define it") into regions that never include
|
|
those headers -> `parse error before cdFileLocTable` in both new region TUs (P32 T1a, resident
|
|
func_800D128C isolation). Derive the provided set from the header's include lines instead."""
|
|
incs = re.findall(r'^\s*#\s*include\s+"([^"]+)"', header or "", re.M)
|
|
paths = []
|
|
for inc in incs:
|
|
b = os.path.basename(inc)
|
|
if b == "engine_core.h":
|
|
# engine_core.h itself is NOT scanned: its typedefs live INSIDE `DEFINE_func_*()` macro
|
|
# bodies and reach a region only where that macro is invoked. What every includer gets at
|
|
# file scope is what engine_core.h #includes — engine_types.h + common.h (the legacy set).
|
|
paths += ["src/shared/engine_types.h", "include/common.h"]
|
|
elif b == "engine_types.h":
|
|
paths += ["src/shared/engine_types.h", "include/common.h"]
|
|
elif b == "common.h":
|
|
paths += ["include/common.h"]
|
|
else:
|
|
for cand in (f"include/{inc}", f"src/shared/{b}", f"include/{b}"):
|
|
if os.path.exists(os.path.join(REPO, cand)):
|
|
paths.append(cand)
|
|
break
|
|
key = tuple(sorted(set(paths)))
|
|
if key not in _PROVIDED_CACHE:
|
|
_PROVIDED_CACHE[key] = _header_types(key)
|
|
return _PROVIDED_CACHE[key]
|
|
|
|
|
|
def _engine_types():
|
|
"""Every type name the SHARED headers provide (engine_types.h / common.h) — the OVERLAY case, kept for
|
|
callers that have no TU header in hand. Region carrying uses `_provided_types(header)` (above)."""
|
|
global _ENGINE_TYPES
|
|
if _ENGINE_TYPES is None:
|
|
_ENGINE_TYPES = _header_types(("src/shared/engine_types.h", "include/common.h"))
|
|
return _ENGINE_TYPES
|
|
|
|
|
|
|
|
# `} __attribute__((packed, aligned(1))) Blk4_9B4;` — the ATTRIBUTE SITS BETWEEN THE BRACE AND THE
|
|
# NAME (P31 S71, §323 blocker 2). Every type-name scan here matched `\}\s*(\w+)\s*;`, which reads
|
|
# `__attribute__` as the name, fails on the following `((`, and yields NOTHING — so a packed
|
|
# file-local typedef never entered `carried`, every decl naming it read as "unknown type", and
|
|
# jr_isolate_all REFUSED the whole overlay. That is the entire §323 carve blocker 2, on
|
|
# ov_SC07_000 (`Blk4_9B4`, `Blk4`) and ov_SC03_029 (`Block8_80181600`). Strip attributes first.
|
|
_ATTR = re.compile(r'__attribute__\s*\(\((?:[^()]|\([^()]*\))*\)\)')
|
|
|
|
|
|
def _strip_attrs(block):
|
|
return _ATTR.sub(' ', block)
|
|
|
|
|
|
# `typedef struct Tag Alias;` — a bodiless typedef of an existing tag (see _type_names, §497)
|
|
_TYPEDEF_TAG_ALIAS = re.compile(r'^\s*typedef\s+(?:struct|union|enum)\s+([A-Za-z_]\w*)\s+([A-Za-z_]\w*)\s*;\s*$')
|
|
|
|
|
|
def _file_scope_decls(items, provided=None):
|
|
"""[(line, [syms])] for every decl that stood at FILE SCOPE in the original TU, in item
|
|
order. TWO sources — the second is the §8b scoping-wall fix:
|
|
|
|
(1) COL-0 extern/proto lines in the `.c` text, with a builtin base type (a file-local
|
|
type would be a parse error if hoisted above its typedef — see _SAFE_TYPE).
|
|
(2) The LEADING EXTERNS of every `DEFINE_func_*()` macro the region invokes. The macro
|
|
expands at file scope to `extern <type> <sym>; ... <def>`, so those externs ARE part
|
|
of the TU's file-scope decl environment — but they live in engine_core.h, so no col-0
|
|
scan of the `.c` can see them. This is what stranded `func_801734BC` from
|
|
`extern s16 D_80126B3E;` (declared only inside DEFINE_func_80173460). Their types come
|
|
from engine_types.h/common.h — included by engine_core.h at every region top — so they
|
|
need no _SAFE_TYPE guard. (Externs *inside* macro bodies are block-scope shadows: they
|
|
expand with the invocation and are never hoisted.)
|
|
(3) The PROTOTYPE IMPLIED BY EVERY FUNCTION DEFINITION (`def` items and the `DEFINE_func_*`
|
|
/ SETTER / RETCONST macros' own definitions). In ONE translation unit a file-scope
|
|
definition declares its function for all code below it — so a cut that moves the
|
|
definition into an earlier region strands every later caller that took its address
|
|
(`func_8012B2CC undeclared`). Every overlay def has external linkage (no `static`), so
|
|
re-declaring it in a later region is always legal.
|
|
(4) The col-0 TYPE definitions, so a carried prototype naming a file-local type
|
|
(`Vec3s *a0`) still parses. Returned flagged so the renderer emits types FIRST.
|
|
|
|
Returns [(text, is_type)] in item order."""
|
|
# Collect the types this layer CARRIES first, so a decl naming one can ride along after its typedef
|
|
# (which is exactly what the _SAFE_TYPE comment has always claimed, and never did).
|
|
carried = set()
|
|
for _, _, _kind, text in items:
|
|
for block in oss.file_scope_types(text):
|
|
block = _strip_attrs(block)
|
|
for a, b in re.findall(r'\}\s*([A-Za-z_]\w*)\s*;|\b(?:struct|union|enum)\s+([A-Za-z_]\w*)', block):
|
|
carried.add(a or b)
|
|
m = _TYPEDEF_TAG_ALIAS.match(block.strip()) # the alias of `typedef struct Tag Alias;` (§497)
|
|
if m:
|
|
carried.add(m.group(2))
|
|
carried |= set(re.findall(r'typedef\s+[^;{}]*?\(\s*\*\s*([A-Za-z_]\w*)\s*\)\s*\([^;]*\)\s*;', block))
|
|
if provided is None: # legacy callers: the overlay assumption
|
|
provided = _engine_types()
|
|
known = carried | provided
|
|
|
|
out, dropped = [], []
|
|
# A CARRIED TYPE MAY BE EMITTED ONCE PER REGION, NOT ONCE PER ITEM (P31 S67, cookbook §321).
|
|
# `file_scope_types` is called per item and the carried layer is the union over every item that
|
|
# feeds this region, so a tag several of them each define at file scope — legal while they were
|
|
# separate TUs — arrives here as N copies of one definition. At file scope in ONE TU that is
|
|
# fatal: measured on ov_SC02_000, the isolation emitted `struct sprite8` FOUR times into
|
|
# `ov_SC02_000_jr_80187B40.c` and cc1 rejected the region with `redefinition of struct sprite8`,
|
|
# taking the whole overlay's build down (and, because the previous object was still on disk, the
|
|
# SHA1 read GREEN afterwards — R53).
|
|
# Dedupe by NAME, comparing bodies with comments and whitespace normalised away, because the
|
|
# copies differ only in their hand-written field comments. Two DIFFERENT bodies under one name
|
|
# are a real conflict that a rename must resolve, so those are refused loudly (R43), never
|
|
# silently merged — picking either one would change what the region compiles to.
|
|
seen_types, type_conflicts = {}, []
|
|
|
|
def _type_names(block):
|
|
block = _strip_attrs(block)
|
|
# `typedef struct Tag Alias;` (no body) DEFINES the alias and merely REFERENCES the tag (P32 T1b,
|
|
# cookbook §497). Keying it by the tag made it collide with the tag's own `struct Tag {...};`
|
|
# definition block — same key, different bodies — and the R43 refusal fired on legal C
|
|
# (ov_SC02_017: `struct Rec801806C8_s {...} __attribute__((packed, aligned(1)));` +
|
|
# `typedef struct Rec801806C8_s Rec801806C8;`). A typedef whose alias EQUALS the tag
|
|
# (`typedef struct X X;`) keeps the old key so it still dedupes/refuses against a second one.
|
|
m = _TYPEDEF_TAG_ALIAS.match(block.strip())
|
|
if m and m.group(1) != m.group(2):
|
|
return {m.group(2)}
|
|
names = {a or b for a, b in
|
|
re.findall(r'\}\s*([A-Za-z_]\w*)\s*;|\b(?:struct|union|enum)\s+([A-Za-z_]\w*)', block)}
|
|
names |= set(re.findall(r'typedef\s+[^;{}]*?\(\s*\*\s*([A-Za-z_]\w*)\s*\)\s*\([^;]*\)\s*;', block))
|
|
return {n for n in names if n}
|
|
|
|
def _norm_body(block):
|
|
return re.sub(r'\s+', ' ', re.sub(r'/\*.*?\*/|//[^\n]*', '', block, flags=re.S)).strip()
|
|
|
|
for _, _, kind, text in items:
|
|
for block in oss.file_scope_types(text): # (4) types first-class
|
|
names = _type_names(block)
|
|
if not names:
|
|
# Anonymous — §321: two identically-spelled anonymous struct typedefs are DISTINCT
|
|
# types, so there is nothing safe to dedupe against. Emit as-is.
|
|
out.append((block, True))
|
|
continue
|
|
# ...AND NEVER RE-EMIT ONE THE SHARED HEADERS ALREADY DEFINE. Every region `#include`s
|
|
# engine_core.h -> engine_types.h at its top, so carrying a definition of a type that
|
|
# lives there is an unconditional `redefinition of struct X`. Measured on ov_SC02_000:
|
|
# `struct sprite8` is engine_types.h:417, and the carry emitted it again into
|
|
# ov_SC02_000_jr_80187B40.c. _engine_types() already recognises body-defined tags (the
|
|
# SESSION-19 PW8017E6D8 fix); nothing was ever consulting it on this path.
|
|
if names <= provided: # P32 T1a: the TU's OWN includes, not the overlay assumption
|
|
continue
|
|
key = tuple(sorted(names))
|
|
body = _norm_body(block)
|
|
# A BARE TAG FORWARD DECL (`struct X;` — no typedef, no body) is not a body at
|
|
# all: C89 lets it repeat and coexist with the later definition in one TU, so it
|
|
# must neither register as the tag's body nor conflict with one. Emit it in
|
|
# place, in original order. (P31 S68: md_MAIN_003 carries `struct S_D2394;`
|
|
# ahead of pointer uses and the full `typedef struct S_D2394 {...}` later; the
|
|
# body-compare refused that legal pair as R43-conflicting.) Typedef forward
|
|
# forms (`typedef struct X X;`) stay on the dedupe path — repeating a typedef
|
|
# IS a C89 redefinition error, so those must still collapse or refuse.
|
|
if re.match(r'^(?:struct|union|enum)\s+\w+\s*;$', body):
|
|
out.append((block, True))
|
|
continue
|
|
if key not in seen_types:
|
|
seen_types[key] = body
|
|
out.append((block, True))
|
|
elif seen_types[key] != body:
|
|
type_conflicts.append((key, seen_types[key], body))
|
|
# THE CARRIED LINE MUST NOT CARRY AN UNTERMINATED COMMENT (P31 S74, byte-witnessed).
|
|
# A col-0 decl whose trailing `/*` note WRAPS —
|
|
# extern void *func_80185C6C(); /* §183 SIGNATURE-adopted-TU;
|
|
# calls go through a (s32,s32) fn-ptr cast, the TU's own idiom */
|
|
# — used to be hoisted VERBATIM, opening a comment in the region's decl layer that then
|
|
# ran on and SILENTLY ATE the next carried decls until the next `*/` (measured in
|
|
# ov_SC06_029_jr_801867D0.c: `extern void func_8012C218(void *a0);` and
|
|
# `extern u8 D_80190348[];` vanished into it). A dropped file-scope decl is a silent
|
|
# byte-changer, so mask on the SAME comment-state model the parser uses (oss.comment_open_at,
|
|
# R33): skip a line that BEGINS inside a comment, and truncate one that OPENS a comment it
|
|
# does not close. Comments emit no code, so dropping the note is byte-neutral.
|
|
_tlines = text.split("\n")
|
|
_opens = oss.comment_open_at(_tlines)
|
|
for _li, line in enumerate(_tlines):
|
|
if _opens[_li]: # interior of a wrapped block comment
|
|
continue
|
|
_code, _still_open = oss._strip(line, False)
|
|
if _still_open: # trailing `/*` that never closes on this line
|
|
line = line[:line.index("/*")].rstrip()
|
|
if not line or line[0].isspace(): # col-0 only (block-scope stays put)
|
|
continue
|
|
if "{" in line or "}" in line:
|
|
continue
|
|
if not _HOIST_RE.match(line):
|
|
continue
|
|
# A col-0 line may GLUE code-emitting macro invocations onto a declaration
|
|
# (`extern s32 aF…(…) __asm__(""); DEFINE_func_8014C4AC() DEFINE_func_8014C568() …`,
|
|
# ov_SC03_107 S62). Carrying it whole re-instantiated the shared bodies inside the new
|
|
# region (duplicate, name-mangled definitions -> `.globl` with no name). Carry only the
|
|
# `;`-terminated declaration segments; for each glued DEFINE_ invocation carry its
|
|
# implied prototype + macro externs instead (a dropped prototype is a silent byte-changer).
|
|
glued = re.findall(r'\bDEFINE_\w+\s*\(\s*\)', re.sub(r'/\*.*?\*/|//.*$', '', line))
|
|
if glued:
|
|
code = re.sub(r'/\*.*?\*/|//.*$', '', line) # comments may hold `;` — strip first
|
|
decls = [x.strip() + ';' for x in code.split(';')[:-1] if x.strip() and 'DEFINE_' not in x]
|
|
for inv in glued:
|
|
for ml in oss.macro_externs(inv):
|
|
out.append((ml, False))
|
|
mp = oss.macro_proto(inv)
|
|
if mp:
|
|
out.append((mp, False))
|
|
else:
|
|
decls = [line.rstrip()] # the original whole-line behaviour
|
|
for d in decls:
|
|
base = _BASE_TYPE.match(d)
|
|
if _SAFE_TYPE.match(d) or (base and base.group(1) in known):
|
|
out.append((d.rstrip(), False))
|
|
else:
|
|
dropped.append(d.rstrip()) # REPORTED, never silently dropped (R32)
|
|
proto = None
|
|
if kind == "define": # (2) macro-injected file-scope externs
|
|
for line in oss.macro_externs(text):
|
|
out.append((line, False))
|
|
proto = oss.macro_proto(text)
|
|
elif kind == "def":
|
|
proto = oss.def_proto(text)
|
|
if proto: # (3) the definition's implied declaration
|
|
out.append((proto, False))
|
|
|
|
# A NAME CARRYING TWO DIFFERENT BODIES IS NOT DEDUPABLE (R43). Emitting either one silently
|
|
# decides which definition the region compiles against; refuse and name the tag instead.
|
|
if type_conflicts:
|
|
sys.exit("[jr_isolate_all] %d carried type name(s) have CONFLICTING bodies — a rename is "
|
|
"needed, not a dedupe (R43):\n%s" % (
|
|
len(type_conflicts),
|
|
"\n".join(" %s\n A: %s\n B: %s" % ("/".join(k), a[:150], b[:150])
|
|
for k, a, b in type_conflicts[:5])))
|
|
|
|
# COVERAGE ASSERTION (R32). A line _HOIST_RE recognised as hoistable but that we could not place is
|
|
# a BUG, never a silent no-op. Print the base-type histogram so the cause is named, not guessed —
|
|
# this single check would have surfaced all 4,040 drops the day the first split shipped.
|
|
if dropped:
|
|
hist = collections.Counter()
|
|
for l in dropped:
|
|
m = _BASE_TYPE.match(l)
|
|
hist[m.group(1) if m else "?"] += 1
|
|
protos = sum(1 for l in dropped if re.search(r'\bfunc_[0-9A-Fa-f]{8}\s*\(', l))
|
|
sys.exit(
|
|
f"[jr_isolate_all] {len(dropped)} file-scope decl(s) matched _HOIST_RE but could not be "
|
|
f"placed — REFUSING to emit a region that silently omits them.\n"
|
|
f" {protos} are function PROTOTYPES: in C89 an undeclared function is implicitly `int f()`, "
|
|
f"so the TU still COMPILES with the WRONG RETURN TYPE — and return type drives delay-slot "
|
|
f"fill in this codebase. A dropped prototype is a SILENT BYTE-CHANGER.\n"
|
|
f" base types: {dict(hist.most_common(12))}\n"
|
|
f" e.g. {dropped[:3]}\n"
|
|
f" Fix: carry the naming type (file_scope_types) or add it to src/shared/engine_types.h.")
|
|
# FINAL PASS (P31 S62): NO carried line may contain a code-emitting `DEFINE_…()` invocation,
|
|
# whichever branch produced it (a one-line `extern …; DEFINE_func_X() DEFINE_func_Y()` item
|
|
# reaches here both as a col-0 decl and as a `define` item's macro_externs text). Keep the
|
|
# `;`-terminated declaration segments, and carry each glued macro's implied prototype instead.
|
|
cleaned = []
|
|
for line, is_block in out:
|
|
if is_block or not re.search(r'\bDEFINE_\w+\s*\(\s*\)', line):
|
|
cleaned.append((line, is_block)); continue
|
|
code = re.sub(r'/\*.*?\*/|//.*$', '', line)
|
|
for x in code.split(';')[:-1]:
|
|
x = x.strip()
|
|
if x and 'DEFINE_' not in x:
|
|
cleaned.append((x + ';', False))
|
|
for inv in re.findall(r'\bDEFINE_\w+\s*\(\s*\)', code):
|
|
mp = oss.macro_proto(inv)
|
|
if mp and (mp, False) not in cleaned:
|
|
cleaned.append((mp, False))
|
|
return cleaned
|
|
|
|
|
|
def _rewrite_includes(text, old_sub, new_sub, syms, obj_start):
|
|
"""Address-aware INCLUDE_ASM/INCLUDE_RODATA path repoint (P31 S68).
|
|
|
|
`rewrite_asm_subseg` repointed EVERY `/nonmatchings/<old_sub>"` occurrence in the item
|
|
text — including a glued §154-A LEADING-ISLAND reference. But splat regenerates a
|
|
piece-owned rodata symbol's `.s` under the subseg that owns its ADDRESS: after the
|
|
md_MAIN_003 3-way carve, `D_800CEDF8.s` (island, vram 0x800CEDF8 < the object's code
|
|
start 0x800CEED0) stayed at `nonmatchings/md_MAIN_003/` while the repointed include
|
|
named `nonmatchings/md_MAIN_003_jr_800D12D0/` — `can't open ... for reading` at
|
|
assembly (loud, thankfully). A symbol BELOW the object's code start is exactly the
|
|
leading-island case, so its include keeps its original path; everything else — function
|
|
stubs, in-code data words (D_800D3200), migrated tail tables (addr beyond the object)
|
|
— keeps the proven blanket-rewrite behavior, unresolvable names included."""
|
|
out = []
|
|
inc_re = re.compile(r'\s*INCLUDE_(?:ASM|RODATA)\("[^"]*/nonmatchings/'
|
|
+ re.escape(old_sub) + r'",\s*(\w+)\)')
|
|
for line in text.split("\n"):
|
|
m = inc_re.match(line)
|
|
if m:
|
|
a = oss.addr_of(m.group(1), syms)
|
|
if a is not None and obj_start is not None and a < obj_start:
|
|
out.append(line) # piece-owned island content: path unchanged
|
|
continue
|
|
out.append(oss.rewrite_asm_subseg(line, old_sub, new_sub))
|
|
return "\n".join(out)
|
|
|
|
|
|
def _render_region(header, items, old_sub, new_sub, ambient, syms=None, obj_start=None):
|
|
"""Region .c = header + AMBIENT file-scope decls (from earlier regions of this object, in
|
|
original order, deduped by symbol) + the region's items unchanged.
|
|
|
|
WHY THIS IS BYTE-NEUTRAL AND CONFLICT-FREE BY CONSTRUCTION: `ambient` reproduces the
|
|
original TU's file-scope decl environment, carried strictly FORWARD (regions are in address
|
|
order and file order == address order, so every ambient source textually preceded every item
|
|
of this region in the original). Therefore (a) every carried decl already coexisted with
|
|
every definition in the one original TU, so no NEW `conflicting types` can arise; (b) decl
|
|
compatibility is order-symmetric, so hoisting a decl earlier is safe; (c) decls emit no code.
|
|
The loose-typing shadows — e.g. `func_80173544`, defined at file scope as
|
|
`s32 f(void *)` yet declared `extern void f(void);` *inside* func_801734BC's body — live in
|
|
bodies, travel with their item, and are never hoisted, so the split never creates the clash a
|
|
naive "declare every used symbol" completion would. `make build` (SHA1) remains the sole
|
|
arbiter (G3/P9/R22)."""
|
|
if new_sub != old_sub:
|
|
items = [(a, n, k, _rewrite_includes(t, old_sub, new_sub, syms, obj_start))
|
|
for a, n, k, t in items]
|
|
# Dedup by EXACT decl text, not by symbol: this codebase is loosely typed, so one symbol can
|
|
# legally carry several distinct (even mutually-warning) file-scope decls — the baseline build
|
|
# emits 87 `type mismatch with previous external decl` warnings and is still byte-identical.
|
|
# Collapsing them to the first would drop a decl the original TU had (e.g. hide a definition's
|
|
# own signature behind an earlier, differently-typed canonical extern). Emitting every distinct
|
|
# decl in original order reproduces the original sequence exactly.
|
|
types, decls, seen = [], [], set()
|
|
for text, is_type in ambient:
|
|
key = re.sub(r'\s+', ' ', text.strip())
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
(types if is_type else decls).append(text)
|
|
parts = [header]
|
|
if types or decls:
|
|
# NB the trailing END MARKER is load-bearing, not decoration: family_remap.extract_unit walks
|
|
# BACKWARD from a definition absorbing every preceding extern/comment/blank line as the fn's
|
|
# "preamble". Without a stop, the first item of a region swallows this whole carried layer —
|
|
# which then gets templated into every sibling (dragging ~140 unrelated externs, some naming
|
|
# types the sibling's TU lacks) and the gate fails. The marker bounds the layer.
|
|
parts.append("/* ==== Phase-26 §8b carried decl layer (jr_isolate_all.py) "
|
|
"===================\n"
|
|
" * The file-scope decl environment from earlier code regions of this object —\n"
|
|
" * file-local types, col-0 decls, DEFINE_func macro externs, and each earlier\n"
|
|
" * definition's implied prototype (types first, then decls in original order).\n"
|
|
" * Decls emit no code => byte-neutral. See cookbook §8c. */\n"
|
|
+ "\n".join(types + decls)
|
|
+ "\n/* ==== end §8b carried decl layer ==== */")
|
|
parts.extend(t for _, _, _, t in items)
|
|
return "\n".join(parts) + "\n"
|
|
|
|
|
|
def repoint_overlays_mk(carve_renames, dry, ov=None, cfg_lines=None):
|
|
"""Repoint both overlays.mk consumers of a renamed carve object: the `--order` leaf AND the
|
|
§8e `JTBL_PADS` target var.
|
|
|
|
THE PADS LINE MUST FOLLOW ITS SPAN (P30 S48, the 0b blocker). A multi-table span's pad spec is
|
|
keyed by the OBJECT that emits the tables, so when isolation moves the span's owner into
|
|
`<ov>_jr_<addr>` the spec has to move with it. Leaving it behind fails TWO different ways, both
|
|
observed on ov_SC02_037's 4-table span (spec `0,0,0,0`, tables +0x0,+0x14,+0x34,+0x4c):
|
|
* plain `make build` after a bare isolate -> the stale line arms the pads filter on the
|
|
residual object, which now emits NO jump table:
|
|
`jtbl_rodata_pads: consumed 0 rodata .align(s) but 4 pad spec(s) given` (S47, hard error);
|
|
* the jtbl_family_bank path (isolate -> jtbl_carve) -> `set_pads_vars` regenerates the block
|
|
keyed by the CURRENT subseg names, finds no prior spec under the new name, and the line is
|
|
SILENTLY DROPPED. cc1's natural `.align 3` then pads the span's non-8-aligned interior
|
|
tables (+4 before table 2 here) and the image shifts: `built, bytes differ`.
|
|
Byte-neutral: only the target NAME changes; the spec and its `tables=` record are untouched.
|
|
|
|
Fails loud (R32) if the old object still hosts a `.rodata` piece — then the line is ambiguous
|
|
(jtbl_carve's invariant is one contiguous .rodata run per object, so this should be
|
|
unreachable; if it ever fires, the carve set is the thing to fix, not this rename)."""
|
|
mk = os.path.join(REPO, "config/overlays.mk")
|
|
txt = open(mk).read()
|
|
_mk_base = txt
|
|
changed = []
|
|
for old_sub, new_sub in carve_renames.items():
|
|
pat = rf'(--order[^#\n]*?){re.escape(old_sub)}\.o'
|
|
if re.search(pat, txt):
|
|
txt = re.sub(pat, lambda m: m.group(1) + new_sub + ".o", txt, count=1)
|
|
changed.append(f"--order {old_sub}.o -> {new_sub}.o")
|
|
if ov is None:
|
|
continue
|
|
pads_pat = rf'^(build/src/{re.escape(ov)}/){re.escape(old_sub)}(\.o: JTBL_PADS := )'
|
|
if not re.search(pads_pat, txt, re.M):
|
|
continue
|
|
if cfg_lines is not None and any(
|
|
re.match(rf'^\s*- \[0x[0-9A-Fa-f]+,\s*\.rodata,\s*{re.escape(old_sub)}\]', ln)
|
|
for ln in cfg_lines):
|
|
sys.exit(f"jr_isolate_all: {old_sub} has a JTBL_PADS line AND still hosts a .rodata "
|
|
f"carve after the split — refusing to repoint the spec to {new_sub} (R32). "
|
|
f"One object must own at most one contiguous .rodata run.")
|
|
txt = re.sub(pads_pat, lambda m: m.group(1) + new_sub + m.group(2), txt, count=1, flags=re.M)
|
|
changed.append(f"JTBL_PADS {old_sub}.o -> {new_sub}.o")
|
|
if not dry:
|
|
MKW.write_overlays_mk(txt, path=mk, base=_mk_base)
|
|
return changed
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("ov")
|
|
ap.add_argument("--only", help="comma-separated func_XXXX to isolate (default: all -O2 jr)")
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
a = ap.parse_args()
|
|
only = set(a.only.split(",")) if a.only else None
|
|
|
|
p = plan(a.ov, only)
|
|
n_jr = sum(len(v) for v in p["per_obj"].values())
|
|
print(f"jr_isolate_all {a.ov}: {n_jr} jr in {len(p['per_obj'])} -O2 objects "
|
|
f"{ {k: len(v) for k, v in p['per_obj'].items()} }")
|
|
if p["skipped_o0"]:
|
|
print(f" SKIPPED {len(p['skipped_o0'])} jr in -O0 objects: {[hex(x) for x in p['skipped_o0']]}")
|
|
if not p["per_obj"]:
|
|
print(" nothing to isolate.")
|
|
return
|
|
|
|
cfg_lines, new_files, carve_renames = build_new_config(a.ov, p)
|
|
# FAIL-LOUD VALIDATION (Phase 26 session 8): the code-subseg list must be strictly ascending
|
|
# with unique names, or splat rejects the split ("segments out of order"). The byte-proven
|
|
# corruption path: a failed bank's revert once left an isolation's config lines in place, the
|
|
# committed config gained a DUPLICATE `- [off, c, name]` line (harmless to splat — zero-length),
|
|
# and the NEXT isolation walked the object twice, emitting a reversed duplicate block. Validate
|
|
# BEFORE writing so a corrupt input dies here, not three tools downstream.
|
|
code_re = re.compile(r'^\s*- \[(0x[0-9A-Fa-f]+), c, (\w+)\]')
|
|
seen_off, seen_nm = -1, set()
|
|
for ln in cfg_lines:
|
|
m = code_re.match(ln)
|
|
if not m:
|
|
continue
|
|
off, nm = int(m.group(1), 16), m.group(2)
|
|
if off <= seen_off or nm in seen_nm:
|
|
sys.exit(f"jr_isolate_all: REFUSING to write a corrupt config — code subseg "
|
|
f"[{hex(off)}, {nm}] is {'out of order' if off <= seen_off else 'a duplicate'} "
|
|
f"(prev off {hex(seen_off)}). The INPUT config likely carries duplicate/stale "
|
|
f"subseg lines from an un-reverted isolation — `git diff config/splat.{a.ov}.yaml` "
|
|
f"and clean it first.")
|
|
seen_off, seen_nm = off, seen_nm | {nm}
|
|
mk_changes = repoint_overlays_mk(carve_renames, dry=True, ov=a.ov, cfg_lines=cfg_lines)
|
|
print(f" -> {len(new_files)} region .c files; carve repoints: {carve_renames or '(none)'}")
|
|
for c in mk_changes:
|
|
print(f" overlays.mk: {c}")
|
|
|
|
if a.dry_run:
|
|
print(" [dry-run] no files written.")
|
|
return
|
|
|
|
# write config, source region files, overlays.mk
|
|
cfg_path = os.path.join(REPO, f"config/splat.{a.ov}.yaml")
|
|
open(cfg_path, "w").write("\n".join(cfg_lines) + "\n")
|
|
# remove the original per-object .c files that were replaced (region 0 rewrites them;
|
|
# extra regions are new — but a stale original with the OLD single-object content would
|
|
# shadow nothing since we overwrite region 0 to the same path). Write all region files:
|
|
for path, content in new_files.items():
|
|
open(path, "w").write(content)
|
|
repoint_overlays_mk(carve_renames, dry=False, ov=a.ov, cfg_lines=cfg_lines)
|
|
print(f" wrote config + {len(new_files)} region files + overlays.mk. Run `make extract "
|
|
f"BINARY={a.ov} && make build BINARY={a.ov}` to byte-gate (R22).")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|