From 2dbe59b61ef2fc988d201b18272e55273e6fec6c Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:49:47 -0600 Subject: [PATCH] phase-36: decision log P36 S104 (Drew's four rulings via sotn, the sweep-before-agent rule, the TU-batch lane, six instrument findings); kit corpus regenerated (tool_census --check OK) --- decomp-architect/corpus/tools/P10/delever.py | 886 +++++++++++++++++- .../corpus/tools/P10/delever_pack.py | 4 +- .../corpus/tools/P10/delever_regen.py | 24 +- .../corpus/tools/P10/delever_search.py | 7 +- .../corpus/tools/P10/lever_census.py | 15 +- .../corpus/tools/P10/localalloc_sim.py | 25 +- .../corpus/tools/P7/alloc_table.py | 9 +- docs/decision-log.md | 39 + 8 files changed, 997 insertions(+), 12 deletions(-) diff --git a/decomp-architect/corpus/tools/P10/delever.py b/decomp-architect/corpus/tools/P10/delever.py index 763744f86..8a6d54bc1 100644 --- a/decomp-architect/corpus/tools/P10/delever.py +++ b/decomp-architect/corpus/tools/P10/delever.py @@ -1506,7 +1506,7 @@ def block_wraps(text, tu, fn, d_): if not simple_stmt(s): continue stmt = raw_line.strip() - for tag, spelling in (("block", f"{indent}{{ {stmt} }}"), ("do-while", f"{indent}do {{ {stmt} }} while (0);")): + for tag, spelling in (("block", f"{indent}{{ {stmt} }}"), ("do-while", f"{indent}do {{ {stmt} }} while (0); // !FAKE: do-while — a LOOP-note scheduling barrier (sched.c:2058-2074; P36 R7)")): cand = list(lines) cand[i] = spelling out.append((f"{tag} @{i + 1}", "\n".join(cand))) @@ -2860,8 +2860,10 @@ def split_reused_locals(text, tu, fn, d_): semi = mb.find(";", p) if semi < 0 or re.search(r"(?])%s\b" % re.escape(v), mb[p + len(v):semi]): return None - if before and not before.endswith((";", "{", "}")): + if before and not before.endswith((";", "{", "}")) and \ + not re.search(r"(?:\bcase\s+[^;:?]+|\bdefault\s*):$", before): return None # inside an expression, or a brace-less `if (c) v = E;` + # (a `case K:` / `default:` label IS a statement boundary — S104 d22: R23 never split func_801861FC's `t`) defs.append(p) if len(defs) < 2 or occ[0] != defs[0]: return None @@ -3096,7 +3098,837 @@ def alias_repeated_addresses(text, tu, fn, d_): return out -ALL_FAMILIES = ("R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", "R10", "R12", "R13", "R14", "R15", "R16", "R17", "R18", "R19", "R20", "R21", "R22", "R23", "R24", "R25", "R26") +def _uses(masked, lo, hi, name): + return sum(len(re.findall(r"\b%s\b" % re.escape(name), masked[i])) for i in range(lo, hi)) + + +def _drop_single_decl(lines, masked, lo, hi, name): + """Delete `name`'s own one-name declaration line (no initializer) once the name has no other use; returns True if done.""" + for i in range(lo, hi): + if lines[i] is None: + continue + if re.match(r"^\s*(?:(?:unsigned|signed|const|volatile|struct|union)\s+)*[A-Za-z_]\w*\s*\**\s*%s\s*;\s*$" % re.escape(name), + masked[i]): + lines[i] = None + return True + return False + + +def shift_operand_casts(text, tu, fn, d_): + """[(description, candidate text)] — R31: `v >> N` → `(s16)v >> N` / `(s8)v >> N`, one shift at a time. + + T7 agent d13 (func_8018F694 ×5, P36 S104): the residual was `lhu; sll 16; sra 16` against the target's `lh`, plus one + shift reading the sll register. cse's associative fold (`cse.c:5577-5667`) had rewritten `t >> 6` as `(sll t) >> 22`, so + the load's temp gained a second reader and combine never formed `lh`. A cast on the SHIFT'S OPERAND makes the front end + shift in `short` (`c-typeck.c:2418-2450`), cse folds that new pair instead, and combine reduces `(t << 16) >> 22` to + `t >> 6` (`combine.c:7930-7944`). A declaration width (R12) cannot do it — `s16 t` scored 5.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") # whole-text: a block comment's inner lines are masked too + lo, hi = d_["line"], d_["end"] - 1 + out = [] + sites = [] + for i in range(lo, hi): + for m in re.finditer(r"(?])([A-Za-z_]\w*)\s*>>\s*(0x[0-9A-Fa-f]+|\d+)", masked[i]): + sites.append((i, m)) + for T in ("s16", "s8"): + cand = list(lines) + l = cand[i] + cand[i] = l[:m.start(1)] + f"({T}){m.group(1)}" + l[m.end(1):] + out.append((f"shift-cast ({T}){m.group(1)} >> {m.group(2)} @{i + 1}", "\n".join(cand))) + if len(sites) > 1: # S104 d39 (func_80185484 ×3): BOTH shifts needed the cast; one site alone scored 5 / 4 + cand = list(lines) + for i, m in sorted(sites, key=lambda t: (t[0], -t[1].start(1))): + cand[i] = cand[i][:m.start(1)] + f"(s16){m.group(1)}" + cand[i][m.end(1):] + out.append((f"shift-cast (s16) at all {len(sites)} shifts", "\n".join(cand))) + return out + + +_RELOP = re.compile(r"^(.*?)\s*(<=|>=|==|!=|<|>)\s*(.*)$") +_INV = {"<": ">=", ">=": "<", ">": "<=", "<=": ">", "==": "!=", "!=": "=="} + + +def _invert(cond): + c = cond.strip() + if any(t in c for t in ("&&", "||", "?")) or len(re.findall(r"<=|>=|==|!=|<|>", c)) != 1: + return None + m = _RELOP.match(c) + return f"{m.group(1)} {_INV[m.group(2)]} {m.group(3)}" if m else None + + +def else_arm_assignments(text, tu, fn, d_): + """[(description, candidate text)] — R33: `x = A; if (C) x = B;` → `if (!C) { x = A; } else { x = B; }`. + + T7 agent d1 (func_80185960 ×10, P36 S104): the one-armed form let cse's skip-block path carry a value past the join + (`cse.c:8101-8106`, `:8149`), so a later test reused it from a register. A plain if/else whose ELSE value is a register or + constant is folded straight back to the one-armed form by jump1 (`jump.c:699-750`, guard `:739-741`) — the non-simple + value must sit in the else arm, so the condition is inverted (the relational operator flipped, or `!(C)`). Never a ternary + (`expr.c:5808-5814` expands it one-armed).""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") # whole-text: a block comment's inner lines are masked too + lo, hi = d_["line"], d_["end"] - 1 + ASSIGN = re.compile(r"^(\s*)([A-Za-z_]\w*)\s*=\s*(.+);\s*$") + out = [] + for i in range(lo, hi - 1): + a = ASSIGN.match(masked[i]) + if not a or not simple_stmt(masked[i]): + continue + ind, x = a.group(1), a.group(2) + A = lines[i][lines[i].index("=") + 1:].rsplit(";", 1)[0].strip() + k = None + m = re.match(r"^\s*if\s*\((.*)\)\s*\{?\s*%s\s*=\s*(.+?);\s*\}?\s*$" % re.escape(x), masked[i + 1]) + if m and masked[i + 1].count("{") == masked[i + 1].count("}"): + k, C, B = i + 1, lines[i + 1][masked[i + 1].index("(") + 1:m.end(1)], lines[i + 1][m.start(2):m.end(2)] + elif i + 3 < hi and re.match(r"^\s*if\s*\((.*)\)\s*\{\s*$", masked[i + 1]) and re.match(r"^\s*\}\s*$", masked[i + 3]): + m2 = re.match(r"^\s*%s\s*=\s*(.+?);\s*$" % re.escape(x), masked[i + 2]) + if m2: + mc = re.match(r"^\s*if\s*\((.*)\)\s*\{\s*$", masked[i + 1]) + k, C, B = i + 3, lines[i + 1][mc.start(1):mc.end(1)], lines[i + 2][m2.start(1):m2.end(1)] + if k is None: + continue + for tag, nc in (("inverted", _invert(C)), ("not", f"!({C.strip()})")): + if not nc: + continue + cand = lines[:i] + [f"{ind}if ({nc}) {{", f"{ind} {x} = {A};", f"{ind}}} else {{", f"{ind} {x} = {B};", + f"{ind}}}"] + lines[k + 1:] + out.append((f"else-arm {x} {tag} @{i + 1}", "\n".join(cand))) + return out + + +def compound_assignments(text, tu, fn, d_): + """[(description, candidate text)] — R32: a single-use temp folded into its consumer. + + * (T7 agent d5, func_801837E8 ×4, P36 S104) `v = E; … L = L + v;` → `L += E;` — the compound form loads its left side + first, so the three quantities of the block are BORN in the target's order; local-alloc's three-quantity "sort" is a + fixed compare sequence on birth order, not a sort (`local-alloc.c:1486-1507`). + * (T7 agent d8, func_80186440 ×4) `v = F + 1; … F = v;` → `(F)++;` — the u16 field increment's destination is a SUBREG, + which fails `birthing_insn_p` (`sched.c:2477-2490`), so sched1 does not pull the add down; `F += 1` folds back to SImode. + `v` must have exactly two mentions besides its declaration; the declaration goes with it.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") # whole-text: a block comment's inner lines are masked too + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi): + a = re.match(r"^(\s*)([A-Za-z_]\w*)\s*=\s*(.+);\s*$", masked[i]) + if not a or not simple_stmt(masked[i]): + continue + v = a.group(2) + decl = re.compile(r"^\s*[A-Za-z_][\w\s]*\**\s*%s\s*(?:=[^;]*)?;" % re.escape(v)) + dls = [j for j in range(lo, hi) if decl.match(masked[j])] + s0 = max([j for j in dls if j < i], default=lo) # the scope: this declaration of v to the next one + s1 = min([j for j in dls if j > i], default=hi) + if _uses(masked, s0, s1, v) - (1 if s0 in dls else 0) != 2: + continue + E = lines[i][lines[i].index("=") + 1:].rsplit(";", 1)[0].strip() + for j in range(i + 1, s1): + if not re.search(r"\b%s\b" % re.escape(v), masked[j]): + continue + ind = lines[j][:len(lines[j]) - len(lines[j].lstrip())] + cand, op = None, None + m1 = re.match(r"^\s*(.+?)\s*=\s*(.+?)\s*([-+|&^])\s*%s\s*;\s*$" % re.escape(v), masked[j]) # L = L op v + m2 = re.match(r"^\s*(.+?)\s*=\s*%s\s*([+|&^])\s*(.+?)\s*;\s*$" % re.escape(v), masked[j]) # L = v op L + ns = lambda s: re.sub(r"\s", "", s) + if m1 and ns(m1.group(1)) == ns(m1.group(2)): + op = m1.group(3) + elif m2 and ns(m2.group(1)) == ns(m2.group(3)): + op = m2.group(2) + if op: + L = lines[j][:lines[j].index("=")].strip() + cand = list(lines) + cand[j], cand[i] = f"{ind}{L} {op}= {E};", None + tag = f"compound {L} {op}= @{j + 1}" + ms = re.match(r"^\s*(.+?)\s*=\s*%s\s*;\s*$" % re.escape(v), masked[j]) + mf = re.match(r"^(.+?)\s*([-+])\s*1\s*$", E) + if cand is None and ms and mf and re.sub(r"\s", "", ms.group(1)) == re.sub(r"\s", "", mf.group(1)): + F = lines[j][:lines[j].index("=")].strip() + cand = list(lines) + cand[j], cand[i] = f"{ind}({F}){mf.group(2) * 2};", None + tag = f"increment ({F}){mf.group(2) * 2} @{j + 1}" + if cand is not None: + cm = [sc.mask_text(l) if l is not None else "" for l in cand] + _drop_single_decl(cand, cm, s0, s1, v) # THIS scope's declaration, not the first in the body + out.append((tag, "\n".join(l for l in cand if l is not None))) + break + return out + + +def fold_store_temps(text, tu, fn, d_): + """[(description, candidate text)] — R29: a temp REUSED for several values, each stored once, written as direct stores. + + T7 agent d15 (func_8018594C ×4, P36 S104): `v1 = K; *(u16 *)(s0 + off) = v1; … v1 = K2; …` — one pseudo that "dies in 3 + places" (`.lreg`), which local-alloc refuses (`local-alloc.c:472`), so global gave it the target's other register. Each + `t = E; = t;` pair becomes ` = E;` and `t = ; t |= K; = t;` becomes ` |= K;` + — every pair of one temp at once (the reuse is the defect), then the temp's declaration if it is left unused.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") # whole-text: a block comment's inner lines are masked too + lo, hi = d_["line"], d_["end"] - 1 + pairs = collections.defaultdict(list) # t -> [(first line, last line, {line: new text or None})] + LV = r"(.*[*\[\]>].*?)" + for i in range(lo, hi - 1): + a = re.match(r"^(\s*)([A-Za-z_]\w*)\s*=\s*(.+);\s*$", masked[i]) + if not a or not simple_stmt(masked[i]): + continue + t = a.group(2) + E = lines[i][lines[i].index("=") + 1:].rsplit(";", 1)[0].strip() + k, edits = i + 1, {i: None} + while k < hi and re.match(r"^\s*%s\s*=\s*%s\s*;\s*$" % (LV, re.escape(t)), masked[k]): # `LV = t;` run + edits[k] = f"{lines[k][:lines[k].rindex('=')].rstrip()} = {E};" + k += 1 + if len(edits) > 1: + pairs[t].append((i, k - 1, edits)) + continue + # `t = LV; … t OP= K; … LV = t;` within four lines, the in-between lines not mentioning t + for k in range(i + 1, min(i + 5, hi)): + o = re.match(r"^\s*%s\s*([|&^+-])=\s*(.+);\s*$" % re.escape(t), masked[k]) + if o: + break + if re.search(r"\b%s\b" % re.escape(t), masked[k]): + o = None + break + else: + o = None + if not o: + continue + for s_ in range(k + 1, min(k + 4, hi)): + st = re.match(r"^(\s*)(.+?)\s*=\s*%s\s*;\s*$" % re.escape(t), masked[s_]) + if st and re.sub(r"\s", "", st.group(2)) == re.sub(r"\s", "", E): + K = lines[k][lines[k].index("=") + 1:].rsplit(";", 1)[0].strip() + pairs[t].append((i, s_, {i: None, k: None, s_: f"{st.group(1)}{E} {o.group(1)}= {K};"})) + break + if re.search(r"\b%s\b" % re.escape(t), masked[s_]): + break + out = [] + for t, ps in pairs.items(): + if len(ps) < 2 and _uses(masked, lo, hi, t) <= 3: + continue + cand = list(lines) + for _i, _k, edits in ps: + for x, new in edits.items(): + cand[x] = new + cm = [sc.mask_text(l) if l is not None else "" for l in cand] + if _uses(cm, lo, hi, t) == 1: + _drop_single_decl(cand, cm, lo, hi, t) + out.append((f"fold-stores {t} ×{len(ps)}", "\n".join(l for l in cand if l is not None))) + return out + + +def merge_disjoint_locals(text, tu, fn, d_, max_pairs=40): + """[(description, candidate text)] — R34: two same-type locals whose live ranges do not overlap, merged into one. + + Three T7 closes in one session were this move (P36 S104): d12 (func_801898E4 ×4 — `count`/`descCount`, `table`/`table2`), + d14 (func_801860B8 ×3 — late temps reusing earlier-dead variables so they land in those variables' registers) and d19 + (func_80189030 ×3 — a search loop's index renamed to the counter that lost a `$s0`/`$s1` race). One pseudo with the + combined refs and a longer live range is ranked differently by `allocno_compare` (`global.c:587-610`) and conflicts with + the registers that push it into the target's (`find_reg`, `global.c:945-966`); a block-local pseudo merged into a global + one also leaves local-alloc (`local-alloc.c:1845`). The inverse (a split) is R23. Pairs: every one-name declaration pair + of the same type text whose textual mention spans are disjoint (the earlier's last mention before the later's first); + the later name is renamed to the earlier and its declaration dropped. Textual disjointness inside a loop is not + liveness — the byte oracle judges every candidate.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + decls = {} + for i in range(lo, hi): + m = re.match(r"^\s*((?:(?:unsigned|signed|const)\s+)*[A-Za-z_]\w*\s*\**)\s*([A-Za-z_]\w*)\s*;\s*$", masked[i]) + if m and m.group(1).strip() not in ("return", "goto", "break", "continue") and m.group(2) not in decls: + decls[m.group(2)] = (i, re.sub(r"\s+", " ", m.group(1)).strip()) + span = {} + for n, (di, _t) in decls.items(): + hits = [i for i in range(lo, hi) if i != di and re.search(r"\b%s\b" % re.escape(n), masked[i])] + if hits: + span[n] = (hits[0], hits[-1]) + out = [] + names = [n for n in decls if n in span] + for a in names: + for b in names: + if a == b or decls[a][1] != decls[b][1] or not span[a][1] < span[b][0]: + continue + if _loop_between(masked, span[a][0], span[b][1]): + pass # still a candidate: the oracle decides + cand = list(lines) + cand[decls[b][0]] = None + cand = [re.sub(r"\b%s\b" % re.escape(b), a, l) if l is not None else None for l in cand] + out.append((f"merge-disjoint {b}->{a}", "\n".join(l for l in cand if l is not None))) + if len(out) >= max_pairs: + return out + return out + + +def _loop_between(masked, i, j): + return any(re.match(r"^\s*(?:for|while|do)\b", masked[k]) for k in range(i, j + 1)) + + +def drop_param_copies(text, tu, fn, d_): + """[(description, candidate text)] — R35: a parameter copy `T x = argN;` (or `x = argN;` as the first use) deleted and + `argN` used everywhere instead. + + T7 agents d24 (func_8018003C ×4) and d17 (func_80181DAC ×4), P36 S104: a copy of a parameter that lives past the + parameter's last use becomes the canonical register in cse (`make_regs_eqv`, `cse.c:846-862`), which re-routes later + reads through it — two callee-saved registers where the target has one, and tails that cross-jump could have merged now + load `$a0` differently (`jump.c:2371`). Only a copy that is never reassigned, of a parameter never reassigned after it.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + head = " ".join(masked[d_["line"] - 1:lo + 2]) + pm = re.search(r"\b%s\s*\(([^)]*)\)" % re.escape(fn), head) + if not pm: + return [] + params = [re.findall(r"([A-Za-z_]\w*)\s*(?:\[[^\]]*\])?\s*$", p_.strip())[0] for p_ in pm.group(1).split(",") + if re.findall(r"([A-Za-z_]\w*)\s*$", p_.strip()) and p_.strip() not in ("void", "")] + out = [] + for i in range(lo, hi): + m = re.match(r"^\s*(?:(?:[A-Za-z_]\w*\s*\**\s+)+)?\**\s*([A-Za-z_]\w*)\s*=\s*(?:\([^()]*\)\s*)?([A-Za-z_]\w*)\s*;\s*$", + masked[i]) + if not m or m.group(2) not in params or m.group(1) in params: + continue + x, a = m.group(1), m.group(2) + cm_ = re.search(r"=\s*(\([^()]*\))\s*%s\s*;" % re.escape(a), masked[i]) + repl = f"({cm_.group(1)}{a})" if cm_ else a # S104 d38: a CAST copy `T *p = (T *)a1;` → `((T *)a1)` at each use + body = "\n".join(masked[i + 1:hi]) + assign = r"(?])\b%s\s*(?:[-+*/%%&|^]|<<|>>)?=(?!=)|(?:\+\+|--)\s*%s\b|\b%s\s*(?:\+\+|--)" + if re.search(assign % ((re.escape(x),) * 3), body) or re.search(assign % ((re.escape(a),) * 3), body): + continue + cand = list(lines) + is_decl = bool(re.match(r"^\s*[A-Za-z_]\w*[\w\s]*\**\s*%s\s*=" % re.escape(x), masked[i])) and \ + not re.match(r"^\s*%s\s*=" % re.escape(x), masked[i]) + cand[i] = None + if not is_decl: + cm = [sc.mask_text(l) if l is not None else "" for l in cand] + _drop_single_decl(cand, cm, lo, hi, x) + cand = [re.sub(r"\b%s\b" % re.escape(x), lambda _m: repl, l) if l is not None and k > i else l for k, l in enumerate(cand)] + out.append((f"drop-param-copy {x}->{repl} @{i + 1}", "\n".join(l for l in cand if l is not None))) + return out + + +def merge_set_chains(text, tu, fn, d_): + """[(description, candidate text)] — R36: a local set twice in a row, `x = A; x += B;` / `x = A; x = x + B;`, written as + one assignment `x = A + B;` (the operator kept). + + T7 agent d25 (func_8017E060 ×3, P36 S104) and S103 c35: combine folds every use of such a pseudo into its consumers but + zeroes its ref count only when its set count reaches 0 (`combine.c:2305-2337`; `i2dest_in_i2src` skips the i2 update, + `:1394`), so the dead pseudo keeps refs, gets no register, and reload hands it a stack slot (`reload1.c:2327-2352`) — a + FRAME-ONLY residual: every instruction equal, the frame 8 bytes larger.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi - 1): + a = re.match(r"^(\s*)([A-Za-z_]\w*)\s*=\s*(.+);\s*$", masked[i]) + if not a or not simple_stmt(masked[i]): + continue + x = a.group(2) + A = lines[i][lines[i].index("=") + 1:].rsplit(";", 1)[0].strip() + for j in range(i + 1, min(i + 4, hi)): # up to two lines between that do not mention x (d25: `new_var = r;`) + m1 = re.match(r"^\s*%s\s*([-+|&^])=\s*(.+);\s*$" % re.escape(x), masked[j]) + m2 = re.match(r"^\s*%s\s*=\s*%s\s*([-+|&^])\s*(.+);\s*$" % (re.escape(x), re.escape(x)), masked[j]) + m = m1 or m2 + if m or re.search(r"\b%s\b" % re.escape(x), masked[j]) or not simple_stmt(masked[j]): + break + if not m or re.search(r"\b%s\b" % re.escape(x), m.group(2)): + continue + B = lines[j][m.start(2):m.end(2)] + cand = list(lines) + cand[i], cand[j] = None, f"{a.group(1)}{x} = {A} {m.group(1)} {B};" + out.append((f"merge-set-chain {x} @{i + 1}", "\n".join(l for l in cand if l is not None))) + return out + + +def shift_to_division(text, tu, fn, d_): + """[(description, candidate text)] — R38: the hand-expanded signed division `if (v < 0) v += 2^k-1; v = v >> k;` written + as the division it is, `v = v / 2^k;` (and, when the line before assigns `v = E;`, `v = (E) / 2^k;`). + + T7 agent e7 (func_801831FC, P36 S104): the decompiler printed gcc's own expansion of `x / 0x800`; compiled as that C, + the shift's operand keeps a single preference, while a real division expands through `expand_divmod` into a block-local + quotient in `$v0` whose `set_preference` (`global.c:1535/1545`) gives the dividend the target's register. The byte + oracle judges every candidate.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi): + m = re.match(r"^(\s*)if\s*\(\s*([A-Za-z_]\w*)\s*<\s*0\s*\)\s*(\{)?\s*(?:\2\s*\+=\s*|\2\s*=\s*\2\s*\+\s*)(0x[0-9A-Fa-f]+|\d+)\s*;\s*(\})?\s*$", + masked[i]) + j = i + if m: + ind, v, K = m.group(1), m.group(2), int(m.group(4), 0) + if m.group(3) and not m.group(5): + continue + else: + m = re.match(r"^(\s*)if\s*\(\s*([A-Za-z_]\w*)\s*<\s*0\s*\)\s*\{\s*$", masked[i]) + if not m or i + 2 >= hi: + continue + ind, v = m.group(1), m.group(2) + b = re.match(r"^\s*(?:%s\s*\+=\s*|%s\s*=\s*%s\s*\+\s*)(0x[0-9A-Fa-f]+|\d+)\s*;\s*$" % ((re.escape(v),) * 3), masked[i + 1]) + if not b or not re.match(r"^\s*\}\s*$", masked[i + 2]): + continue + K, j = int(b.group(1), 0), i + 2 + N = K + 1 + if N & K or N < 2 or j + 1 >= hi: + continue + k = N.bit_length() - 1 + s = re.match(r"^\s*(?:%s\s*>>=\s*|%s\s*=\s*%s\s*>>\s*)(0x[0-9A-Fa-f]+|\d+)\s*;\s*$" % ((re.escape(v),) * 3), masked[j + 1]) + if not s or int(s.group(1), 0) != k: + continue + Nh = f"0x{N:X}" + cand = lines[:i] + [f"{ind}{v} = {v} / {Nh};"] + lines[j + 2:] + out.append((f"shift-to-division {v} / {Nh} @{i + 1}", "\n".join(cand))) + a = re.match(r"^(\s*)%s\s*=\s*(.+);\s*$" % re.escape(v), masked[i - 1]) if i - 1 >= lo else None + if a and not re.search(r"\b%s\b" % re.escape(v), a.group(2)): + E = lines[i - 1][lines[i - 1].index("=") + 1:].rsplit(";", 1)[0].strip() + cand = lines[:i - 1] + [f"{ind}{v} = ({E}) / {Nh};"] + lines[j + 2:] + out.append((f"shift-to-division {v} = (E) / {Nh} @{i}", "\n".join(cand))) + return out + + +def duplicate_join_statement(text, tu, fn, d_, max_sites=12): + """[(description, candidate text)] — R39: the single simple statement right after an if/else's closing brace copied to + the end of BOTH arms (and deleted after the join), one site at a time. + + T7 agents e12 (func_8017E35C) and e14 (func_8017BEBC, func_8017CAD4), P36 S104: `global.c:594-603` truncates allocno + priorities to int, so two loop-live pseudos tie (245/245) and the lower allocno takes the wrong register. One extra insn + inside the loop lengthens every loop-live pseudo by one (flow.c:1660-1684) and splits the tie (244 vs 245); post-reload + cross-jump merges the two copies back into one (`toplev.c:3142`, `jump.c:2371`), so the bytes keep a single store. + `tools/alloc_table.py` flags such ties as TIE.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for c in range(lo, hi - 1): + if not re.match(r"^\s*\}\s*$", masked[c]): + continue + k = c + 1 + while k < hi and not masked[k].strip(): # blank lines between the join and the statement (S104 e24) + k += 1 + if k >= hi or not simple_stmt(masked[k]) or is_decl_line(masked[k].strip()): + continue + # find the matching `if (…) {` … `} else {` … `}` that closes at c + depth, j, else_at = 0, c, None + while j >= lo: + depth += masked[j].count("}") - masked[j].count("{") + if re.match(r"^\s*\}\s*else\s*\{\s*$", masked[j]) and depth == 1: + else_at = j + if depth == 0: + break + j -= 1 + if else_at is None or j < lo or not re.match(r"^\s*if\s*\(", masked[j]): + continue + ind = lines[else_at][:len(lines[else_at]) - len(lines[else_at].lstrip())] + " " + # the first 1..3 simple statements after the join (S104 e24 func_80180E24: a STORE PAIR had to move together) + n = 0 + while n < 3 and k + n < hi and simple_stmt(masked[k + n]) and not is_decl_line(masked[k + n].strip()) \ + and not re.match(r"^\s*(?:return|goto|break|continue)\b", masked[k + n]): + n += 1 + stmts = [lines[x].strip() for x in range(k, k + n)] + cand = (lines[:else_at] + [ind + t for t in stmts] + [lines[else_at]] + lines[else_at + 1:c] + + [ind + t for t in stmts] + [lines[c]] + lines[c + 1:k] + lines[k + n:]) + out.append((f"dup-join ×{n} {stmts[0][:24]} @{k + 1}", "\n".join(cand))) + if len(out) >= max_sites: + break + return out + + +def return_preincrement(text, tu, fn, d_): + """[(description, candidate text)] — R40: `return x + 1;` (x a local) → `return ++x;`. + + T7 agents e2 (func_800348A8) and e16 (func_800331D4), P36 S104: a loop counter whose only exit use is `return i + 1` + loses an `allocno_compare` race (`global.c:587-603`, refs weighted by loop depth `flow.c:2067`) to a loop pointer; the + pre-increment adds refs, and combine folds `i = i + 1; $v0 = i` back into one `addiu` — zero bytes.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi): + m = re.match(r"^(\s*)return\s+([A-Za-z_]\w*)\s*([-+])\s*1\s*;\s*$", masked[i]) + if m: + cand = list(lines) + cand[i] = f"{m.group(1)}return {m.group(3) * 2}{m.group(2)};" + out.append((f"return-preinc {m.group(3) * 2}{m.group(2)} @{i + 1}", "\n".join(cand))) + return out + + +def swap_if_else_arms(text, tu, fn, d_, max_sites=12): + """[(description, candidate text)] — R41: an `if (C) { A } else { B }` rewritten `if (!(C)) { B } else { A }`, one site + at a time. + + T7 agent e16 (func_800336A8, P36 S104): the arm ORDER decides which block falls through and which one reorg's delay-slot + filler can steal from (`update_block` reorg.c:2233, `mark_target_live_regs` :2696-2704, `fill_eager_delay_slots` :3368); + swapping the arms closed a barrier class the generators never reached (they never swap arms). One compile per site.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi): + m = re.match(r"^(\s*)if\s*\((.*)\)\s*\{\s*$", masked[i]) + if not m or masked[i].count("(") != masked[i].count(")"): + continue + depth, j, e = 0, i, None + while j < hi: + depth += masked[j].count("{") - masked[j].count("}") + if depth == 1 and j > i and re.match(r"^\s*\}\s*else\s*\{\s*$", masked[j]): + e = j + if depth == 0 and j > i: + break + j += 1 + if e is None or j >= hi or not re.match(r"^\s*\}\s*$", masked[j]): + continue + ind = m.group(1) + C = lines[i][masked[i].index("(") + 1:m.end(2)] + nc = _invert(C) or f"!({C.strip()})" + cand = lines[:i] + [f"{ind}if ({nc}) {{"] + lines[e + 1:j] + [f"{ind}}} else {{"] + lines[i + 1:e] + [f"{ind}}}"] + lines[j + 1:] + out.append((f"swap-arms @{i + 1}", "\n".join(cand))) + if len(out) >= max_sites: + break + return out + + +def move_statement_far(text, tu, fn, d_, max_dist=6, cap=120): + """[(description, candidate text)] — R42: one simple statement moved DOWN past 2..max_dist following simple statements of + the same block (R9 only exchanges neighbours). + + T7 agent e19 (func_8018230C, P36 S104): `w = D + D * c;` moved below four `base[]` statements let sched1 put its `addu` + inside z's range (26..30 → 24..30), dropping z below base[1] in `qty_compare_1` (`local-alloc.c:1598`); the header's + "@stuck: every source-order permutation was INERT" had only permuted neighbours. e19 counted this gap in three of its + four closes. The byte oracle judges every candidate (a move past a dependent statement changes the program and simply + does not score).""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + ok = lambda i: simple_stmt(masked[i]) and not is_decl_line(masked[i].strip()) and not re.match( + r"^\s*(?:return|goto|break|continue|case|default)\b", masked[i]) and masked[i].strip().endswith(";") + out = [] + for i in range(lo, hi): + if not ok(i): + continue + j = i + 1 + while j < hi and j - i <= max_dist and ok(j): + if j - i >= 2: + cand = lines[:i] + lines[i + 1:j + 1] + [lines[i]] + lines[j + 1:] + out.append((f"move-far @{i + 1} below @{j + 1}", "\n".join(cand))) + if len(out) >= cap: + return out + j += 1 + return out + + +def sign_test_to_mask(text, tu, fn, d_): + """[(description, candidate text)] — R43: `if (E < 0)` / `if (E >= 0)` whose arms set or clear bit 31 (`0x80000000` / + `0x7FFFFFFF` within the next lines) rewritten as a mask test `if ((u32)(E) & 0x80000000)` (resp. `!(…)`). + + T7 agents e24 (func_8017F694) and e26 (func_8017F438, func_8017F600 + seven siblings), P36 S104: the mask is loaded BEFORE + the branch as the AND's operand, cse hands the arm's `|= 0x80000000` the same register, combine still makes `bgez`, and + reorg's `fill_simple_delay_slots` moves the `lui` into the delay slot (`reorg.c:2799ff`); the `< 0` spelling lets + `mostly_true_jump` fill the slot from the other arm instead (`reorg.c:1335-1420`). combine then leaves a `(use)` of the + dead AND whose pseudo reload gives a stack slot (`combine.c:10831-10845`) — what the trees' dead pads were faking.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for i in range(lo, hi): + m = re.match(r"^(\s*(?:\}\s*else\s+)?if\s*\()(.+?)\s*(<|>=)\s*0\s*(\)\s*\{?\s*)$", masked[i]) + if not m or masked[i].count("(") != masked[i].count(")"): + continue + near = "\n".join(masked[i:min(hi, i + 8)]) + if "0x80000000" not in near and "0x7FFFFFFF" not in near and "0x7fffffff" not in near: + continue + E = lines[i][m.start(2):m.end(2)].strip() + test = f"(u32)({E}) & 0x80000000" if m.group(3) == "<" else f"!((u32)({E}) & 0x80000000)" + cand = list(lines) + cand[i] = lines[i][:m.start(1)] + m.group(1) + test + m.group(4) + out.append((f"sign-to-mask @{i + 1}", "\n".join(cand))) + # the mask test creates the dead-AND slot a tree pad was faking (e24/e26 both deleted the pad): the combination + nopad = _drop_dead_pads(cand, lo, hi) + if nopad is not None: + out.append((f"sign-to-mask @{i + 1} + dead pad dropped", "\n".join(l for l in nopad if l is not None))) + return out + + +def _drop_dead_pads(lines, lo, hi): + """lines with every never-used (or only `(void)&x;`-used) array local deleted; None if there is none.""" + masked = [sc.mask_text(l) for l in lines] + body = "\n".join(masked[lo:hi]) + cand, hit = list(lines), False + for i in range(lo, hi): + m = re.match(r"^\s*[A-Za-z_][\w\s]*\s+([A-Za-z_]\w*)\s*\[[^\]]*\]\s*;\s*$", masked[i]) + if not m: + continue + n = m.group(1) + uses = [j for j in range(lo, hi) if j != i and re.search(r"\b%s\b" % re.escape(n), masked[j])] + if all(re.match(r"^\s*\(void\)\s*&?\s*%s\s*;\s*$" % re.escape(n), masked[j]) for j in uses): + for j in [i] + uses: + cand[j] = None + hit = True + return cand if hit else None + + +def return_constants(text, tu, fn, d_): + """[(description, candidate text)] — R37: a result local `r = 0; if (A) r = (B); return r;` (or with `{ }`) written as + `if (A && B) return 1; return 0;` — and the nested form `if (A) { if (B) return 1; } return 0;`. + + T7 agent d27 (func_80184B94 + three copies + the shared header func_8013E448.h ×141, P36 S104): the result pseudo + `r = 0` was hoisted by sched1 above the call-result copy and took `$a1`, costing a final `move v0,a1`. With constant + returns jump1's store-flag works on the hard `$v0` (`jump.c:1140-1210`, the `x = b; if (…) x = a` hoist `:700-760`), + which sched1 must keep after the copy. Only when `r` has exactly these three mentions; its declaration goes.""" + lines = text.split("\n") + masked = sc.mask_text(text).split("\n") + lo, hi = d_["line"], d_["end"] - 1 + out = [] + for k in range(lo, hi): + rm = re.match(r"^(\s*)return\s+([A-Za-z_]\w*)\s*;\s*$", masked[k]) + if not rm: + continue + ind, r = rm.group(1), rm.group(2) + hits = [i for i in range(lo, hi) if re.search(r"\b%s\b" % re.escape(r), masked[i])] + decl = [i for i in hits if re.match(r"^\s*[A-Za-z_][\w\s]*\**\s*%s\s*;\s*$" % re.escape(r), masked[i])] + use = [i for i in hits if i not in decl] + z = next((i for i in use if re.match(r"^\s*%s\s*=\s*0\s*;\s*$" % re.escape(r), masked[i])), None) + if z is None: + continue + rest = [i for i in use if i not in (z, k)] + # `if (A) r = (B);` on one line, or `if (A) {` / `r = (B);` / `}` + span = None + if len(rest) == 1: + i = rest[0] + m1 = re.match(r"^\s*if\s*\((.*)\)\s*\{?\s*%s\s*=\s*(.+?);\s*\}?\s*$" % re.escape(r), masked[i]) + if m1 and masked[i].count("(") == masked[i].count(")"): + span = (i, i, lines[i][masked[i].index("(") + 1:m1.end(1)], lines[i][m1.start(2):m1.end(2)]) + elif i >= 1 and re.match(r"^\s*if\s*\((.*)\)\s*\{\s*$", masked[i - 1]) and re.match(r"^\s*\}\s*$", masked[i + 1]): + m2 = re.match(r"^\s*%s\s*=\s*(.+?);\s*$" % re.escape(r), masked[i]) + mc = re.match(r"^\s*if\s*\((.*)\)\s*\{\s*$", masked[i - 1]) + if m2 and mc: + span = (i - 1, i + 1, lines[i - 1][mc.start(1):mc.end(1)], lines[i][m2.start(1):m2.end(1)]) + if not span or not (z < span[0] and span[1] < k): + continue + a, b = span[2].strip(), span[3].strip() + if b.startswith("(") and b.endswith(")"): + b = b[1:-1].strip() + for tag, new in (("and", [f"{ind}if (({a}) && ({b})) {{", f"{ind} return 1;", f"{ind}}}", f"{ind}return 0;"]), + ("nested", [f"{ind}if ({a}) {{", f"{ind} if ({b}) {{", f"{ind} return 1;", f"{ind} }}", + f"{ind}}}", f"{ind}return 0;"])): + cand = list(lines) + for x in [z] + list(range(span[0], span[1] + 1)) + decl: + cand[x] = None + cand[k] = "\n".join(new) + out.append((f"return-constants {r} {tag} @{k + 1}", "\n".join(l for l in cand if l is not None))) + return out + + +def merge_pinned_twins(tu, fn, free_text): + """[(description, candidate text)] — R28: locals the TREE pins to the same hard register, merged into one variable. + + T7 agent d12 (func_801898E4 ×4, P36 S104): the tree pinned `count`/`descCount` to one register and `table`/`table2` to + another — each pair was ONE original variable the decompiler split. Merged, the variable spans both phases and conflicts + with the registers that push it into the target's (`find_reg`, `global.c:945-966`). Reads the tree body for the pins + (the start text has them stripped); renames the later names to the first in the start text and drops their one-name + declarations — every group at once, then each group alone.""" + raw = (REPO / tu).read_text(errors="surrogateescape") + tree = _fn_text(raw, tu, fn) + if not tree: + return [] + groups = collections.defaultdict(list) + for m in re.finditer(r"register\s+[^;=()]*?\b([A-Za-z_]\w*)\s*__asm__\s*\(\s*\"\$(\d+)\"\s*\)", tree): + if m.group(1) not in groups[m.group(2)]: + groups[m.group(2)].append(m.group(1)) + gs = [(r, ns) for r, ns in groups.items() if len(ns) >= 2 and r != "0"] + if not gs: + return [] + d_ = sc_body_span(free_text, "src/fx/regen.c", fn) + if not d_: + return [] + + def apply(sel): + lines = free_text.split("\n") + for _r, ns in sel: + keep = ns[0] + for n in ns[1:]: + masked = [sc.mask_text(l) if l is not None else "" for l in lines] + if not _drop_single_decl(lines, masked, d_["line"], d_["end"] - 1, n): + return None + lines = [re.sub(r"\b%s\b" % re.escape(n), keep, l) if l is not None else None for l in lines] + return "\n".join(l for l in lines if l is not None) + out = [] + for tag, sel in ([("all", gs)] if len(gs) > 1 else []) + [(f"${r}", [(r, ns)]) for r, ns in gs]: + c = apply(sel) + if c and c != free_text: + out.append((f"merge-pinned {tag} " + ";".join("=".join(ns) for _r, ns in sel), c)) + return out + + +_NAMED_DEFS = None +_SYM = r"(?:D|g)_[0-9A-Fa-f]{8}" + + +def named_definitions(): + """{name: [(file, line)]} — every definition-looking line of a `func_XXXXXXXX` in src/ (.c and shared .h), cached once.""" + global _NAMED_DEFS + if _NAMED_DEFS is None: + r = subprocess.run(["git", "grep", "-nE", r"^[A-Za-z_][^;]*\bfunc_[0-9A-Fa-f]{8}(_body)?[[:space:]]*\([^;]*$", "--", "src/*.c", "src/*.h"], + cwd=REPO, capture_output=True, text=True, errors="surrogateescape") + idx = collections.defaultdict(list) + for ln in r.stdout.splitlines(): + f, n, t = ln.split(":", 2) + for m in re.finditer(r"\b(func_[0-9A-Fa-f]{8})(?:_body)?\s*\(", t): # `func_X_body(` = an asm-label definition (S104) + idx[m.group(1)].append((f, int(n))) + break + _NAMED_DEFS = dict(idx) + return _NAMED_DEFS + + +def _fn_text(raw, rel, fn): + d_ = sc_body_span(raw, rel, fn) + if not d_: + return None + ls = line_starts(raw) + return raw[ls[d_["line"] - 1]:ls[d_["end"]]] if d_["end"] < len(ls) else raw[ls[d_["line"] - 1]:] + + +def sc_body_span(text, rel, fn): + return next((r for r in sc.scan_text(text, rel, shared_defs=None) if r["form"] == "def" and r["name"] == fn), None) + + +def _uniq(seq): + seen, out = set(), [] + for x in seq: + if x not in seen: + seen.add(x) + out.append(x) + return out + + +def _obj_of(rel): + """The original-bytes object that carries `rel`'s code: a .c file's own object (snapshot first); a shared header's + first includer's.""" + if rel.endswith(".h"): + r = subprocess.run(["git", "grep", "-l", "-F", pathlib.Path(rel).name, "--", "src/*.c"], cwd=REPO, + capture_output=True, text=True) + inc = r.stdout.split() + if not inc: + return None + rel = inc[0] + p = oracle.baseline_path("build/" + rel[:-2] + ".o") + return p if p.exists() else None + + +def fn_relocs(obj, fn): + """[symbol] — the relocation targets of `fn` in `obj`, in address order (objdump -dr).""" + r = subprocess.run(["mipsel-linux-gnu-objdump", "-dr", "--no-show-raw-insn", str(obj)], capture_output=True, text=True) + out, inside = [], False + for ln in r.stdout.splitlines(): + m = re.match(r"^[0-9a-f]+ <([^>]+)>:$", ln) + if m: + inside = m.group(1) == fn + continue + if inside: + m = re.search(r"\bR_MIPS_\w+\s+(\S+)", ln) + if m: + out.append(m.group(1).split("+")[0]) + return out + + +def reloc_map(donor_rel, tu, fn): + """{donor symbol: target symbol} — the two ORIGINAL objects' relocation sequences for `fn` paired in order (the same + function at the same address in two binaries: same instructions, per-binary data symbols). None when the sequences do + not align (different lengths) or one donor symbol would map to two targets.""" + a, b = _obj_of(donor_rel), _obj_of(tu) + if not a or not b: + return None + ra, rb = fn_relocs(a, fn), fn_relocs(b, fn) + if not ra or len(ra) != len(rb): + return None + m = {} + for x, y in zip(ra, rb): + if not re.match(r"(?:D|g|func)_[0-9A-Fa-f]{8}$", x) or not re.match(r"(?:D|g|func)_[0-9A-Fa-f]{8}$", y): + continue + if m.setdefault(x, y) != y: + return None + return {x: y for x, y in m.items() if x != y} + + +def _carry_decls(ported, m, donor_raw, target_raw): + """Body-local copies of the donor TU's file-scope `extern` declarations for every renamed symbol the target TU does not + declare anywhere (a donor symbol declared at file scope is 'undeclared' in the target — d6's COMPILE-ERROR).""" + add = [] + for x, y in m.items(): + if not y.startswith(("D_", "g_")) or re.search(r"\b%s\b" % re.escape(y), target_raw): + continue + if re.search(r"extern\b[^;]*\b%s\b" % re.escape(y), ported): + continue + dm = re.search(r"^extern\b[^;\n]*\b%s\b[^;\n]*;" % re.escape(x), donor_raw, re.M) + if dm: + add.append(" " + re.sub(r"\b%s\b" % re.escape(x), y, dm.group(0))) + if not add: + return ported + i = ported.index("{") + 1 + return ported[:i] + "\n" + "\n".join(add) + ported[i:] + + +def named_ports(tu, fn, max_donors=6): + """[(description, candidate text)] — R27: the SAME function already lever-free in another binary, ported with its + symbols renamed onto this binary's. + + T7 agents d2 (func_80166F58, ov_MAIN_012 ×6 from ov_SC04_011's shared header) and d6 (func_8017B614, ov_SC07_010 ×5 + from ov_SC01_000), P36 S104: both closed on their FIRST `--try` by porting a banked lever-free variant — the overlays + carry one engine function at one address with per-overlay data symbols. The sweep had spent 1,699 compiles on d6's + class without getting below 13. The port needs two repairs, both mechanical: (1) the data symbols renamed — pairing + the two bodies' `extern` declaration lists by POSITION (d6's port.py) or their first occurrences in order; (2) the + return type taken from the TARGET, whose TU declares the function again later (`extern void …` vs a donor's `s32`: + "conflicting types", d2). Donors: every definition of `fn` in src/ with no `register`/`__asm__`/`!FAKE`, nearest line + count first. Every candidate is judged on the bytes; a wrong pairing just does not score.""" + raw_t = (REPO / tu).read_text(errors="surrogateescape") + target = _fn_text(raw_t, tu, fn) + if not target: + return [] + t_head = target[:target.index("{")] if "{" in target else "" + t_ext = _uniq(re.findall(r"extern\b[^;]*?\b(%s)\b" % _SYM, target)) + t_occ = _uniq(re.findall(r"\b(%s)\b" % _SYM, target)) + donors, seen = [], set() + for f, _n in named_definitions().get(fn, ()): + if f == tu: + continue + try: + body = _fn_text((REPO / f).read_text(errors="surrogateescape"), f, fn) + except (OSError, ValueError): + continue + if not body or re.search(r"__asm__|\bregister\b|!FAKE", body): + continue + key = re.sub(r"\s+", " ", re.sub(r"\b%s\b" % _SYM, "D", body)) + if key in seen: + continue + seen.add(key) + donors.append((abs(body.count("\n") - target.count("\n")), f, body)) + out = [] + for _d, f, body in sorted(donors)[:max_donors]: + d_ext = _uniq(re.findall(r"extern\b[^;]*?\b(%s)\b" % _SYM, body)) + d_occ = _uniq(re.findall(r"\b(%s)\b" % _SYM, body)) + maps = [] + rm = reloc_map(f, tu, fn) + if rm is not None: + maps.append(("reloc", rm)) + maps.append(("same", {})) + for tag, a, b in (("extern-order", d_ext, t_ext), ("first-occurrence", d_occ, t_occ)): + if a and len(a) == len(b): + m = {x: y for x, y in zip(a, b) if x != y} + if len(set(m.values())) == len(m): + maps.append((tag, m)) + donor_raw = (REPO / f).read_text(errors="surrogateescape") + for tag, m in maps: + ported = re.sub(r"\b(?:D|g|func)_[0-9A-Fa-f]{8}\b", lambda x: m.get(x.group(0), x.group(0)), body) if m else body + if m: # the target's own body is replaced, so its local declarations do not count + ported = _carry_decls(ported, m, donor_raw, raw_t.replace(target, "", 1)) + if "{" not in ported: + continue + d_head = ported[:ported.index("{")] + k = d_head.find(fn) + variants = [("", ported)] + tk = t_head.find(fn) + if k >= 0 and tk >= 0 and d_head[:k] != t_head[:tk]: + variants.append((" +target-return", t_head[:tk] + ported[k:])) + if t_head and d_head != t_head: + variants.append((" +target-signature", t_head + ported[ported.index("{"):])) + for vtag, cand in variants: + out.append((f"port {f.split('/')[-1]} {tag}{vtag}", cand)) + return out + + +ALL_FAMILIES = ("R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", "R10", "R12", "R13", "R14", "R15", "R16", "R17", "R18", "R19", "R20", "R21", "R22", "R23", "R24", "R25", "R26", "R27", "R28", "R29", "R31", "R32", "R33", "R34", "R35", "R36", "R37", "R38", "R39", "R40", "R41", "R42", "R43") RUNG_R_FAMILIES = ("R2", "R3", "R4", "R5", "R6", "R7") # the free sweep's set (R8/R9 are the search engine's until measured) @@ -3237,6 +4069,54 @@ def recipe_candidates(text, tu, fn, names, limit=24, rng=None, cap=40, blocks=Tr if "R26" in fam: for desc, cand in alias_repeated_addresses(text, tu, fn, d_): out.append(("R26", desc, cand)) + if "R29" in fam: + for desc, cand in fold_store_temps(text, tu, fn, d_): + out.append(("R29", desc, cand)) + if "R31" in fam: + for desc, cand in shift_operand_casts(text, tu, fn, d_): + out.append(("R31", desc, cand)) + if "R32" in fam: + for desc, cand in compound_assignments(text, tu, fn, d_): + out.append(("R32", desc, cand)) + if "R33" in fam: + for desc, cand in else_arm_assignments(text, tu, fn, d_): + out.append(("R33", desc, cand)) + if "R35" in fam: + for desc, cand in drop_param_copies(text, tu, fn, d_): + out.append(("R35", desc, cand)) + if "R36" in fam: + for desc, cand in merge_set_chains(text, tu, fn, d_): + out.append(("R36", desc, cand)) + if "R40" in fam: + for desc, cand in return_preincrement(text, tu, fn, d_): + out.append(("R40", desc, cand)) + if "R43" in fam: + for desc, cand in sign_test_to_mask(text, tu, fn, d_): + out.append(("R43", desc, cand)) + if "R42" in fam: + for desc, cand in move_statement_far(text, tu, fn, d_): + out.append(("R42", desc, cand)) + if "R41" in fam: + for desc, cand in swap_if_else_arms(text, tu, fn, d_): + out.append(("R41", desc, cand)) + if "R39" in fam: + for desc, cand in duplicate_join_statement(text, tu, fn, d_): + out.append(("R39", desc, cand)) + if "R38" in fam: + for desc, cand in shift_to_division(text, tu, fn, d_): + out.append(("R38", desc, cand)) + if "R37" in fam: + for desc, cand in return_constants(text, tu, fn, d_): + out.append(("R37", desc, cand)) + if "R34" in fam: + for desc, cand in merge_disjoint_locals(text, tu, fn, d_): + out.append(("R34", desc, cand)) + if "R28" in fam and not tu.startswith("src/fx/") and (REPO / tu).exists(): # the tree's pins live in the real TU + for desc, cand in merge_pinned_twins(tu, fn, text): + out.append(("R28", desc, cand)) + if "R27" in fam and not tu.startswith("src/fx/") and (REPO / tu).exists(): # the named port needs the real TU + for desc, cand in named_ports(tu, fn): + out.append(("R27", desc, cand)) if blocks and "R7" in fam: # last: one candidate per statement, so the targeted recipes go first for desc, cand in block_wraps(text, tu, fn, d_): out.append(("R7", desc, cand)) diff --git a/decomp-architect/corpus/tools/P10/delever_pack.py b/decomp-architect/corpus/tools/P10/delever_pack.py index 52e6bee8a..067cf8454 100644 --- a/decomp-architect/corpus/tools/P10/delever_pack.py +++ b/decomp-architect/corpus/tools/P10/delever_pack.py @@ -49,7 +49,9 @@ def related_bodies(tu, fn, target_text, alias, top=6, max_lines=600): want = set(SYM.findall(target_text)) - {fn} if not want: return "" - files = sorted((REPO / "src" / alias).glob("*.c")) + d = REPO / "src" / alias + # main's units live directly under src/ (src/800*.c), not src/main/ — S104 e3: every main pack's related.txt was empty + files = sorted((d if d.is_dir() else (REPO / tu).parent).glob("*.c")) inc = re.compile(r'#include\s+"\.\./(shared/[^"]+\.h)"') heads = set() for f in files: diff --git a/decomp-architect/corpus/tools/P10/delever_regen.py b/decomp-architect/corpus/tools/P10/delever_regen.py index 5c38d7e61..2f1be1c8f 100644 --- a/decomp-architect/corpus/tools/P10/delever_regen.py +++ b/decomp-architect/corpus/tools/P10/delever_regen.py @@ -61,6 +61,15 @@ def starts(e): bp = ds.RUN / "bodies" / f"{e['alias']}__{e['fn']}.c" if bp.exists(): out.append(("best", bp.read_text(errors="surrogateescape"))) + # an AGENT's lever-free near-miss (S104 e22, func_8017C954: the sweep had only ever seen the 632 free body; re-running + # the families on the agent's improved 28 body found the finisher in one pass — 1,655 candidates, two at 0) + ap = REPO / ".run" / "P36" / "agents" / f"{e['alias']}__{e['fn']}" / "body.c" + if ap.exists(): + t = ap.read_text(errors="surrogateescape") + # no lever AND no marked fake: S104's first run started from d20's PARKED body (invented always-false branches + # marked `!FAKE: dead-branch`, awaiting Drew) and the families "closed" it — a marked body is never a start text + if not re.search(r"__asm__|\bregister\b[^;]*\$|!FAKE", t) and all(t != x for _n, x in out): + out.append(("agent", t)) return out @@ -74,9 +83,18 @@ def one(e, fams, label): # the census's site lines no longer fit the file — the tree moved under the pass (S103: a bank during the R26 run # crashed the whole pool here). One class refused loudly, never the pass (R43); rerun after a census refresh. return dict(e, verdict="STALE-SITES", err=str(x)[:120], tried=0) - for sname, body in st: + # R27 (the named port) reads the REAL translation unit and the objects — it is not a rewrite of a start text, so it + # runs once per class, first (S104: 4 of 24 donor classes closed at the first candidate, d2/d6's move made mechanical) + srcs = ([("port", None)] if "R27" in fams else []) + st + for sname, body in srcs: try: - cands = dl.recipe_candidates(body, "src/fx/regen.c", e["fn"], [], cap=None, families=fams) + if sname == "port": + cands = [("R27", d, c) for d, c in dl.named_ports(e["tu"], e["fn"])] + else: + cands = dl.recipe_candidates(body, "src/fx/regen.c", e["fn"], [], cap=None, + families=tuple(f for f in fams if f not in ("R27", "R28"))) + if "R28" in fams: # R28 reads the tree's pins from the REAL unit + cands += [("R28", d, c) for d, c in dl.merge_pinned_twins(e["tu"], e["fn"], body)] except Exception as x: # a generator crash is a finding, not a silent skip (R43) return dict(e, verdict="GEN-ERROR", err=str(x)[:160], tried=tried) for rec, desc, cand in cands: @@ -116,6 +134,8 @@ def run(a): ds.sites_by_body() if "R19" in fams: dl.real_signatures() + if "R27" in fams: + dl.named_definitions() with cf.ProcessPoolExecutor(max_workers=a.jobs, mp_context=multiprocessing.get_context("fork")) as pool: futs = {pool.submit(one, e, fams, label): e for e in ex} # one line per judged class AS IT LANDS (R55): the agent lane draws only classes this pass has already judged diff --git a/decomp-architect/corpus/tools/P10/delever_search.py b/decomp-architect/corpus/tools/P10/delever_search.py index 0579130b3..fd7455695 100644 --- a/decomp-architect/corpus/tools/P10/delever_search.py +++ b/decomp-architect/corpus/tools/P10/delever_search.py @@ -308,9 +308,14 @@ def lever_free_body(tu, raw, fn, sites): if not edits: return raw try: - return dl.apply_edits(raw, edits) + out = dl.apply_edits(raw, edits) except dl.Refuse as ex: raise Unstrippable([("", 0, str(ex)[:120])]) + # R43 (S104 e24, func_80180E24): a deleted lever line that OPENED a multi-line comment left the comment's tail as code — the + # start text did not compile, and every sweep read the class as UNSCORED, never as refused. Refuse instead. + if out.count("/*") - out.count("*/") != raw.count("/*") - raw.count("*/"): + raise Unstrippable([("", 0, "a stripped lever line opened or closed a block comment")]) + return out def load_outcomes(): diff --git a/decomp-architect/corpus/tools/P10/lever_census.py b/decomp-architect/corpus/tools/P10/lever_census.py index 4fd824317..749f0ee1f 100644 --- a/decomp-architect/corpus/tools/P10/lever_census.py +++ b/decomp-architect/corpus/tools/P10/lever_census.py @@ -81,6 +81,7 @@ MIPS_REG_NAMES = {"zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", "t0", "t1", GTE_MNEMONICS = {"lwc2", "swc2", "mtc2", "mfc2", "ctc2", "cfc2", "cop2", "rtps", "rtpt", "nclip", "ncds", "nccs", "ncdt", "ncct", "ncs", "nct", "cdp", "cc", "dpcs", "dpct", "dpcl", "intpl", "sqr", "op", "gpf", "gpl", "avsz3", "avsz4", "mvmva"} FAKE_MARK = "!FAKE:" +C_FAKE_RX = re.compile(r"while\s*\(\s*0\s*\)|!FAKE:\s*(?:do-while|dead-init)") NON_LEVER_KINDS = {"gte", "verbatim-body", "gte-unsigned"} # Sony's coprocessor idiom; a manifest-listed hand-asm routine — censused, never a lever GTE_LEVER_KIND = "gte-lever" # a GTE op whose clobbers exceed its canonical's (a `_m`/`_v` variant macro, or a direct statement): a steer GTE_VARIANT_NAME = re.compile(r"(_m|_v[0-9a-f]{4})$") @@ -708,10 +709,15 @@ def walk_file(raw, rel, is_header, global_names=None): # orphan markers: a `// !FAKE:` line with no class A/B site on it, and none on the line below either (unless that line carries its own # marker) — a stale honesty claim (a marked site later removed or rewritten; the delever tool consumes a trailing marker, --scrub cleans) ab_lines = {s["line"] for s in sites if s["cls"] in "AB"} + # a marker on a KEPT ordinary-C fake — `do { … } while (0)` or a dead initialiser — is Drew's S104 ruling (a) and + # sotn-decomp's STYLE.md rule: the fake stays as C and carries `// !FAKE:`. Counted apart; never an orphan, never a lever. + cfake = [ln for ln, l in enumerate(raw_lines, start=1) if FAKE_MARK in l and C_FAKE_RX.search(l)] orphans = [ln for ln, l in enumerate(raw_lines, start=1) - if FAKE_MARK in l and ln not in ab_lines and not (ln + 1 in ab_lines and FAKE_MARK not in raw_line(ln + 1))] + if FAKE_MARK in l and ln not in ab_lines and not (ln + 1 in ab_lines and FAKE_MARK not in raw_line(ln + 1)) + and ln not in cfake] return dict(rel=rel, sites=sites, defs=[dict(name=d["name"], line=d["line"], end=d["end"], nhash=d["nhash"], nlines=d["nlines"]) for d in defs], - macro_defs=mdefs, coverage=cov, live_tokens=live_tokens, roled=len(roles), unclassified=unclassified, orphan_markers=orphans) + macro_defs=mdefs, coverage=cov, live_tokens=live_tokens, roled=len(roles), unclassified=unclassified, orphan_markers=orphans, + cfake_markers=cfake) def _walk_worker(args): @@ -819,6 +825,7 @@ def run_census(jobs, use_cache=True, out_dir=OUT_DIR_DEFAULT, want_sites=False, # attribute aliases, exclude the verbatim bodies, gather sites, defs_all, mdefs_all, unclassified = [], [], [], [] orphan_all = [] + cfake_all = [] cov_total = {k: collections.Counter() for k in TOKEN_CLASSES} verbatim_sites = 0 verbatim_fns = set() @@ -859,6 +866,7 @@ def run_census(jobs, use_cache=True, out_dir=OUT_DIR_DEFAULT, want_sites=False, mdefs_all.append(dict(md, tu=rel)) unclassified.extend(r["unclassified"]) orphan_all.extend(f"{rel}:{ln}" for ln in r.get("orphan_markers", ())) + cfake_all.extend(f"{rel}:{ln}" for ln in r.get("cfake_markers", ())) for k, c in r["coverage"].items(): cov_total[k].update(c) coverage = {k: dict(v) for k, v in cov_total.items()} @@ -903,6 +911,7 @@ def run_census(jobs, use_cache=True, out_dir=OUT_DIR_DEFAULT, want_sites=False, generated=time.strftime("%Y-%m-%d"), binaries=len(aliases), tus=len(tu_aliases), headers=len(headers), head=git_head(), src_stamp=src_stamp(), orphan_markers=dict(count=len(orphan_all), sample=orphan_all[:40]), + cfake_markers=dict(count=len(cfake_all), sample=cfake_all[:40]), coverage=coverage, coverage_ok=cov_ok, unclassified=len(unclassified), verbatim_excluded=dict(sites=verbatim_sites, functions=len(verbatim_fns), manifest_rows=len(verb)), classes=classes, @@ -1020,6 +1029,8 @@ def render(s): L.append(f" THE PHASE'S NUMBER (pins + asm statements, GTE excluded): {ab['sites']:,} sites in {ab['bodies']:,} bodies " f"({ab['distinct_bodies']:,} distinct) · marked !FAKE {ab['marked']:,} · UNMARKED {ab['unmarked']:,}") L.append(f" orphan !FAKE markers (no pin/asm site on the line nor below): {s.get('orphan_markers', {}).get('count', 0)}") + L.append(f" marked ordinary-C fakes kept by Drew's S104 ruling (a) (`do {{ }} while (0)`, dead initialisers; NOT levers): " + f"{s.get('cfake_markers', {}).get('count', 0)}") g = s.get("gte_levers", {}) if g: L.append(f" GTE levers (clobbers beyond the canonical macro's): {g['sites']:,} sites ({g['via_macro']:,} via a variant macro, {g['direct']:,} direct) · " diff --git a/decomp-architect/corpus/tools/P10/localalloc_sim.py b/decomp-architect/corpus/tools/P10/localalloc_sim.py index dccc38600..8f09db447 100644 --- a/decomp-architect/corpus/tools/P10/localalloc_sim.py +++ b/decomp-architect/corpus/tools/P10/localalloc_sim.py @@ -254,8 +254,11 @@ def main(): GR = sorted(GR) ids = list(range(len(qties))) # suggested first (qty_sugg_compare_1 approx: fewer suggestions first, then priority) + skey = lambda q: (len(qties[q]['copysugg']) or len(qties[q]['sugg']) * 64, -pri(qties[q]), q) sug = [q for q in ids if qties[q]['sugg'] or qties[q]['copysugg']] - sug.sort(key=lambda q: (len(qties[q]['copysugg']) or len(qties[q]['sugg']) * 64, -pri(qties[q]), q)) + sug.sort(key=skey) + if len(qties) == 3: # the same three-quantity switch in the suggested pass (local-alloc.c:1439-1462) + sug = [q for q in three_qty_order(lambda a, b: (skey(a) > skey(b)) - (skey(a) < skey(b))) if q in sug] phys = {} for q in sug: cs = qties[q]['copysugg'] or qties[q]['sugg'] @@ -265,6 +268,12 @@ def main(): for k in range(qties[q]['birth'], qties[q]['death']): live_at.setdefault(k, set()).add(r) ids.sort(key=lambda q: (-pri(qties[q]), q)) + if len(qties) == 3: + # local-alloc.c:1486-1507 — a block with exactly THREE quantities is not sorted: `qty_compare (0, 1)` / `(1, 2)` / + # `(0, 1)` compare the quantity NUMBERS (birth order) while EXCHANGE swaps positions in qty_order, so the first-born + # is allocated first unless priorities rise strictly in birth order. (S104 agent d5, func_801837E8: the full sort + # mispredicted 10-11 of 40 three-quantity blocks; this rule, 0 in the function, 6 left in the whole TU — separate.) + ids = three_qty_order(lambda a, b: pri(qties[b]) - pri(qties[a])) for q in ids: if q in phys: continue @@ -292,4 +301,18 @@ def main(): print(f" {2*k:4d} {u:5d} {txt}") + +def three_qty_order(qty_compare): + """gcc 2.7.2 block_alloc's `case 3:` / `case 2:` fall-through (local-alloc.c:1486-1507), literally: the compares take + quantity numbers 0, 1, 2 — never qty_order entries.""" + o = [0, 1, 2] + if qty_compare(0, 1) > 0: + o[0], o[1] = o[1], o[0] + if qty_compare(1, 2) > 0: + o[2], o[1] = o[1], o[2] + if qty_compare(0, 1) > 0: + o[0], o[1] = o[1], o[0] + return o + + main() diff --git a/decomp-architect/corpus/tools/P7/alloc_table.py b/decomp-architect/corpus/tools/P7/alloc_table.py index a92592e68..ad095e9a4 100644 --- a/decomp-architect/corpus/tools/P7/alloc_table.py +++ b/decomp-architect/corpus/tools/P7/alloc_table.py @@ -77,12 +77,15 @@ def main(): order = [int(x) for x in mo.group(2).split()] rows = [] for n, r in regs.items(): - pri = (math.floor(math.log2(r["refs"])) * r["refs"] / r["live"]) * 10000 if r["refs"] > 0 and r["live"] > 0 else 0 + # global.c:594-603 stores the priority in a `register int` — TRUNCATED — and breaks a tie by allocno number (:607). + # S104 e12/e14: two closes hinged on ties the old float column hid (245.6 vs 245.2 are both 245). + pri = int((math.floor(math.log2(r["refs"])) * r["refs"] / r["live"]) * 10000) if r["refs"] > 0 and r["live"] > 0 else 0 rows.append((pri, n, r)) rows.sort(key=lambda x: (-x[0], x[1])) print(f"alloc_table {fn} ({tag}): {len(regs)} pseudo(s) in the .lreg table" + (f"; .greg order: {' '.join(str(x) for x in order)}" if order else "; .greg printed no order line")) print(f" {'pseudo':>7} {'hard':>5} {'pri':>10} {'refs':>5} {'live':>5} {'blk':>5} prefs / notes") + tied = {p_ for p_, c in __import__("collections").Counter(p_ for p_, _n, _r in rows).items() if c > 1 and p_ > 0} for pri, n, r in rows: h = hard.get(n) hn = NAMES.get(h, str(h)) if h is not None else "-" @@ -93,7 +96,9 @@ def main(): hardconf = [NAMES[x] for x in conf[n] if x in NAMES] # the HARD regs it may not take if hardconf: note = f"conflicts {','.join(hardconf)} " + note - print(f" r{n:<6} {hn:>5} {pri:10.1f} {r['refs']:5d} {r['live']:5d} {str(r['block']):>5} {note}") + if pri in tied: + note = "TIE (lower allocno wins, global.c:607) " + note + print(f" r{n:<6} {hn:>5} {pri:10d} {r['refs']:5d} {r['live']:5d} {str(r['block']):>5} {note}") # R32: the order line over-approximates the allocnos; anything it names that this table lacks is a SILENT GAP missing = [x for x in order if x not in regs] if missing: diff --git a/docs/decision-log.md b/docs/decision-log.md index f3efe08bf..56ab68dc1 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -3748,3 +3748,42 @@ rest for free — the harvest→toolify gate at the granularity of one crack. `--explain` on one body and a line of the compiler's source, and every hour of wider search bought less than the previous one. The engine's real value was not the search but the SCORE — a distance that turns a stall into a named shape — and the instrument that prints it. Build that on day one of a de-lever campaign, before any search. + +## P36 S104 (2026-09-10/11) — Drew's four rulings read against sotn, the sweep-before-agent rule, and the TU-batch lane + +**Context/belief.** S103 ended with four open questions (do-while as a lever? the GTE header? proven-irreducible sites? the +signature changes?) and a residue of 5,097 sites in ~1,020 classes, 912 of them singletons. **Drew (2026-09-10):** "search the +sotn decomp" for all four. Read from sotn-decomp's tree (a shallow clone under the ignored `tools/reference/`, X2 — data): 0 +register pins and 0 asm statements in its PSX C across 2,987 files (it keeps 3,101 `INCLUDE_ASM` stubs instead); its fakes are +ordinary C marked `// !FAKE:` by `docs/STYLE.md`; one GTE header with seven project-local macros Sony never had. **Rulings:** +(a) `do { } while (0)` and dead initialisers stay as marked ordinary C — a census class apart, never an orphan, never a lever; +(b) GTE stays T5 and sotn is the precedent for a project-local macro; (c) every site is reducible to C — an unclosed site stays +marked and goes to the STRUCTS phase (the milestone amended: "0" is that phase's line); (d) "the types phase" IS the structs +phase; signature changes go there. **The sweep-before-agent rule (Drew):** "are we running sweeps with the new tooling on all +remaining funcs … before an agent touches them?" — from then on an agent draws only classes every current generator pass has +judged and not closed. + +**What failed and why it looked right.** (1) The first all-families regen ran on THREADS and sat on the GIL for five minutes +with zero compiles — R18 emits 1,000+ texts per body, R19 rebuilds a 29 s table per thread; processes forked after warming the +caches fixed it. (2) A `census && next` chain read the census's last line as OK while it had exited 1 on four orphan markers +(commit `830650946`, R97 again). (3) A pack-builder bug had left every MAIN pack's `related.txt` empty (main's units are +`src/800*.c`, not `src/main/`) — three main closes came from same-TU siblings the agents had to find by hand. (4) A lever strip +that deleted a line opening a block comment left the start text as invalid C; the sweeps read the class as UNSCORED — seven +classes named by a refusal now. (5) The float priority column in `alloc_table` hid the integer ties gcc actually breaks by +allocno number (`global.c:594-607`); two closes hinged on them. (6) The agent-start regen "closed" 14 of 27 — every one already +closed by its agent or PARKED (it had started from d20's marked invented-branch body): a marked body is never a start text. + +**The pivot.** With the ≥100-copy head worked, the residue is singletons: the lane became **TU batches** — one agent takes 3–4 +classes of ONE translation unit with that TU's closed mechanisms in its brief. Nearly every batch closed everything it took; +`ov_SC02_017_jr_8017DF34.c` reached 0 lever sites. The moves are a small catalog (METHOD steps 12–16; cookbook §456): one value +per temp, calls at their real arity, parameter copies deleted, narrow widths, stores in each arm, walked pointers → indexed +loops, goto chains → structured C, `& 0x80000000` sign tests, struct/array spellings where sched's alias test decides. Seventeen +generator families (R27–R43) were harvested, each run against its agent's own start text, then across the residue. + +**Measurement.** 5,097 → 4,223 sites; ~105 draws, ~205 classes at 0 (nearly all zero-lever), five bodies with a marked +do-while; the all-families sweep 74 MATCH over 1,020 classes (20,821 s); R27 25 of 995; the newer families 4–16 each. + +**Hindsight (the better path).** The sweep-before-agent rule and the TU-batch brief should have been the lane from T7's first +draw: the "closed mechanisms of this TU" are the strongest lead an agent gets, and the free sweep is the cheapest reviewer of an +agent's near-miss. Two structural gaps remain for tooling: a goto-chain → structured-C rewriter (about one close in five), and +a link-aware scorer (six parked closes are identical only after linking).