mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-05 08:57:59 -04:00
Drew T
85671bc1f7
build_wave_atlas: --one-per-gid collapses same-skeleton siblings to one card and defers
them to <out>.siblings.json for the post-bank family_sweep remap (R32 accounting asserted);
--rank total ranks gate groups by DELIVERED mass (card + deferred siblings). Measured on the
wave-T draw: 6,557 drafted ins carrying 12,709 sibling ins behind 69 of 71 gids = 19,266
instructions of potential for 71 agents, vs 9,985 behind 57 under --rank mass. R39 NC: the
flag is byte-inert on a pool whose gids are unique.
gate_main/pregate_check (§192): three defects that made the pre-gate ladder main-only while
reporting "clean" on overlay slates — (1) resolve_conflicts/substitute hardcoded
corpus.stubs('main') -> per-binary _stubs_for(); (2) sym_of returned the keyword `void` for
every `extern void (*D_x[])(...)`, manufacturing 192 phantom CONFLICTING-EXTERNs (NC over
5,526,100 declarations: 189,301 changed verdicts, 0 regressions); (3) `void f()` and
`void f(void)` were normalized together, costing 40 more phantoms — C89's unspecified-
parameter rule is now gate_main.sig_conflict. §192b: the tool refuses when it substituted 0
files, and prints the per-draft [DROP] reasons it used to compute and discard.
Same overlay slate now reports 2 failures, both real (duplicate typedef; memcpy declared two
ways). Cookbook §192/§192b + index regenerated (585 sections).
268 lines
15 KiB
Python
268 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""pregate_check.py — validate a main-EXE slate WITHOUT building it (~2s instead of ~5min).
|
|
|
|
WHY THIS EXISTS (P31 S52). Wave P drafted at 97% -- 58 of 60 functions byte-correct on the first
|
|
pass, zero symbol errors -- and then cost A DOZEN CLEAN REBUILDS to bank. Not one of those
|
|
rebuilds failed on a matching problem. Every single one failed on N independently-written drafts
|
|
having to agree with each other and with a translation unit none of them can see, and EVERY ONE OF
|
|
THOSE FAILURES WAS A TEXTUAL PROPERTY OF THE SUBSTITUTED FILE. We were paying five minutes of
|
|
`make` to be told something a grep could have said instantly.
|
|
|
|
`gate_main`'s own `resolve_conflicts` cannot answer this, and not because it is careless: it
|
|
inspects the DRAFTS, while the compiler sees the FILE THEY LAND IN -- after typedef stripping and
|
|
renaming, at each draft's own insertion offset, interleaved with declarations the file already had.
|
|
Those transformations happen at substitution time, *after* the conflict check has passed. So this
|
|
tool checks the artifact itself: `substitute(..., write=False)` into memory, then assert.
|
|
|
|
THE FIVE CHECKS, each earned by a real rebuild lost this session (cookbook §176h):
|
|
1. TYPEDEF-USED-ABOVE-DEFINITION -- src/800.c defines `Rec14` at line 7336 while stubs wanting
|
|
it sit at 7272; a stripped duplicate leaves the name undefined there (gcc: implicit-int, then
|
|
a collision reported at the *real* declaration, nowhere near the cause).
|
|
2. TYPE-NEVER-DEFINED -- a typedef deleted outright vanishes from any check that only looks at
|
|
names which ARE defined. (That was the hole in my own first control. R32 inside the R32 fix.)
|
|
3. DUPLICATE-TYPEDEF -- same name defined twice with different bodies.
|
|
4. CONFLICTING-EXTERN -- one symbol declared with two incompatible signatures in the final text,
|
|
whoever contributed them (file, draft A, draft B).
|
|
5. DEF-VS-DECL -- a function DEFINITION contradicting a prototype visible in the same file (the
|
|
DEF-side wall, §20 / wave law 3): `s32 func_8001ABBC(...)` under `extern void func_...`.
|
|
|
|
Comments are MASKED (via cdecl, the project's single masking oracle) before any use-site scan: an
|
|
unmasked scan reported 7 phantom "used before defined" hits, all of them agents' prose.
|
|
|
|
Usage:
|
|
pregate_check.py <slate.json> # report; exit 1 if any FAIL
|
|
pregate_check.py <slate.json> --quiet # exit code only
|
|
Leaves the working tree untouched.
|
|
"""
|
|
import argparse, json, os, re, sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import cdecl
|
|
import gate_main as gm
|
|
|
|
IDENT = r'[A-Za-z_]\w*'
|
|
|
|
|
|
def _depth_map(masked):
|
|
"""Brace depth at every offset, computed on COMMENT/STRING-MASKED text.
|
|
|
|
Scope matters and ignoring it makes this tool a liar: the project deliberately uses
|
|
BLOCK-SCOPE `extern` blocks (the §16/§17 local-block idiom, and the fix for a symbol whose
|
|
file-scope type disagrees), and a declaration inside one function cannot conflict with a
|
|
definition elsewhere. My first version compared every declaration in the file regardless of
|
|
depth and reported 4 hard failures on a slate that had just built BYTE-IDENTICAL -- the exact
|
|
over-refusal R39 exists to prevent, in the tool written to prevent it."""
|
|
depth, out = 0, bytearray(len(masked) + 1)
|
|
for i, ch in enumerate(masked):
|
|
if ch == '{':
|
|
depth += 1
|
|
elif ch == '}':
|
|
depth = max(0, depth - 1)
|
|
out[i] = min(depth, 255)
|
|
return out
|
|
|
|
|
|
STRUCT_EXTERN = re.compile(
|
|
r'^[ \t]*extern\s+(?:const\s+|volatile\s+)*(struct|union)\s*\{([^{}]*)\}\s*(\**)\s*(\w+)\s*(?:\[[^\]]*\])?\s*;',
|
|
re.M | re.S)
|
|
|
|
|
|
def _norm_sig(sig):
|
|
"""Delegates to the single normalization oracle in gate_main (R33).
|
|
|
|
It used to collapse `void f()` AND `void f(void)` to the same (), which is right about the
|
|
first (C89's unspecified parameter list, compatible with any prototype) and wrong about the
|
|
second (exactly zero parameters, incompatible with `f(s32)`). Since this tool could only ever
|
|
see main's TUs, the difference never showed; the first overlay slate it ran on produced 40
|
|
phantom CONFLICTING-EXTERN failures against a file that compiles today. Use `gm.sig_conflict`
|
|
for comparisons -- `!=` on these tuples is not the compatibility relation."""
|
|
return gm.norm_sig(sig)
|
|
|
|
|
|
def _typedefs(text):
|
|
"""name -> (offset, normalized_body) for every typedef the text defines."""
|
|
out = {}
|
|
for pat in (gm.TYPEDEF_BLOCK, gm.TYPEDEF_PLAIN):
|
|
for m in pat.finditer(text):
|
|
out.setdefault(m.group(1), []).append((m.start(), ' '.join(m.group(0).split())))
|
|
return out
|
|
|
|
|
|
def _func_defs(text):
|
|
"""Function definitions in the text -> {name: (offset, typesig)}."""
|
|
out = {}
|
|
for m in re.finditer(r'^[ \t]*([A-Za-z_][\w \t\*]*?)\b(%s)\s*\(([^;{]*)\)\s*\{' % IDENT,
|
|
text, re.M):
|
|
ret, name, params = m.group(1).strip(), m.group(2), m.group(3)
|
|
if name in ('if', 'for', 'while', 'switch', 'return', 'sizeof'):
|
|
continue
|
|
out[name] = (m.start(), gm.typesig('%s %s(%s)' % (ret, name, params)))
|
|
return out
|
|
|
|
|
|
def check_text(path, text):
|
|
"""Return a list of (severity, code, message) for one substituted file."""
|
|
findings = []
|
|
masked = cdecl._mask(text) # R33: the one comment/string masking oracle
|
|
depth = _depth_map(masked)
|
|
# SCAN THE MASKED TEXT, NEVER THE RAW TEXT (P31 S53, R35). This originally read
|
|
# `_typedefs(text)`, so a typedef quoted in a COMMENT counted as a definition: src/800.c's own
|
|
# bank note at line 2409 quotes `typedef struct { s32 a; s32 b[4]; } OtBlk_80016450;` inside a
|
|
# /* */ block, and the tool reported DUPLICATE-TYPEDEF against a file that has compiled
|
|
# byte-identically for weeks. Seven of nine FAILs on a real wave-R slate were comment-borne.
|
|
# The masking oracle was already computed one line above and simply was not used here; because
|
|
# _mask is length-preserving, every reported offset stays valid.
|
|
tds = _typedefs(masked)
|
|
|
|
# 3. duplicate typedef -- ANY redefinition, identical body or not.
|
|
# C89 has no "compatible redefinition" allowance for typedefs: `typedef struct {...} T;` twice
|
|
# is an error even when the two are character-identical. My first version only flagged
|
|
# DIFFERING bodies and therefore missed the very case this tool was built to catch -- a draft
|
|
# whose typedef was renamed to match the TU's, giving two identical definitions of
|
|
# `OtBlk_80016450`, which the compiler rejected on the next rebuild. Measured, not reasoned.
|
|
for name, defs in tds.items():
|
|
if len(defs) > 1:
|
|
same = len({b for _, b in defs}) == 1
|
|
findings.append(('FAIL', 'DUPLICATE-TYPEDEF',
|
|
f'{path}: `{name}` defined {len(defs)}x at offsets '
|
|
f'{[o for o, _ in defs]} ({"identical bodies -- still illegal in C89"
|
|
if same else "DIFFERENT bodies"})'))
|
|
|
|
# 1. typedef used above its definition
|
|
for name, defs in tds.items():
|
|
first_def = min(o for o, _ in defs)
|
|
uses = [m.start() for m in re.finditer(r'\b%s\b' % re.escape(name), masked)]
|
|
early = [u for u in uses if u < first_def]
|
|
if early:
|
|
findings.append(('FAIL', 'TYPEDEF-USED-ABOVE-DEFINITION',
|
|
f'{path}: `{name}` used at offset {min(early)} but first defined at '
|
|
f'{first_def} — a stripped duplicate whose survivor sits below'))
|
|
|
|
# 2. a type named in a declaration that nothing in this file defines
|
|
known = set(tds) | gm.TYPES | set(gm._ALIASES) | set(gm._ALIASES.values())
|
|
header_types = set()
|
|
for hp in ('include', 'src/shared'):
|
|
for root, _dirs, files in os.walk(hp):
|
|
for f in files:
|
|
if f.endswith('.h'):
|
|
try:
|
|
h = open(os.path.join(root, f), errors='ignore').read()
|
|
except OSError:
|
|
continue
|
|
header_types |= set(_typedefs(h))
|
|
header_types |= set(re.findall(r'\btypedef\s+[^;]*?\(\s*\*\s*(%s)\s*\)' % IDENT, h))
|
|
# function-pointer typedefs are not matched by the block/plain patterns; pick them up here too
|
|
known |= header_types | set(re.findall(r'\btypedef\s+[^;]*?\(\s*\*\s*(%s)\s*\)' % IDENT, text))
|
|
for m in re.finditer(r'^\s*extern\s+(?:const\s+|volatile\s+)*(%s)\b' % IDENT, masked, re.M):
|
|
t = m.group(1)
|
|
if t not in known and not t.startswith(('func_', 'D_')):
|
|
findings.append(('WARN', 'TYPE-NEVER-DEFINED',
|
|
f'{path}: `extern {t} ...` at offset {m.start()} — `{t}` is not '
|
|
f'defined in this file or any project header (PsyQ SDK type?)'))
|
|
|
|
# 4. one symbol declared two incompatible ways anywhere in the final text
|
|
decls = {}
|
|
# BRACE-BODIED EXTERNS FIRST (P31 S53). gm.DECL is single-line, so a draft declaring
|
|
# extern struct { u8 pad[0x34]; s32 (*field_0x34)(s32); } *D_80072780;
|
|
# was invisible to this check while a sibling declared the same symbol `void *` — the clash
|
|
# surfaced only as a compile error, one rebuild later (§183.5). Bodies are flat here (no nested
|
|
# braces), and the normalized body is part of the signature so two IDENTICAL struct declarations
|
|
# do not read as a conflict.
|
|
for m in STRUCT_EXTERN.finditer(masked):
|
|
if depth[m.start()]:
|
|
continue
|
|
body = ' '.join(m.group(2).split())
|
|
sig = (f'{m.group(1)}{{{body}}}{m.group(3)}', '')
|
|
decls.setdefault(m.group(4), (m.start(), sig))
|
|
for m in gm.DECL.finditer(text):
|
|
if depth[m.start()]: # block-scope decl: private to its function
|
|
continue
|
|
d = m.group(1)
|
|
s = gm.sym_of(d)
|
|
if not s:
|
|
continue
|
|
sig = _norm_sig(gm.typesig(d))
|
|
if s in decls and gm.sig_conflict(decls[s][1], sig):
|
|
findings.append(('FAIL', 'CONFLICTING-EXTERN',
|
|
f'{path}: `{s}` declared {decls[s][1]} at offset {decls[s][0]} and '
|
|
f'{sig} at offset {m.start()}'))
|
|
else:
|
|
decls.setdefault(s, (m.start(), sig))
|
|
|
|
# 5. definition vs a visible prototype
|
|
for name, (off, sig) in _func_defs(masked).items(): # masked, not raw — see the note above
|
|
sig = _norm_sig(sig)
|
|
if name in decls and gm.sig_conflict(decls[name][1], sig):
|
|
# SEVERITY CALIBRATED AGAINST THE COMPILER, not against C89 pedantry. Measured on the
|
|
# wave-P slate that built BYTE-IDENTICAL: gcc-2.7.2 accepted `void f(void*,s32)` vs a
|
|
# `void f(s8*,s32)` definition, and even `G3P *f(...)` vs `G4P *f(...)`. What it
|
|
# REJECTED was func_8001ABBC declared `void` and defined `s32` -- a void/non-void
|
|
# return split. So that is the only FAIL; everything else is a WARN worth reading but
|
|
# not worth blocking a rebuild over (R39: an over-refusal costs verified-correct work).
|
|
# RECALIBRATED (P31 S53). The rule was "only a void/non-void split is fatal", measured on
|
|
# the wave-P slate. S53 refuted the generalization: `func_80027F4C` DEFINED `G4P *` under
|
|
# a visible `G3P *` declaration -- both non-void -- and gcc-2.7.2 REJECTED it
|
|
# ("previous declaration of func_80027F4C", src/800.c:9225), costing a rebuild after this
|
|
# tool had reported only a WARN. So ANY return-type disagreement is a FAIL; parameter
|
|
# disagreements stay a WARN, which is still the calibration the wave-P bytes support.
|
|
d_ret, m_ret = decls[name][1][0], sig[0]
|
|
voidness = d_ret != m_ret
|
|
findings.append(('FAIL' if voidness else 'WARN', 'DEF-VS-DECL',
|
|
f'{path}: `{name}` DEFINED {sig} at offset {off} but DECLARED '
|
|
f'{decls[name][1]} at offset {decls[name][0]} — the DEF-side wall; '
|
|
f'adopt the declaration and cast at the use site (§176d)'))
|
|
return findings
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('slate')
|
|
ap.add_argument('--quiet', action='store_true')
|
|
a = ap.parse_args()
|
|
|
|
slate = json.load(open(a.slate))
|
|
kept, dropped = gm.resolve_conflicts(slate)
|
|
_n, texts = gm.substitute(kept, write=False)
|
|
|
|
# R32 COVERAGE ASSERTION (P31 S54). Until `gate_main` learned per-binary stub maps, an OVERLAY
|
|
# slate resolved to zero stubs and this tool printed "checking 0 substituted file(s) ... clean"
|
|
# -- a green light from a checker that had examined nothing, on exactly the slates (overlay
|
|
# waves) that carry most of the work. A checker that checked nothing must never read as a pass.
|
|
if kept and not texts:
|
|
print(f'REFUSING: {len(kept)} kept draft(s) but 0 substituted files — every entry resolved '
|
|
f'to no stub. Does each slate record carry its "binary"? Is the binary extracted '
|
|
f'(make extract BINARY=...)? This is not a clean result.')
|
|
sys.exit(2)
|
|
|
|
findings = []
|
|
for path, text in sorted(texts.items()):
|
|
findings += check_text(path, text)
|
|
|
|
fails = [f for f in findings if f[0] == 'FAIL']
|
|
if not a.quiet:
|
|
print(f'slate {len(slate)} -> {len(kept)} after resolve_conflicts '
|
|
f'({len(dropped)} dropped); checking {len(texts)} substituted file(s)')
|
|
# A DROP IS THE HEADLINE, NOT A FOOTNOTE (P31 S54). The drop reasons were computed and
|
|
# thrown away, so a slate that lost EVERY draft to declaration conflicts still printed
|
|
# "clean — the batch is worth a rebuild". They are the §183 playbook's actual worklist.
|
|
for d in dropped:
|
|
print(f' [DROP] {d["fn"]}: `{d["symbol"]}` clashes with {d["against"]} in '
|
|
f'{d["file"]} — kept {d["kept"]} vs this {d["this"]}')
|
|
for sev, code, msg in findings:
|
|
print(f' [{sev}] {code}: {msg}')
|
|
if not findings and not dropped:
|
|
print(' clean — no textual defect found; the batch is worth a rebuild')
|
|
elif not findings:
|
|
print(f' no textual defect in what SURVIVED, but {len(dropped)} draft(s) were dropped '
|
|
f'above — reconcile those before gating (they are unbanked work, not noise)')
|
|
elif not fails:
|
|
print(f' {len(findings)} warning(s), no hard failure — worth a rebuild')
|
|
else:
|
|
print(f'\n{len(fails)} FAILURE(S) — fix these BEFORE spending a clean rebuild.')
|
|
# exit 1 also when the slate was emptied: 0 kept is not a pass, it is total refusal.
|
|
sys.exit(1 if (fails or (slate and not kept)) else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|