mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
02f060f607
800c3 (0x8005CE18-0x8005FC68, one contiguous run of 33 interleaved Sony objects) is now four
stub rows — libapi1 (21 BIOS trampolines + COUNTER), libpad1 (PADENTRY + PADMAIN 760), libapi2
(L02/L03), libpad2 (PADCMD PADIF PADPORTD PADSEQD WAITRC2) — fed by two WINDOWED psyq_integrate
calls from the raw .run/obj42/{libapi42,libpad421} dirs (integrate tiles each stub with one
library; every boundary checked against .text SECTION sizes). The apicard region's three
"game code" rows were libapi 4.2's C objects to the byte: 800c2 = FIRST.o (firstfile + the
"no jump table wall" stub func_80062144), 800c2_2 = PAD.o, 800c2_3 = PATCH.o + CHCLRPAD.o ->
apicard5/6/7; make_apicard_used.py sources libapi from 4.2 (the EXE's real libapi; libcard
stays 4.0) into .run/obj42/apicard_used, 26 objects / 7 blocks, no game code left in
0x80061F38-0x80062888. src/800c3.c (129 hand-matched "C", 62 verbatim bodies, 19 stubs incl.
the four §332 %lo-in-a-delay-slot "walls"), src/800c2.c, src/800c2_2.c, src/800c2_3.c removed;
REORDER_TUS is empty (mechanism kept). Cookbook §490.
Two stale instruments fixed: exclude_audit let a pinned WALL outrank LINKED (PopMatrix/
PushMatrix had sat as walls since S68 while living in libgte3, linked since Phase 8) — LINKED
dominates now, config/wave_exclude.txt 13 -> 3; frontier_classify carried a hard-coded 49-name
LINKED set (R51) and reported 337 "stubs" — derived from the Makefile now.
Verified: main 143dbb89f34491258bbc27810d0a12ec8b43a8dd WITH all SDK dirs and WITHOUT them from
a fresh extract; make tools-health OK; R22 fleet extract-all 212/212 + check-all 213/213.
Metrics: main REAL 839->773, LINKED 1,150->1,256, VERBATIM 29->3, stubs 29->16, byte-identical
2,075/2,091 = 99.2%; game-code weighted 93.3% (38,748/41,534), remainder 2,786 = the open-stub
sum; fleet stubs 51->38 (frontier_classify: 39 rows incl. the data word). Verbatim manifest
33 -> 6. Docs: worklist rows + "S79 task #5", SETUP (fresh-clone obj42 commands, Makefile
blocks, exclude_audit), decision-log "S79 addendum 2", accelerators "S79 (2)", CURRENT_PHASE
S79 FINAL refreshed (census, metrics, the task #6 brief).
332 lines
17 KiB
Python
332 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""frontier_classify.py — classify EVERY remaining open stub by its TRUE BLOCKER, deterministically.
|
|
|
|
WHY THIS EXISTS (P31 S75). "69 functions left" is a count of stubs, not a measure of difficulty, and
|
|
this session proved the difference is enormous: five functions carried as a five-session "codegen
|
|
wall" were one jump-table extent bug, and `ov_SC03_108:func_8016AE5C` -- logged as
|
|
"match_one MATCH but the whole-binary gate rejected -- CAUSE NOT DETERMINED" -- banked with a
|
|
byte-perfect body the moment the carve was fixed. A frontier list that does not name the BLOCKER
|
|
routes drafting agents at problems that are not drafting problems.
|
|
|
|
Everything here is computed from artifacts already on disk (R33 / offline-tooling-first): the .s,
|
|
the sig registries, the backlog, the corpus. Zero tokens, no agents, no builds.
|
|
|
|
tools/frontier_classify.py [--json out.json]
|
|
"""
|
|
import argparse, collections, glob, importlib.util, json, os, re, subprocess, sys
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, os.path.join(REPO, 'tools'))
|
|
|
|
def _load(m, rel):
|
|
s = importlib.util.spec_from_file_location(m, os.path.join(REPO, rel))
|
|
x = importlib.util.module_from_spec(s); sys.modules[m] = x; s.loader.exec_module(x); return x
|
|
|
|
def _linked_subsegs():
|
|
"""main's LINKED subsegs, DERIVED from the Makefile's psyq_integrate stub lists (progress.py's parser).
|
|
P31 S79: this was a hard-coded 49-name set (R51 — a derived property stored as config goes stale and
|
|
takes a metric with it). It missed every block wired after it was written (libgte23-30, libgs7/8,
|
|
snd10-12, libgpu2, libapi1/2, libpad1/2, apicard5-7) and reported 337 "open stubs" — 300 of them the
|
|
INCLUDE_ASM records of linked Sony objects — while progress.py, reading the same Makefile, said 16."""
|
|
progress = _load('progress', 'tools/progress.py')
|
|
return set(progress._main_linked_segs_from_makefile())
|
|
|
|
|
|
LINKED = _linked_subsegs()
|
|
INS = re.compile(r'\s*/\* [0-9A-F]+ [0-9A-F]{8} ')
|
|
|
|
|
|
|
|
def fleet_binaries():
|
|
"""The REAL fleet, derived from the Makefile + the two .mk lists (R33 — never a hardcoded list)."""
|
|
out = {'main', 'resident'}
|
|
for f, var in (('config/overlays.mk', 'OVERLAY_BINARIES'), ('config/modules.mk', 'MODULE_BINARIES')):
|
|
p = os.path.join(REPO, f)
|
|
if os.path.exists(p):
|
|
m = re.search(rf'^{var}\s*:=\s*(.*)$', open(p).read(), re.M)
|
|
if m:
|
|
out |= set(m.group(1).split())
|
|
return out
|
|
|
|
|
|
def open_stubs():
|
|
"""[(fn, binary, path, asm_path, nins)] for every live non-LINKED INCLUDE_ASM stub."""
|
|
out = []
|
|
for root, _d, files in os.walk(os.path.join(REPO, 'src')):
|
|
for f in files:
|
|
if not f.endswith('.c') or f[:-2] in LINKED:
|
|
continue
|
|
p = os.path.join(root, f)
|
|
rel = os.path.relpath(root, os.path.join(REPO, 'src'))
|
|
binary = 'main' if rel == '.' else rel
|
|
for line in open(p, errors='ignore'):
|
|
m = re.match(r'\s*INCLUDE_ASM\("([^"]+)",\s*(\w+)', line)
|
|
if not m:
|
|
continue
|
|
a = os.path.join(REPO, m.group(1), m.group(2) + '.s')
|
|
n = sum(1 for l in open(a, errors='ignore') if INS.match(l)) if os.path.exists(a) else 0
|
|
out.append((m.group(2), binary, os.path.relpath(p, REPO), a, n))
|
|
return out
|
|
|
|
|
|
def asm_facts(asm_path):
|
|
"""Structural tells read straight from the function's own .s."""
|
|
if not os.path.exists(asm_path):
|
|
return {}
|
|
txt = open(asm_path, errors='ignore').read()
|
|
return {
|
|
'jtbl': bool(re.search(r'\bjtbl_[0-9A-Fa-f]{8}\b', txt)), # owns a switch jump table
|
|
'jr': bool(re.search(r'\bjr\s+\$(?!ra)', txt)), # indirect jump (dispatch)
|
|
'jalr': bool(re.search(r'\bjalr\b', txt)), # indirect call
|
|
'mult': bool(re.search(r'\b(mult|multu|div|divu)\b', txt)),
|
|
'float': bool(re.search(r'\b(cop2|mtc2|mfc2|ctc2|cfc2)\b', txt)),
|
|
'calls': len(re.findall(r'\bjal\s+\w', txt)),
|
|
}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--json', default=None)
|
|
a = ap.parse_args()
|
|
|
|
corpus = _load('corpus', 'tools/corpus.py')
|
|
stubs = open_stubs()
|
|
|
|
# ---- evidence layers, all pre-existing artifacts -------------------------------------------
|
|
# BEST closeness, not LAST — and keep the draft that ACHIEVED it (P31 S75).
|
|
#
|
|
# `.run/backlog.jsonl` is APPEND-ONLY: a function accumulates one row per attempt, across every
|
|
# lane and session. Taking the last row means taking whichever lane wrote most recently, which is
|
|
# evidence about THAT LANE'S SEED, not about the function. Two redraft agents caught this within
|
|
# an hour of each other:
|
|
#
|
|
# func_8017DB98 — brief said "closeness 115, materially wrong, re-derive". journal_notes showed
|
|
# THREE of six recorded attempts were already MATCH; attempt #6 records that the
|
|
# pack's warm-start had been the OLDER, WORSE shard. A warm-start regression
|
|
# looks exactly like an unsolved function from inside the wave.
|
|
# func_800CB00C — "the 'materially wrong 168' and a byte-exact closeness-0 body were two
|
|
# different files for the same function."
|
|
#
|
|
# So a function with a banked-quality draft in its history was being classified F-FAR (redraft)
|
|
# and would have been handed to an agent to re-derive work that was already done. MIN is the
|
|
# honest reduction; `close_last` is kept alongside so a large gap between them is itself visible
|
|
# as a warm-start regression signal.
|
|
close, close_last, verdict, draft, attempts = {}, {}, {}, {}, collections.Counter()
|
|
bl = os.path.join(REPO, '.run/backlog.jsonl')
|
|
if os.path.exists(bl):
|
|
for ln in open(bl):
|
|
try: r = json.loads(ln)
|
|
except Exception: continue
|
|
n = r.get('name')
|
|
if not n: continue
|
|
attempts[n] += 1
|
|
c = r.get('closeness')
|
|
if c is not None:
|
|
close_last[n] = c
|
|
if n not in close or c < close[n]:
|
|
close[n] = c
|
|
if r.get('best_draft'):
|
|
draft[n] = r['best_draft'] # the draft that ACHIEVED the best score
|
|
if r.get('where_stuck'): verdict[n] = r['where_stuck']
|
|
if r.get('best_draft') and n not in draft: draft[n] = r['best_draft']
|
|
for p in glob.glob(os.path.join(REPO, '.run/harvest_failed.*.classified.txt')):
|
|
for ln in open(p):
|
|
q = ln.rstrip('\n').split('\t')
|
|
if len(q) >= 2 and q[0] not in verdict:
|
|
verdict[q[0]] = q[1]
|
|
|
|
# SECOND EVIDENCE SOURCE: the agent journals (R34 — a disagreeing oracle, not a better assertion).
|
|
#
|
|
# `.run/backlog.jsonl` is not the only record of what has been tried, and it is not the best one.
|
|
# `tools/journal_notes.py` mines the AGENT JOURNALS, which carry outcomes the backlog never
|
|
# received. Measured on func_8017DB98 (P31 S75): backlog best == last == 115, so best-vs-last
|
|
# could not help — while the journal holds "Attempt 2 (MATCH · closeness 0) … MATCH 122/122 …
|
|
# BANK BLOCKER is TU plumbing, not the body (§376/§378)" WITH the draft path and the exact line
|
|
# to change. Classified F-FAR ("redraft") on the backlog alone; it is really C-PLUMBING, and an
|
|
# agent would have re-derived a body that was already byte-exact.
|
|
#
|
|
# Cheap: one subprocess per function that has no closeness-0 already, and only for the functions
|
|
# we are about to classify.
|
|
def journal_best(fn):
|
|
"""(closeness, draft_path) from the agent journals, or (None, None)."""
|
|
try:
|
|
r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/journal_notes.py'),
|
|
'--fn', fn], capture_output=True, text=True, timeout=60, cwd=REPO)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return None, None
|
|
# SPLIT on the attempt marker; do NOT consume past it (P31 S75, third instance of this
|
|
# defect class in one session). The first cut used
|
|
# `re.finditer(r'\*\*Attempt \d+\*\* \(([^)]*)\)(.{0,400})', ..., re.S)` — the 400-char
|
|
# body window SWALLOWS THE NEXT ATTEMPT'S HEADER, so every record that follows another was
|
|
# invisible. On func_8017DB98 that hid attempts 2 AND 6, both `MATCH · closeness 0`, and
|
|
# returned 2 (attempt 1's NEAR) as the best — the exact records the oracle exists to find.
|
|
# A regex that consumes an unbounded body cannot enumerate the items after the first.
|
|
chunks = re.split(r'(?=\*\*Attempt \d+\*\*)', r.stdout)
|
|
best, path = None, None
|
|
for ch in chunks:
|
|
hm = re.match(r'\*\*Attempt \d+\*\* \(([^)]*)\)', ch)
|
|
if not hm:
|
|
continue
|
|
head = hm.group(1)
|
|
cm = (re.search(r'closeness\s*[=:]?\s*(\d+)', head)
|
|
or re.search(r'closeness\s*[=:]?\s*(\d+)', ch))
|
|
# a header that says MATCH with no number IS closeness 0
|
|
c = int(cm.group(1)) if cm else (0 if re.match(r'\s*MATCH\b', head) else None)
|
|
if c is None:
|
|
continue
|
|
if best is None or c < best:
|
|
best = c
|
|
pm = re.search(r'(\.run/[\w./-]+\.c)', ch)
|
|
path = pm.group(1) if pm else None
|
|
return best, path
|
|
|
|
# h_exact reach: is this function byte-identical to code in another binary? (a twin is a REMAP,
|
|
# never a redraft -- §168) and is any twin already MATCHED (i.e. is there a proven exemplar)?
|
|
#
|
|
# THE SIG DIRECTORY IS NOT THE FLEET (P31 S75 — caught by the known-true check, R14). Alongside
|
|
# the 213 real binaries `.run/` also holds `SLUS_007.26` (a STALE duplicate of main under the ROM
|
|
# filename), `resident_image`, and two CROSS-BUILD binaries — `sep8_SLUS_007.26` and
|
|
# `aug31_USA_DEMO.EXE`. Counting those as peers made every main stub look like it had a
|
|
# byte-identical twin ALREADY BANKED somewhere: 38 functions / 7,516 instructions of pure
|
|
# artifact, most of them "proven" in main ITSELF under its other name, the rest in a PROTOTYPE
|
|
# that R13 forbids as verified evidence. Derive the fleet from the Makefile and refuse the rest.
|
|
fleet = fleet_binaries()
|
|
seen_sigs = {os.path.basename(q)[4:-6] for q in glob.glob(os.path.join(REPO, '.run/sig.*.jsonl'))}
|
|
dropped = sorted(seen_sigs - fleet)
|
|
if dropped:
|
|
print(f"[classify] IGNORING {len(dropped)} non-fleet sig registr(ies) as twin evidence: "
|
|
f"{', '.join(dropped)}", file=sys.stderr)
|
|
if not fleet:
|
|
sys.exit("frontier_classify: could not derive BINARIES from the Makefile — refusing to "
|
|
"classify against an unknown fleet (R32)")
|
|
sig, banked_addrs = {}, set()
|
|
for p in glob.glob(os.path.join(REPO, '.run/sig.*.jsonl')):
|
|
b = os.path.basename(p)[4:-6]
|
|
if b not in fleet:
|
|
continue
|
|
for ln in open(p):
|
|
try: r = json.loads(ln)
|
|
except Exception: continue
|
|
sig.setdefault(int(r['addr'], 16), []).append((b, r.get('h_exact')))
|
|
open_pairs = {(f, b) for f, b, _p, _a, _n in stubs}
|
|
for addr, rows in sig.items():
|
|
for b, _h in rows:
|
|
if (f'func_{addr:08X}', b) not in open_pairs:
|
|
banked_addrs.add((addr, b))
|
|
|
|
# Any stored draft on disk, anywhere under .run — ONE pruned walk, not a `.run/S7*` glob.
|
|
#
|
|
# The glob was `.run/S7*/**/*.c`, which is NARROWER THAN THE CLAIM IT SUPPORTED (P31 S75, R32 —
|
|
# the session's recurring defect, committed by me in the very tool written to classify it). Real
|
|
# drafts live in `.run/S69m2/`, `.run/S68m1/`, `.run/s67m1/` (lowercase), `.run/wave_ds2/`,
|
|
# `.run/gate_lane/`, `.run/backlog_drafts/` — none of which match `S7*`. Result: all 32 drafted
|
|
# main functions were classified H-VIRGIN ("never drafted -- genuine new drafting work"), which
|
|
# would have sent agents to redraft 6,328 instructions that already have drafts sitting on disk.
|
|
# The worktrees are pruned because they are full repo copies (7.4 GB) of the SAME sources.
|
|
disk_drafts = collections.defaultdict(list)
|
|
for dp, dn, fns in os.walk(os.path.join(REPO, '.run')):
|
|
dn[:] = [d for d in dn if d != '.git' and not d.startswith('wt_')
|
|
and d not in ('pgate', 'obj40', 'audit')]
|
|
for f in fns:
|
|
if f.endswith('.c'):
|
|
disk_drafts[f[:-2]].append(os.path.join(dp, f))
|
|
|
|
rows = []
|
|
for fn, binary, path, asm, nins in stubs:
|
|
try: addr = int(fn[5:], 16)
|
|
except ValueError: addr = None
|
|
f = asm_facts(asm)
|
|
h = None; twins = []
|
|
if addr in sig:
|
|
me = [x for x in sig[addr] if x[0] == binary]
|
|
h = me[0][1] if me else None
|
|
if h:
|
|
twins = [b for b, hh in sig[addr] if hh == h and b != binary]
|
|
proven_twin = [b for b in twins if (addr, b) in banked_addrs]
|
|
c = close.get(fn)
|
|
jc, jpath = (None, None)
|
|
if c is None or c > 0: # only ask when the backlog has no proven body
|
|
jc, jpath = journal_best(fn)
|
|
if jc is not None and (c is None or jc < c):
|
|
c = jc
|
|
if jpath:
|
|
draft[fn] = jpath
|
|
v = (verdict.get(fn) or '')
|
|
o0 = False
|
|
try: o0 = bool(corpus.is_o0(binary, addr)) if addr else False
|
|
except Exception: pass
|
|
|
|
# ---- the blocker ladder: most specific evidence first ----------------------------------
|
|
if proven_twin:
|
|
k = 'A-TWIN-REMAP' # a byte-identical copy is ALREADY BANKED elsewhere
|
|
elif f.get('jtbl'):
|
|
k = 'B-CARVE' # owns a switch table -> the S75 carve class
|
|
elif c == 0:
|
|
k = 'C-PLUMBING' # body proven, TU refuses it
|
|
elif c is not None and c <= 25:
|
|
k = 'D-NEAR' # permuter fuel, not drafting
|
|
elif o0:
|
|
k = 'E-O0' # -O0 island
|
|
elif c is not None and c <= 150:
|
|
k = 'F-FAR' # a draft exists but is materially wrong
|
|
elif fn in disk_drafts or fn in draft:
|
|
k = 'G-DRAFTED-UNKNOWN' # tried, no usable verdict recorded
|
|
else:
|
|
k = 'H-VIRGIN' # never drafted: genuine new work
|
|
regressed = (close_last.get(fn) is not None and c is not None
|
|
and close_last[fn] > c + 10) # a later attempt scored materially worse
|
|
rows.append(dict(fn=fn, binary=binary, nins=nins, klass=k, closeness=c,
|
|
closeness_last=close_last.get(fn), attempts=attempts.get(fn, 0),
|
|
warmstart_regression=regressed, journal_closeness=jc,
|
|
jtbl=f.get('jtbl'), jr=f.get('jr'), jalr=f.get('jalr'), o0=o0,
|
|
calls=f.get('calls'), twins=len(twins), proven_twin=proven_twin[:3],
|
|
drafts=len(disk_drafts.get(fn, [])), verdict=v[:90], path=path))
|
|
|
|
rows.sort(key=lambda r: (r['klass'], -r['nins']))
|
|
by = collections.defaultdict(list)
|
|
for r in rows: by[r['klass']].append(r)
|
|
NAME = {
|
|
'A-TWIN-REMAP': 'a byte-identical copy is ALREADY BANKED elsewhere -> mechanical remap',
|
|
'B-CARVE': 'owns a switch jump table -> the S75 jtbl_carve class',
|
|
'C-PLUMBING': 'closeness 0: body proven, the TU refuses the signature',
|
|
'D-NEAR': 'closeness <=25 -> permuter fuel, NOT drafting',
|
|
'E-O0': '-O0 island',
|
|
'F-FAR': 'a draft exists but is materially wrong -> redraft',
|
|
'G-DRAFTED-UNKNOWN':'drafted before, no usable verdict on record',
|
|
'H-VIRGIN': 'never drafted -- genuine new drafting work',
|
|
}
|
|
tot_n = sum(r['nins'] for r in rows)
|
|
print(f"OPEN STUBS: {len(rows)} ({len({r['fn'] for r in rows})} distinct) {tot_n:,} instructions\n")
|
|
print(f"{'class':20s} {'fns':>4s} {'ins':>7s} what it actually needs")
|
|
print('-' * 100)
|
|
for k in sorted(by):
|
|
g = by[k]
|
|
print(f"{k:20s} {len(g):4d} {sum(x['nins'] for x in g):7,d} {NAME[k]}")
|
|
print('-' * 100)
|
|
print(f"{'TOTAL':20s} {len(rows):4d} {tot_n:7,d}")
|
|
for k in sorted(by):
|
|
print(f"\n== {k} — {NAME[k]}")
|
|
for r in by[k]:
|
|
extra = []
|
|
if r['closeness'] is not None:
|
|
extra.append(f"close={r['closeness']}(best of {r['attempts']})")
|
|
if r.get('journal_closeness') is not None:
|
|
extra.append(f"journal={r['journal_closeness']}")
|
|
if r.get('warmstart_regression'):
|
|
extra.append(f"!!WARMSTART-REGRESSION last={r['closeness_last']}")
|
|
if r['jtbl']: extra.append('jtbl')
|
|
if r['jr']: extra.append('jr')
|
|
if r['o0']: extra.append('-O0')
|
|
if r['twins']: extra.append(f"twins={r['twins']}")
|
|
if r['proven_twin']: extra.append(f"proven@{','.join(r['proven_twin'])}")
|
|
if r['drafts']: extra.append(f"drafts={r['drafts']}")
|
|
print(f" {r['nins']:5d} {r['fn']:20s} {r['binary']:14s} {' '.join(extra)}")
|
|
if r['verdict']: print(f" verdict: {r['verdict']}")
|
|
if a.json:
|
|
json.dump(rows, open(a.json, 'w'), indent=1)
|
|
print(f"\n-> {a.json}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|