mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
7473640838
- MY BUG, found by reconciling against the old sweep (R14): when I introduced the two-oracle UNION I updated the BUCKET accounting but left the ledger's row-listing condition on L1 alone. So the byte total was already right (3,564,021) while the LIST under-reported — 34 rows instead of 78. Same "two code paths, one updated" shape as the day's other defects. Fixed: rows use the same union. - THE COMPLETION CONTRACT'S "39 type-1 modules" IS SUPERSEDED: the real backlog is **78 unclaimed code payloads / 3.56 MB** — MAIN.CD 42, SC03 18, SC05 7, SC04 7, SC07 2, SC02 2. The 39 came from disc_code_sweep, which reads only the RAW layer through a 4,096-WORD WINDOW and has no notion of a claim. Reconciled decisively: all 39 hash-checked against config/check.*.sha -> 0 of 39 claimed, so the new set strictly CONTAINS the old one. docs/disc-completeness.md updated, old text kept for provenance. - WORKED EXAMPLE of why the window mattered: SC07.CD FILE_003/1.1 is 345,132 B whose HEAD is code — the old window saw valid=100%, the whole-payload average is valid=0.571 (L1 says data), and L2 carves 3 real functions. Only the union gets it right, which is the entire argument for R34. - Partition still holds: residue 0 over 416,021,760 B, 1,291 payloads examined.
307 lines
15 KiB
Python
307 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""disc_audit.py — L1 of the definitive disc audit: a PARTITION over every byte on the disc.
|
|
|
|
Drew, 2026-08-05: "im getting tired of learning there was more code all along, we really need a full
|
|
audit that definitively lists ALL code that we need to decomp."
|
|
|
|
THE INVARIANT (R32): every byte on the disc belongs to exactly ONE bucket —
|
|
onboarded-code | unclaimed-code | classified-data | audio-video | filesystem-metadata
|
|
— the buckets SUM TO THE DISC, and residue is a DEFECT. A partition with an asserted residue of zero
|
|
is a completeness proof; a longer list is only a longer list.
|
|
|
|
WHY THIS EXISTS. Three "there was more code all along" surprises each came from a tool that was
|
|
correct about the subset it examined and silent about the rest: a `0.4.dec` glob missed 4 SC07
|
|
overlays whose code sits at PAC entry 1; `disc_code_sweep` decoded only the RAW layer, so it was
|
|
structurally blind to COMPRESSED code (its type-4 row was vacuous for 138 known binaries); a
|
|
4,096-word window classified only the head of each payload. So this walks the DISC IMAGE rather than
|
|
our configs, classifies WHOLE payloads with no window, and decodes BOTH layers.
|
|
|
|
CLAIMED-BY IS DERIVED, NOT RE-PARSED (R33). `config/check.<bin>.sha` IS the SHA1 of that binary's disc
|
|
payload — the build's own byte-identity gate. So payload→binary is a hash lookup against the committed
|
|
check files: it cannot drift from the build, and needs no config-walking heuristic.
|
|
|
|
ACCOUNTING MODEL (exact by construction). Tracks 2..N are CD-DA in full. On track 1 each file costs its
|
|
SECTOR FOOTPRINT (ceil(size/2048) * 2352); inside a `.CD` container the raw payload bytes go to their
|
|
own buckets and the container's leftover footprint (TOC + padding) goes to filesystem-metadata; the
|
|
track-1 bytes outside every file (system area, PVD, path tables, directory records) go to
|
|
filesystem-metadata. Totals therefore sum to the disc exactly, or the run FAILS.
|
|
|
|
python3 tools/disc_audit.py # -> docs/disc-ledger.md + .run/disc_audit.json
|
|
make audit-disc
|
|
"""
|
|
import glob, hashlib, json, os, re, sys
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
os.chdir(REPO)
|
|
sys.path.insert(0, os.path.join(REPO, 'tools', 'bfm_extract'))
|
|
|
|
import iso9660, cd_archive, pac, lzss # noqa: E402 (the frozen Phase-2 extractor)
|
|
|
|
RAW_SECTOR = 2352
|
|
USER_BYTES = 2048
|
|
DISKS = 'disks'
|
|
LEDGER = 'docs/disc-ledger.md'
|
|
|
|
VALID_OPS = frozenset((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
|
|
32, 33, 34, 35, 36, 37, 38, 40, 41, 42, 43, 46, 49, 57))
|
|
|
|
|
|
def classify(buf):
|
|
"""('code'|'data', valid_ratio, jr_density) over the WHOLE buffer — no window (that was defect #3).
|
|
|
|
Deliberately the same heuristic `disc_code_sweep` uses, so L2's independent oracle (sig_image
|
|
boundary carving) has something to DISAGREE with (R34); disagreements are a review queue."""
|
|
n = len(buf) // 4
|
|
if n < 16:
|
|
return 'data', 0.0, 0.0
|
|
valid = jr = 0
|
|
for i in range(0, n * 4, 4):
|
|
w = int.from_bytes(buf[i:i + 4], 'little')
|
|
if (w >> 26) in VALID_OPS:
|
|
valid += 1
|
|
if w == 0x03E00008: # jr $ra
|
|
jr += 1
|
|
vr, jd = valid / n, jr / n
|
|
return ('code' if vr >= 0.90 and jd >= 0.01 else 'data'), vr, jd
|
|
|
|
|
|
_SIG = None
|
|
|
|
|
|
def l2_carve(buf, cap=3):
|
|
"""L2 (R34): an INDEPENDENT verdict — does a boundary carver find real functions here?
|
|
|
|
L1's test is a statistical heuristic (valid>=0.90 AND jr>=0.01); it can miss a small code payload
|
|
whose `jr $ra` density falls under 1%, and it has no way to know it missed one. This oracle asks a
|
|
structurally different question: walk the payload as sig_image does, cutting each function at the
|
|
first `jr $ra` that lies at/after every forward branch target seen so far. Data has no regular
|
|
epilogue, so the walk stops immediately; code yields function after function.
|
|
|
|
Returns (n_functions_found, instructions_covered), stopping at `cap` — a YES is what we need, and
|
|
stopping early keeps the whole-disc pass affordable. The two oracles are then made to ARGUE: any
|
|
payload where they disagree lands in the ledger's review queue rather than being silently resolved
|
|
in favour of either."""
|
|
global _SIG
|
|
if _SIG is None:
|
|
sys.path.insert(0, os.path.join(REPO, 'tools'))
|
|
import sig_image as _si
|
|
_SIG = _si
|
|
n = len(buf) // 4 * 4
|
|
if n < 64:
|
|
return 0, 0
|
|
base, pos, found, covered = 0x80000000, 0x80000000, 0, 0
|
|
hi = base + n
|
|
while pos < hi and found < cap:
|
|
try:
|
|
end = _SIG.func_end(buf, base, pos, hi)
|
|
except Exception:
|
|
break
|
|
if end >= hi: # no qualifying return -> left the code region
|
|
break
|
|
found += 1
|
|
covered += end - pos
|
|
pos = end
|
|
return found, covered
|
|
|
|
|
|
def bucket_of(who, k1, k2):
|
|
"""Resolve a payload's bucket from the claim + BOTH oracles.
|
|
|
|
Two L1 defects that the L2 disagreement exposed (P30 S43, and exactly why R34 exists):
|
|
|
|
1. **A CLAIM OUTRANKS A HEURISTIC.** If the payload's SHA1 equals a committed
|
|
`config/check.<bin>.sha`, it IS that onboarded binary — the build gates on that hash every
|
|
day. Letting a statistical verdict file a claimed payload as `classified-data` was simply
|
|
wrong, and it silently shrank the onboarded bucket.
|
|
2. **WHOLE-PAYLOAD AVERAGING DILUTES CODE.** A real location overlay is code followed by a large
|
|
data tail, so its whole-payload valid-ratio lands ~0.87 — under L1's 0.90 gate — while L2
|
|
carves real functions out of its head. Classifying the whole payload (the fix for the old
|
|
4,096-word window) traded a head-only bias for an averaging bias.
|
|
|
|
So: a claim wins outright, and for the rest take the UNION of the two oracles. The union is the
|
|
conservative direction for the question this audit exists to answer — over-reporting a payload as
|
|
code puts it in a review queue, under-reporting it hides code, which is the failure mode that
|
|
produced three "there was more code all along" surprises."""
|
|
if who:
|
|
return 'onboarded-code'
|
|
return 'unclaimed-code' if (k1 == 'code' or k2 == 'code') else 'classified-data'
|
|
|
|
|
|
def onboarded_payload_shas():
|
|
"""{sha1: alias} from every committed config/check.<bin>.sha (R33 — the build's own oracle)."""
|
|
out = {}
|
|
for p in sorted(glob.glob('config/check.*.sha')):
|
|
alias = re.sub(r'^check\.|\.sha$', '', os.path.basename(p))
|
|
parts = open(p).read().split()
|
|
if parts:
|
|
out[parts[0].lower()] = alias
|
|
return out
|
|
|
|
|
|
FORM2_BYTES = 2324 # MODE2 FORM2 user bytes (STR/XA streaming sectors)
|
|
|
|
|
|
def footprint(size, form2=False):
|
|
"""Raw sector cost of a file. MODE2 FORM1 carries 2048 user bytes per 2352-byte sector; STR/XA
|
|
streaming data is FORM2 and carries 2324 — using 2048 for those over-counts their footprint."""
|
|
per = FORM2_BYTES if form2 else USER_BYTES
|
|
return ((size + per - 1) // per) * RAW_SECTOR
|
|
|
|
|
|
def main():
|
|
claimed = onboarded_payload_shas()
|
|
tracks = sorted(glob.glob(os.path.join(DISKS, '*Track *.bin')))
|
|
if not tracks:
|
|
sys.exit('disc_audit: no tracks under disks/ — this audit reads the DISC, not our configs')
|
|
t1 = [t for t in tracks if 'Track 1' in t][0]
|
|
disc_bytes = sum(os.path.getsize(t) for t in tracks)
|
|
t1_size = os.path.getsize(t1)
|
|
print(f'disc: {disc_bytes:,} bytes across {len(tracks)} tracks; '
|
|
f'onboarded payload hashes: {len(claimed)}')
|
|
|
|
B = dict.fromkeys(('onboarded-code', 'unclaimed-code', 'classified-data',
|
|
'audio-video', 'filesystem-metadata'), 0)
|
|
B['audio-video'] += sum(os.path.getsize(t) for t in tracks if 'Track 1' not in t)
|
|
|
|
rows, payloads, file_fp_total, disputes = [], 0, 0, []
|
|
with iso9660.Iso9660Image(t1) as img:
|
|
recs = list(img.list_files())
|
|
print(f'ISO root files: {len(recs)}')
|
|
t1_sectors = t1_size // RAW_SECTOR
|
|
for rec in recs:
|
|
name = rec.name
|
|
up = name.upper()
|
|
# A `.DA` entry's LBA points PAST track 1 into the CD-DA tracks (Phase 2 deviation:
|
|
# "the .DA ISO entries point past Track 1 into the audio tracks"). Those bytes are
|
|
# already counted whole as audio-video, so charging a track-1 footprint too
|
|
# double-counts them — this was 49.7 MB of the first run's residue.
|
|
if rec.lba >= t1_sectors or up.endswith('.DA'):
|
|
continue
|
|
fp = footprint(rec.size, form2=up.endswith(('.STR', '.XA')))
|
|
file_fp_total += fp
|
|
if up.endswith(('.STR', '.XA')):
|
|
B['audio-video'] += fp
|
|
continue
|
|
data = img.extract_file(name)
|
|
if not up.endswith('.CD'):
|
|
k, vr, jd = classify(data)
|
|
nf, _cov = l2_carve(data)
|
|
k2 = 'code' if nf >= 2 else 'data'
|
|
h = hashlib.sha1(data).hexdigest()
|
|
who = claimed.get(h)
|
|
b = bucket_of(who, k, k2)
|
|
B[b] += fp
|
|
payloads += 1
|
|
if k != k2:
|
|
disputes.append((name, '-', '-', len(data), k, k2, nf, f'{vr:.2f}/{jd:.4f}'))
|
|
if b != 'classified-data':
|
|
rows.append((name, '-', '-', len(data), b, who or 'UNCLAIMED',
|
|
f'{vr:.2f}/{jd:.4f} L2={nf}'))
|
|
continue
|
|
|
|
acct = 0
|
|
try:
|
|
toc = cd_archive.parse_toc(data)
|
|
subs = list(cd_archive.iter_subfiles(data, toc))
|
|
except Exception as e:
|
|
print(f' ! {name}: TOC walk failed ({e}) — whole file booked as data')
|
|
B['classified-data'] += fp
|
|
rows.append((name, '-', '-', len(data), 'classified-data', 'TOC-FAIL', '-'))
|
|
continue
|
|
for si, (_ent, sbuf) in enumerate(subs):
|
|
try:
|
|
entries = pac.split_pac(sbuf) if pac.is_pac(sbuf) else []
|
|
except Exception:
|
|
entries = []
|
|
if not entries:
|
|
items = [(-1, -1, sbuf, len(sbuf))]
|
|
else:
|
|
items = [(e.index, e.type, e.payload, e.length) for e in entries]
|
|
for idx, etype, raw, rawlen in items:
|
|
payloads += 1
|
|
buf = raw
|
|
if etype == pac.COMPRESSED_TYPE:
|
|
try:
|
|
buf = lzss.decompress(raw, expected_len=len(raw)).data or raw
|
|
except Exception:
|
|
buf = raw
|
|
k, vr, jd = classify(buf)
|
|
nf, _cov = l2_carve(buf)
|
|
k2 = 'code' if nf >= 2 else 'data'
|
|
if k != k2:
|
|
disputes.append((f'{name}/{si}', idx, etype, len(buf), k, k2, nf,
|
|
f'{vr:.2f}/{jd:.4f}'))
|
|
h = hashlib.sha1(buf).hexdigest()
|
|
who = claimed.get(h)
|
|
b = bucket_of(who, k, k2)
|
|
B[b] += rawlen
|
|
acct += rawlen
|
|
if b != 'classified-data':
|
|
rows.append((f'{name}/{si}', idx, etype, len(buf), b,
|
|
who or 'UNCLAIMED', f'{vr:.2f}/{jd:.4f} L2={nf}'))
|
|
B['filesystem-metadata'] += max(0, fp - acct)
|
|
|
|
# track-1 bytes outside every file: system area, PVD, path tables, directory records, gaps
|
|
B['filesystem-metadata'] += max(0, t1_size - file_fp_total)
|
|
|
|
total = sum(B.values())
|
|
residue = disc_bytes - total
|
|
unclaimed = [r for r in rows if r[5] == 'UNCLAIMED']
|
|
|
|
os.makedirs('.run', exist_ok=True)
|
|
with open(LEDGER, 'w') as fh:
|
|
fh.write('# Disc ledger — the completeness partition (L1)\n\n')
|
|
fh.write('> `tools/disc_audit.py` (`make audit-disc`). Walks the DISC IMAGE, not our configs;\n'
|
|
'> classifies WHOLE payloads (no window) at BOTH the raw and LZSS-decompressed layer.\n'
|
|
'> `claimed-by` is derived from `config/check.<bin>.sha` — the build\'s own\n'
|
|
'> byte-identity gate (R33). Residue is a DEFECT (R32), not a rounding note.\n\n')
|
|
fh.write(f'**Disc:** {disc_bytes:,} bytes · {len(tracks)} tracks · {payloads:,} payloads examined\n\n')
|
|
fh.write('| bucket | bytes | share |\n|---|---:|---:|\n')
|
|
for k, v in B.items():
|
|
fh.write(f'| {k} | {v:,} | {100*v/disc_bytes:.2f}% |\n')
|
|
fh.write(f'| **TOTAL** | **{total:,}** | {100*total/disc_bytes:.2f}% |\n')
|
|
fh.write(f'\n**Residue: {residue:,} bytes — '
|
|
f'{"PARTITION HOLDS" if residue == 0 else "DEFECT, unaccounted"}**\n\n')
|
|
fh.write(f'## L2 review queue — {len(disputes)} oracle disagreement(s)\n\n')
|
|
fh.write('> L1 = statistical heuristic (valid>=0.90 AND jr>=0.01). L2 = independent boundary\n'
|
|
'> carving (sig_image `func_end` walk). R34: the two are made to ARGUE; a disagreement\n'
|
|
'> is a review item, never silently resolved. L1=data/L2=code is the DANGEROUS\n'
|
|
'> direction — that is missed code.\n\n')
|
|
if disputes:
|
|
fh.write('| file | entry | type | bytes | L1 | L2 | fns carved | valid/jr |\n')
|
|
fh.write('|---|---|---|---:|---|---|---:|---|\n')
|
|
for d in sorted(disputes, key=lambda d: (d[4] != 'data', -d[3])):
|
|
fh.write(f'| {d[0]} | {d[1]} | {d[2]} | {d[3]:,} | {d[4]} | {d[5]} | {d[6]} | {d[7]} |\n')
|
|
fh.write('\n')
|
|
else:
|
|
fh.write('_The two oracles agree on every payload._\n\n')
|
|
fh.write(f'## Code payloads — {len(unclaimed)} UNCLAIMED of '
|
|
f'{len([r for r in rows if "code" in r[4]])}\n\n')
|
|
fh.write('| file | entry | type | bytes | bucket | claimed-by | valid/jr |\n')
|
|
fh.write('|---|---|---|---:|---|---|---|\n')
|
|
for r in sorted(rows, key=lambda r: (r[5] != 'UNCLAIMED', str(r[0]))):
|
|
if 'code' in r[4]:
|
|
fh.write(f'| {r[0]} | {r[1]} | {r[2]} | {r[3]:,} | {r[4]} | {r[5]} | {r[6]} |\n')
|
|
|
|
json.dump({'buckets': B, 'disc_bytes': disc_bytes, 'residue': residue,
|
|
'payloads': payloads, 'unclaimed_code': len(unclaimed),
|
|
'l2_disputes': len(disputes)},
|
|
open('.run/disc_audit.json', 'w'), indent=1)
|
|
|
|
for k, v in B.items():
|
|
print(f' {k:22s} {v:>14,}')
|
|
print(f' {"TOTAL":22s} {total:>14,} disc {disc_bytes:,} residue {residue:,}')
|
|
print(f'L2 disagreements (review queue): {len(disputes)}')
|
|
for d in sorted(disputes, key=lambda d: (d[4] != 'data', -d[3]))[:12]:
|
|
print(f' {d[0]} entry={d[1]} type={d[2]} {d[3]:,} B L1={d[4]} L2={d[5]} fns={d[6]} {d[7]}')
|
|
print(f'UNCLAIMED code payloads: {len(unclaimed)}')
|
|
for r in unclaimed[:25]:
|
|
print(f' {r[0]} entry={r[1]} type={r[2]} {r[3]:,} B valid/jr={r[6]}')
|
|
print(f'ledger -> {LEDGER}')
|
|
if residue != 0:
|
|
sys.exit(f'AUDIT-DISC FAIL (R32): {residue:,} bytes unaccounted — the partition does not hold')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|