diff --git a/.run/disc_code_sweep.txt b/.run/disc_code_sweep.txt new file mode 100644 index 000000000..864ac9d91 --- /dev/null +++ b/.run/disc_code_sweep.txt @@ -0,0 +1,51 @@ +disc code sweep — 1189 payloads under extracted/retail, window=4096w, valid>=0.9, jr_ra>=0.01 +type payloads code onboarded HIDDEN + 0 301 0 0 0 + 1 166 40 1 39 + 2 201 0 0 0 + 3 201 0 0 0 + 4 138 138 138 0 + 6 29 0 0 0 + 7 139 0 0 0 + 8 14 0 0 0 + +HIDDEN code-bearing payloads (valid>=0.9 AND jr_ra>=0.01, NOT onboarded): 39 + type 1 valid 96.4% jr_ra 7.23% (83w, raw) extracted/retail/MAIN.CD.dir/FILE_024.dir/0.1 + type 1 valid 96.9% jr_ra 6.19% (97w, raw) extracted/retail/MAIN.CD.dir/FILE_023.dir/0.1 + type 1 valid 98.4% jr_ra 5.13% (312w, raw) extracted/retail/MAIN.CD.dir/FILE_035.dir/1.1 + type 1 valid 100.0% jr_ra 3.39% (4096w, raw) extracted/retail/MAIN.CD.dir/FILE_012.dir/1.1 + type 1 valid 100.0% jr_ra 3.39% (4096w, raw) extracted/retail/SC02.CD.dir/FILE_037.dir/0.1 + type 1 valid 100.0% jr_ra 3.39% (4096w, raw) extracted/retail/SC03.CD.dir/FILE_107.dir/0.1 + type 1 valid 99.4% jr_ra 3.24% (1018w, raw) extracted/retail/MAIN.CD.dir/FILE_022.dir/1.1 + type 1 valid 100.0% jr_ra 2.86% (210w, raw) extracted/retail/MAIN.CD.dir/FILE_008.dir/1.1 + type 1 valid 98.8% jr_ra 2.70% (482w, raw) extracted/retail/MAIN.CD.dir/FILE_016.dir/1.1 + type 1 valid 96.0% jr_ra 2.68% (373w, raw) extracted/retail/MAIN.CD.dir/FILE_017.dir/1.1 + type 1 valid 99.2% jr_ra 2.67% (900w, raw) extracted/retail/MAIN.CD.dir/FILE_026.dir/1.1 + type 1 valid 100.0% jr_ra 2.52% (159w, raw) extracted/retail/MAIN.CD.dir/FILE_042.dir/0.1 + type 1 valid 99.0% jr_ra 2.49% (522w, raw) extracted/retail/MAIN.CD.dir/FILE_014.dir/1.1 + type 1 valid 100.0% jr_ra 2.35% (978w, raw) extracted/retail/MAIN.CD.dir/FILE_043.dir/0.1 + type 1 valid 93.4% jr_ra 2.34% (726w, raw) extracted/retail/MAIN.CD.dir/FILE_037.dir/1.1 + type 1 valid 98.8% jr_ra 2.33% (816w, raw) extracted/retail/MAIN.CD.dir/FILE_033.dir/1.1 + type 1 valid 98.4% jr_ra 2.28% (1229w, raw) extracted/retail/MAIN.CD.dir/FILE_036.dir/1.1 + type 1 valid 99.2% jr_ra 2.27% (484w, raw) extracted/retail/MAIN.CD.dir/FILE_021.dir/1.1 + type 1 valid 96.4% jr_ra 2.22% (496w, raw) extracted/retail/MAIN.CD.dir/FILE_029.dir/1.1 + type 1 valid 98.6% jr_ra 2.20% (1318w, raw) extracted/retail/MAIN.CD.dir/FILE_027.dir/1.1 + type 1 valid 97.4% jr_ra 2.13% (469w, raw) extracted/retail/MAIN.CD.dir/FILE_030.dir/1.1 + type 1 valid 99.2% jr_ra 1.97% (1115w, raw) extracted/retail/MAIN.CD.dir/FILE_028.dir/1.1 + type 1 valid 93.9% jr_ra 1.97% (2230w, raw) extracted/retail/MAIN.CD.dir/FILE_044.dir/0.1 + type 1 valid 98.9% jr_ra 1.89% (636w, raw) extracted/retail/MAIN.CD.dir/FILE_009.dir/0.1 + type 1 valid 92.4% jr_ra 1.82% (1425w, raw) extracted/retail/MAIN.CD.dir/FILE_039.dir/1.1 + type 1 valid 94.8% jr_ra 1.62% (678w, raw) extracted/retail/MAIN.CD.dir/FILE_018.dir/1.1 + type 1 valid 90.1% jr_ra 1.61% (4096w, raw) extracted/retail/SC03.CD.dir/FILE_076.dir/2.1 + type 1 valid 90.1% jr_ra 1.61% (4096w, raw) extracted/retail/SC03.CD.dir/FILE_135.dir/2.1 + type 1 valid 90.2% jr_ra 1.61% (4096w, raw) extracted/retail/SC04.CD.dir/FILE_027.dir/2.1 + type 1 valid 90.4% jr_ra 1.61% (4096w, raw) extracted/retail/SC05.CD.dir/FILE_026.dir/2.1 + type 1 valid 99.5% jr_ra 1.50% (1861w, raw) extracted/retail/MAIN.CD.dir/FILE_034.dir/1.1 + type 1 valid 99.9% jr_ra 1.47% (818w, raw) extracted/retail/MAIN.CD.dir/FILE_047.dir/0.1 + type 1 valid 99.9% jr_ra 1.46% (4096w, raw) extracted/retail/SC07.CD.dir/FILE_004.dir/1.1 + type 1 valid 90.5% jr_ra 1.46% (1025w, raw) extracted/retail/MAIN.CD.dir/FILE_041.dir/1.1 + type 1 valid 97.3% jr_ra 1.43% (1261w, raw) extracted/retail/MAIN.CD.dir/FILE_031.dir/1.1 + type 1 valid 100.0% jr_ra 1.37% (4096w, raw) extracted/retail/SC07.CD.dir/FILE_003.dir/1.1 + type 1 valid 93.0% jr_ra 1.30% (1690w, raw) extracted/retail/MAIN.CD.dir/FILE_019.dir/1.1 + type 1 valid 99.5% jr_ra 1.25% (4096w, raw) extracted/retail/MAIN.CD.dir/FILE_003.dir/1.1 + type 1 valid 99.5% jr_ra 1.25% (4096w, raw) extracted/retail/MAIN.CD.dir/FILE_004.dir/0.1 diff --git a/tools/disc_code_sweep.py b/tools/disc_code_sweep.py index adbea6493..ee9e868c7 100644 --- a/tools/disc_code_sweep.py +++ b/tools/disc_code_sweep.py @@ -27,7 +27,9 @@ import struct import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "bfm_extract")) import sig_image # make_insn — the shared rabbitizer decode (GTE-aware) +import lzss # decompress — the SAME game-semantics decoder the extractor uses REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # a raw payload is /. with NO .dec suffix; type-4 also has a .dec sibling @@ -36,27 +38,29 @@ _PAYLOAD = re.compile(r"/(\d+)\.(\d+)$") def onboarded_payloads(): """The set of payload paths already wired as a build binary (main, resident, every overlay). - Read from the Makefile/overlays.mk *_EXE assignments — the single source of truth (R33).""" + Read from the Makefile/overlays.mk *_EXE assignments — the single source of truth (R33). + + KEYED BY THE RAW PAYLOAD PATH (Phase-28 T7). A type-4 overlay's `_EXE` is the DECOMPRESSED path + (`…/0.4.dec`), but this sweep iterates the RAW payloads (`…/0.4`) so it can decode the compressed + layer itself. Store the `.dec`-stripped form too, or all 138 type-4 overlays — now correctly seen + as code via decompression — would fail the onboarded match and flag as false HIDDEN hits.""" paths = set() for mk in ("Makefile", "config/overlays.mk"): p = os.path.join(REPO, mk) if not os.path.exists(p): continue for m in re.finditer(r"^\w+_EXE\s*:=\s*(\S+)", open(p).read(), re.M): - paths.add(os.path.normpath(m.group(1))) + norm = os.path.normpath(m.group(1)) + paths.add(norm) + if norm.endswith(".dec"): + paths.add(norm[:-len(".dec")]) # the raw payload this .dec came from return paths _JR_RA = 0x03E00008 # `jr $ra` — a function return; the discriminator rabbitizer.isValid() lacks -def code_signals(path, window): - """(valid_ratio, jr_ra_density, nwords). isValid() alone is too permissive — structured DATA - decodes ~100% valid (type-0/type-2 blobs hit 1.00 with ZERO returns). Real MIPS code carries a - `jr $ra` roughly once per function (~2.9-3.4% of words across the onboarded overlays + resident); - data carries ~0%. Requiring BOTH separates code from valid-looking data (verified against the - positive controls).""" - data = open(path, "rb").read() +def _signals(data, window): n = min(window, len(data) // 4) if n == 0: return 0.0, 0.0, 0 @@ -70,6 +74,39 @@ def code_signals(path, window): return ok / n, jr / n, n +def code_signals(path, window): + """(valid_ratio, jr_ra_density, nwords, layer). isValid() alone is too permissive — structured + DATA decodes ~100% valid (type-0/type-2 blobs hit 1.00 with ZERO returns). Real MIPS code carries + a `jr $ra` roughly once per function (~2.9-3.4% of words across the onboarded overlays + resident); + data carries ~0%. Requiring BOTH separates code from valid-looking data. + + DECODES BOTH LAYERS (Phase-28 T7 — the fix). The original decoded only the RAW payload, which is + STRUCTURALLY BLIND to compressed code: type-4 overlay code is LZSS-compressed, so raw bytes are + noise and every type-4 row read "code 0" — a VACUOUS row for 138 known-code binaries. That is not + a curiosity: this tool is the disc-completeness oracle (R34), and it could NOT have found the 4 + hidden SC07 overlays (their code is compressed like every type-4) — they were caught by + hand-reconciling 138-vs-134. It found the 39 type-1 modules ONLY because those happen to be + uncompressed. So decode both and take the stronger signal: + * uncompressed code (type-1 resident-class): lives in the RAW bytes. + * compressed code (type-4 overlays): lives in the LZSS-DECOMPRESSED bytes. + A payload is code if EITHER layer is code; `layer` records which, so the report is auditable.""" + raw = open(path, "rb").read() + r_ratio, r_jr, r_n = _signals(raw, window) + best = (r_ratio, r_jr, r_n, "raw") + try: + res = lzss.decompress(raw) + dec = res.data if hasattr(res, "data") else bytes(res) + except Exception: + dec = b"" + if len(dec) >= 64: + d_ratio, d_jr, d_n = _signals(dec, window) + # "stronger" = the layer that clears the code bar, or (if neither/both) the higher jr density + # (the discriminator). This never downgrades a raw-code hit to a decompressed non-hit. + if (d_jr, d_ratio) > (best[1], best[0]): + best = (d_ratio, d_jr, d_n, "dec") + return best + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--root", default="extracted/retail") @@ -94,16 +131,16 @@ def main(): continue seen += 1 rel = os.path.relpath(p, REPO) - ratio, jr, nwords = code_signals(p, a.window) + ratio, jr, nwords, layer = code_signals(p, a.window) is_code = ratio >= a.threshold and jr >= a.jr_min # BOTH signals (jr is the discriminator) - rows.append((typ, ratio, jr, nwords, rel, os.path.normpath(rel) in onboard, is_code)) + rows.append((typ, ratio, jr, nwords, rel, os.path.normpath(rel) in onboard, is_code, layer)) assert len(rows) == seen, f"classified {len(rows)} of {seen} payloads — a silent skip (R32)" hidden = [r for r in rows if r[6] and not r[5]] by_type = {} - for typ, ratio, jr, _, _, onb, is_code in rows: + for typ, ratio, jr, _, _, onb, is_code, _layer in rows: d = by_type.setdefault(typ, [0, 0, 0]) d[0] += 1 d[1] += is_code @@ -118,8 +155,8 @@ def main(): print(f"{typ:>4} {tot:>9} {c:>5} {onb:>9} {c - onb:>7}") print(f"\nHIDDEN code-bearing payloads (valid>={a.threshold} AND jr_ra>={a.jr_min}, NOT onboarded): {len(hidden)}") - for typ, ratio, jr, nwords, rel, _, _ in sorted(hidden, key=lambda r: -r[2]): - print(f" type {typ} valid {ratio:5.1%} jr_ra {jr:5.2%} ({nwords}w) {rel}") + for typ, ratio, jr, nwords, rel, _, _, layer in sorted(hidden, key=lambda r: -r[2]): + print(f" type {typ} valid {ratio:5.1%} jr_ra {jr:5.2%} ({nwords}w, {layer}) {rel}") if not hidden: print(" (none — the onboarded set accounts for every code-bearing payload the sweep sees)") return 0 if not hidden else 3