#!/usr/bin/env python3 """audit_public.py — the first-push gate: no ROM-derived bytes among the tracked files (P33 B7; CI job `audits`; extended P33.5 task 8). tools/audit_public.py # every git-tracked file at HEAD's index (git ls-files) tools/audit_public.py --paths a b … # an explicit file list (tests / a pre-commit hook) Four checks, each derived from something the repo already asserts (R33), never from a typed list of hashes: 1. PURGE PATHS — no tracked file lies under a rule of tools/public_rewrite/purge_set.txt (the C1 rewrite's own input, filter-repo syntax: bare prefix or `glob:`) OR of tools/public_rewrite/untracked_after_rewrite.txt (the audit-only sibling: paths untracked AFTER the rewrite without a second rewrite — kept out of purge_set.txt because gate_scan.py reads that file as the census of what the rewrite removed from history). 2. ROM CONTENT — no tracked file's SHA1 is a known ROM-derived hash: every `sha1` in extracted/retail/manifest.jsonl (the manifest IS the list of ROM-derived artifacts), every binary's SHA1 in config/check.*.sha, and the redump Track-1 SHA1 (tools/bfm_extract/extract_exe.REDUMP_TRACK1_SHA1). A renamed copy is caught by content. 3. SIZE — no tracked file over 100 MiB (GitHub's HARD limit: a push with a larger file is rejected; 50 MiB is only its warning). Raised from 50 MiB on 2026-09-12 (Drew, P37 S107) when the Phase-36 judgement ledger — our own tool's rows, no ROM content — reached 78 MB and the cap alone turned the public CI red. 4. CONTENT — no tracked TEXT file carries a long contiguous run of disassembly-shaped lines (an assembler listing, an objdump, a splat `/* ADDR HEX hex */` block, a glabel block): the class-3 case a path-and-hash audit cannot see — a notes file that pastes a function's instructions is ROM-derived even when the tracked C reproduces the bytes. The shapes were derived from the two offenders the P33.5 audit found; the criterion is the LONGEST CONTIGUOUS run per file, threshold 64 lines, so a short quoted diff in a cookbook section passes and a full listing does not. Prints every count with its denominator (R41) and exits 1 on any offender, naming each (R43: loud, never partial). Ubuntu CI runs it without the disc, asm/, expected/ or build/ — it reads only tracked text and hashes tracked files. Controls (R39): S87 (before C3) the private tree MUST fail naming exactly the purge set (28 dumps, the EXE, ghidra/**, tools/psyq/**, the 3 session-archive parts, 2 zips, brave.exe); P33.5 task 8: before the pair was untracked, check 4 MUST name exactly .run/giants/fable_cd4/{mine_full,target_full}.txt and nothing else (the xsig fixtures, 40 lines, pass). """ import fnmatch import hashlib import json import os import pathlib import re import subprocess import sys REPO = pathlib.Path(__file__).resolve().parent.parent PURGE = REPO / "tools" / "public_rewrite" / "purge_set.txt" UNTRACKED_AFTER = REPO / "tools" / "public_rewrite" / "untracked_after_rewrite.txt" MANIFEST = REPO / "extracted" / "retail" / "manifest.jsonl" SIZE_CAP = 100 * 1024 * 1024 SIZE_WARN = 80 * 1024 * 1024 # a tracked file this close to GitHub's hard limit is announced before a push can be rejected (R54) RUN_CAP = 64 # contiguous disassembly-shaped lines that make a text file an offender DISASM_RES = [ # asm-differ: `12: addiu $sp, $sp, -0x40`, `44: nop`, `50: j .L80133D40` (an operand-less mnemonic and a # `.L` label operand both continue the run — `nop` lines broke the first cut's run on the target listing, 60 < 64) re.compile(r"^\s*\d+:\s+[a-z]{2,8}(?:\.[a-z]+)?(?:\s+(?:\$|-?0x|-?\d|[a-z_.])|\s*$)"), re.compile(r"^\s*[0-9a-f]+:\s+[0-9a-f]{8}\s+\w+"), # objdump / `0: 27bdffc0 addiu sp,sp,-64` re.compile(r"^\s*/\* [0-9A-F]{4,} [0-9A-F]{8} [0-9A-F]{8} \*/"), # splat: `/* 1A2B0 80018730 27BDFFC0 */` re.compile(r"^(?:glabel|dlabel)\s"), ] def read_rules(path, label): """filter-repo syntax; refuses regex: rules and an empty file (R43). Returns (prefixes, globs).""" if not path.exists(): sys.exit(f"audit_public: {path.relative_to(REPO)} is missing — cannot derive the {label} rules (R32)") prefixes, globs = [], [] for ln in path.read_text(encoding="utf-8").splitlines(): ln = ln.strip() if not ln or ln.startswith("#"): continue if ln.startswith("glob:"): globs.append(ln[5:]) elif ln.startswith("regex:"): sys.exit(f"audit_public: regex: rules are not supported here ({ln}) — R43") else: prefixes.append(ln) if not prefixes and not globs: sys.exit(f"audit_public: {path.relative_to(REPO)} holds no rules — refusing to pass on an empty {label} set (R43)") return prefixes, globs def under_rules(path, prefixes, globs): for p in prefixes: if path == p or path.startswith(p if p.endswith("/") else p + "/"): return p for g in globs: if fnmatch.fnmatchcase(path, g): return "glob:" + g return None def rom_hashes(): """Every SHA1 the repo itself declares ROM-derived. Coverage-asserted: each source must exist and be non-empty.""" out = {} n_manifest = 0 if not MANIFEST.exists(): sys.exit(f"audit_public: {MANIFEST} is missing — the ROM-hash set cannot be derived (R32)") for ln in MANIFEST.read_text(encoding="utf-8").splitlines(): if ln.strip(): o = json.loads(ln) out[o["sha1"].lower()] = "manifest:" + o["path"] n_manifest += 1 checks = sorted((REPO / "config").glob("check.*.sha")) if not checks: sys.exit("audit_public: no config/check.*.sha files — cannot derive the binary hashes (R32)") for c in checks: for ln in c.read_text(encoding="utf-8").splitlines(): parts = ln.split() if len(parts) >= 2 and len(parts[0]) == 40: out[parts[0].lower()] = f"{c.name}:{parts[1]}" sys.path.insert(0, str(REPO / "tools" / "bfm_extract")) from extract_exe import REDUMP_TRACK1_SHA1 # the disc itself out[REDUMP_TRACK1_SHA1.lower()] = "redump:Track 1" if n_manifest < 1000: sys.exit(f"audit_public: manifest has {n_manifest} rows (expected the 1,801-row retail manifest) — refusing (R32)") return out, n_manifest, len(checks) def tracked_files(): r = subprocess.run(["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True) return [p for p in r.stdout.decode("utf-8", "surrogateescape").split("\0") if p] def sha1_of(path): h = hashlib.sha1() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def longest_disasm_run(path): """The longest contiguous run of disassembly-shaped lines in a text file; 0 for a binary file.""" with open(path, "rb") as f: data = f.read() if b"\0" in data[:8192]: return 0, False text = data.decode("utf-8", errors="replace") best = run = 0 for line in text.splitlines(): if any(r.match(line) for r in DISASM_RES): run += 1 if run > best: best = run else: run = 0 return best, True def main(argv): if "--paths" in argv: files = argv[argv.index("--paths") + 1:] else: files = tracked_files() p_prefixes, p_globs = read_rules(PURGE, "purge") u_prefixes, u_globs = read_rules(UNTRACKED_AFTER, "untracked-after-rewrite") hashes, n_manifest, n_checks = rom_hashes() offenders = [] warnings = [] n_hashed = n_text = 0 top_runs = [] for rel in files: p = REPO / rel rule = under_rules(rel, p_prefixes, p_globs) if rule: offenders.append((rel, f"purge path ({rule})")) rule = under_rules(rel, u_prefixes, u_globs) if rule: offenders.append((rel, f"untracked-after-rewrite path ({rule})")) if not p.is_file(): # a submodule gitlink or a file deleted in the worktree — nothing to hash continue size = p.stat().st_size if size > SIZE_CAP: offenders.append((rel, f"{size:,} bytes > 100 MiB")) elif size > SIZE_WARN: warnings.append((rel, f"{size:,} bytes — within 20 MiB of GitHub's 100 MiB hard limit")) if size == 0: # the empty-file SHA1 is also a zero-length disc payload's (SC05/029/1.6) — not ROM bytes continue h = sha1_of(p) n_hashed += 1 if h in hashes: offenders.append((rel, f"ROM-derived content: sha1 == {hashes[h]}")) run, is_text = longest_disasm_run(p) if is_text: n_text += 1 if run: top_runs.append((run, rel)) if run >= RUN_CAP: offenders.append((rel, f"disassembly-shaped content: {run} contiguous lines (cap {RUN_CAP})")) top_runs.sort(reverse=True) print(f"audit_public: {len(files)} tracked paths, {n_hashed} files hashed against {len(hashes)} ROM hashes " f"({n_manifest} manifest rows + {n_checks} check.*.sha + redump), " f"{len(p_prefixes) + len(p_globs)} purge rules + {len(u_prefixes) + len(u_globs)} untracked-after-rewrite rules, " f"cap 100 MiB; {n_text} text files scanned for disassembly runs (cap {RUN_CAP} lines), " f"longest runs: {', '.join(f'{r} {p}' for r, p in top_runs[:3]) or 'none'}") for rel, why in warnings: print(f" WARN {rel}: {why}") if offenders: for rel, why in offenders: print(f" OFFENDER {rel}: {why}") print(f"audit_public: FAIL — {len(offenders)} offender(s) among {len(files)} tracked paths") return 1 print(f"audit_public: OK — 0 offenders among {len(files)} tracked paths" + (f" ({len(warnings)} size warning(s))" if warnings else "")) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))