diff --git a/Makefile b/Makefile index 661fcd426..6607a5345 100644 --- a/Makefile +++ b/Makefile @@ -310,6 +310,10 @@ tools-health: else echo "[skip] gitignore-template: decomp-architect/templates/gitignore.decomp does not exist yet (Phase 33.5 task 11)" fi + # P33.5 task 11: the day-one decomp kit stays free of this project's names/paths/addresses/rule numbers (fence-aware), + # honours its placeholder contract, and its scripts parse; the selftest is the R39 control (a planted leak MUST fail). + $(VENV_PY) tools/kit_lint.py --selftest + $(VENV_PY) tools/kit_lint.py $(VENV_PY) tools/xsig/tests/test_xsig.py 2>&1 | tail -1 | grep -q '^OK' && echo 'xsig tests: OK (8)' || { echo 'xsig tests: FAIL'; exit 1; } # Behavioural guards (P31 S70): tools-health audits DATA integrity; these assert that a tool # ACTUALLY DID the work it reports. A guard that is not running is not a guard (R54). diff --git a/decomp-architect/templates/.clang-format b/decomp-architect/templates/.clang-format new file mode 100644 index 000000000..df21c82a6 --- /dev/null +++ b/decomp-architect/templates/.clang-format @@ -0,0 +1,26 @@ +# .clang-format β€” the community's style for a matching decompilation, installed by decomp-architect (Step 8). +# Settings follow the published style guide of a large, community-run PlayStation decompilation, read as data on +# 2026-09-07: 4-space indentation, 80 columns, braces on the same line, the pointer on the type, braces on every +# conditional and loop body. `make format` applies it over src/; a bank is formatted before it is committed. +# (InsertBraces needs clang-format 15 or newer; older versions ignore the key.) +BasedOnStyle: LLVM +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ColumnLimit: 80 +BreakBeforeBraces: Attach +PointerAlignment: Left +DerivePointerAlignment: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +InsertBraces: true +SortIncludes: Never +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignTrailingComments: true +SpaceAfterCStyleCast: false +IndentCaseLabels: false +ReflowComments: false diff --git a/decomp-architect/templates/CLAUDE.decomp-overlay.md b/decomp-architect/templates/CLAUDE.decomp-overlay.md new file mode 100644 index 000000000..2bb1e9bad --- /dev/null +++ b/decomp-architect/templates/CLAUDE.decomp-overlay.md @@ -0,0 +1,34 @@ + + +## Decomp fail-safes (decomp-architect, Phase 0.5 β€” installed {{INSTALL_DATE}}) + +*Duplicated here so they survive even if the Session Start Protocol is skipped; the full rules are the registry's G-group.* + +- **Never commit game-derived bytes.** The dump, extracted payloads, generated disassembly, assets, build output, memory + images, the reverse-engineering database, the vendor SDK, transcripts that quote disassembly, and any text file that + pastes the target's instructions stay out of git β€” from the first commit, private or not. `config/firewall.txt` and + `tools/audit_public.py` enforce it; review `git status` before every commit. +- **A match is byte-for-byte AND the whole-binary hash stays green.** Never report a functionally-equivalent function, a + passing-looking build or any unverified outcome as done; verified from a clean rebuild, never incremental + (`{{FLEET_CHECK_CMD}}`); a build is verified by its exit code. +- **Never `git clean -x` / `git clean -fdx` in this tree.** The game-derived data is ignored-but-present; a `-x` clean deletes + the reverse-engineering database. `make clean` is the only clean. +- **The byte gate is the only claim.** "Banked" is written only from a tool's printed success line; names and types are + evidence-based, never guessed; outward text to third parties is written by a person. + +## Session-start extras (decomp-architect) + +- **The digest replaces the full PhaseEnd read once it exists.** When `phase-ends/DIGEST.md` exists, step 3 of the Session + Start Protocol reads the digest (every phase's synopsis) and the **three most recent** PhaseEnds in full, instead of every + PhaseEnd β€” the protocol's cost stays bounded as phases accumulate. Every PhaseEnd appends its synopsis to the digest. +- **Replay the checkpoint verbatim.** The `## πŸ›‘ SESSION CHECKPOINT` block at the end of `phase-ends/CURRENT_PHASE.md` is + reproduced in full in the session-start message, never summarised; it is the only in-phase context the session inherits. +- **The oracle ping only when the next task is reverse-engineering.** Verify the disassembler server with one cheap call + before any RE task β€” not merely because the phase contains RE tasks somewhere. After a server restart or a program switch, + pause and ask the developer to reconnect the client; the agent cannot. +- **The flywheel.** Before recurring matching work, consult `docs/{{COOKBOOK_NAME}}` by symptom (grep the index; never read a + large cookbook whole) and the pinned triple; after a hard-won match, feed the generalisable lesson back into both the + cookbook and the tooling. +- **Effort.** Max for the compiler pin, an address derivation, the segmentation decision, any wall verdict, the phase plan and + the PhaseEnd; breadth (isolated agents) for fleet-wide audits and bulk drafting; every transition prompted and waited for. diff --git a/decomp-architect/templates/CONTRIBUTING.skeleton.md b/decomp-architect/templates/CONTRIBUTING.skeleton.md new file mode 100644 index 000000000..5bff2b914 --- /dev/null +++ b/decomp-architect/templates/CONTRIBUTING.skeleton.md @@ -0,0 +1,48 @@ +# Contributing to {{PROJECT_NAME}} + +## Bring your own copy of the game + +Every contributor extracts from their own legally obtained copy. Nothing derived from it is committed β€” no executable, +payloads, disassembly, assets, build output, memory images, database, vendor SDK, or notes that paste the target's +instructions. The `.gitignore` firewall stops the ordinary case; `tools/audit_public.py` refuses the rest and runs in CI +on every push; review `git status` before every commit. + +## What "done" means + +A function is matched when its compiled output is instruction-identical to the original, register allocation included, +**and** the whole binary's hash check still passes with it linked. "Functionally equivalent" is never done. Unmatched C +lives under the non-matching guard with the assembly stub in the default build. A match is verified from a clean rebuild, +never an incremental one; a pull request states which compilation each claim survived (standalone / the real translation +unit / the whole binary). + +## Names, types and formatting + +A symbol is renamed only on recorded evidence β€” a string it prints, a cross-reference chain, a debug menu, a live-memory +datapoint, a community label with provenance. **If you are not sure what something does, leave it unnamed rather than +name it wrongly**; the address-named placeholder is honest and greppable. One definition per structure; a duplicate type +is a defect. Run `make format` before committing (`.clang-format` is the community's style); mark anything that exists +only to force a match with `// !FAKE:` and its reason; mark an original bug with `//! @bug`. + +## AI use β€” conduct + +{{AI_DISCLOSURE}} + +The rules that keep that honest, for every contributor and every agent: + +1. **The byte gate is the only claim of success.** Never report a match the gate has not proven; "it compiled" and "looks + equivalent" are not results. "Banked" is written from a tool's printed success line. +2. **Names and types are evidence-based, never guessed.** A model may propose a name; a person with recorded evidence asserts + one. Hallucinated meaning is the specific way a model damages a decompilation without any test catching it. +3. **Outward text is written by a person.** Issues, pull requests, forum posts and outreach to other projects are written + by their author the way a developer writes β€” short, plain, from the facts β€” never a model draft with the tells removed. + Before contributing to another project, read and follow its own AI-contribution policy. +4. **No automated traffic against community infrastructure.** Shared services are used by a person in a browser; anything + repetitive is replicated locally. +5. **Agents assist; a person owns.** Every change is justifiable by a person from the record (the phase logs, the decision + log, the cookbook's byte proofs). + +## Submitting + +Small, per-function or per-family pull requests; the hash check green on every binary the change touches; the clean +fleet verification for anything that touches a shared body, a shared header or the executable; a commit message that +names what was matched and how it was verified; no AI-attribution trailers. diff --git a/decomp-architect/templates/LICENSE.skeleton.md b/decomp-architect/templates/LICENSE.skeleton.md new file mode 100644 index 000000000..df2cad9b4 --- /dev/null +++ b/decomp-architect/templates/LICENSE.skeleton.md @@ -0,0 +1,7 @@ + + +LICENSE β€” replace this file with the verbatim text of: {{LICENSE_CHOICE}} diff --git a/decomp-architect/templates/NOTICE.src.md b/decomp-architect/templates/NOTICE.src.md new file mode 100644 index 000000000..b82e398d9 --- /dev/null +++ b/decomp-architect/templates/NOTICE.src.md @@ -0,0 +1,22 @@ +# NOTICE β€” the sources under `src/` + +The C files under `src/` are a **reimplementation of the executable code of *{{GAME_TITLE}}*** ({{PLATFORM}}, +{{GAME_SERIAL}}), written so that the game's own compiler turns them back into the original machine code byte for byte. +They exist for **study, interoperability and preservation**: to document how the game works, to make its code readable, +and to keep it buildable after the original tools and media are gone. + +*{{GAME_TITLE}}* and its code are **Β© TODO(publisher, year)**. The original work is theirs. This project is not +affiliated with, sponsored by or endorsed by the rights holder. + +**No license is asserted over the contents of `src/`.** They are derived from the copyrighted program and are published +as a reimplementation for the purposes above, in the manner of other matching decompilations of commercial games. Nothing +here grants anyone more rights in the original work than they already have, and nothing here should be read as a license +to distribute the game or its compiled code. The compiled output of these sources is, by design, identical to the +original binaries β€” distributing that output is distributing the game's code; don't. + +What this repository does **not** contain: the game's executable, its disc or cartridge data, any disassembly listing, +any memory image, or the vendor's SDK. Building requires your own copy of the game (see the top-level `README.md`). + +The rest of the repository β€” the tooling under `tools/`, the build system and the documentation under `docs/` β€” is the +project's own work and is licensed under {{LICENSE_CHOICE}} (`LICENSE`). Third-party components are listed with their +licenses in `THIRD_PARTY.md`. diff --git a/decomp-architect/templates/PLACEHOLDERS.md b/decomp-architect/templates/PLACEHOLDERS.md index e0225da07..499213fe4 100644 --- a/decomp-architect/templates/PLACEHOLDERS.md +++ b/decomp-architect/templates/PLACEHOLDERS.md @@ -16,7 +16,7 @@ | Placeholder | Filled from | Used in | |---|---|---| -| `{{PROJECT_NAME}}` | the installed `CLAUDE.md` title | the README/NOTICE/CONTRIBUTING skeletons, the docs and scratch READMEs | +| `{{PROJECT_NAME}}` | the installed `CLAUDE.md` title | the README and CONTRIBUTING skeletons, the docs and scratch READMEs | | `{{INSTALL_DATE}}` | today, in the same format ProjectArchitect used | every overlay's "installed" line | | `{{COOKBOOK_NAME}}` | the cookbook file ProjectArchitect created under `docs/` | the CLAUDE overlay's flywheel line, the cookbook overlay | | `{{DOMAIN_FAILSAFES}}` | **referenced only, never filled by the kit** β€” ProjectArchitect's own CLAUDE.md placeholder, which its generation step fills from the intake's Part D | `intake.decomp.md` Part D names it so the generation step knows which four fail-safes to write | @@ -26,12 +26,12 @@ | Placeholder | What it is | Used in | |---|---|---| | `{{GAME_TITLE}}` | the game's title | the intake, the README skeleton, the ops-setup overlay | -| `{{GAME_SERIAL}}` | region and serial (the identifier printed on the medium) | the intake, the README skeleton, the per-binary contract naming | -| `{{PLATFORM}}` | the console or platform | the intake, the README skeleton, `TODO(platform)` resolution | +| `{{GAME_SERIAL}}` | region and serial (the identifier printed on the medium) | the intake, the README and NOTICE skeletons, the ops-setup overlay | +| `{{PLATFORM}}` | the console or platform | the intake, the README and NOTICE skeletons, the ops-setup overlay, `TODO(platform)` resolution | | `{{TARGET_BINARY}}` | the main executable's file name on the medium | the ops-setup overlay, the firewall config's first purge path | -| `{{DUMP_PATH}}` | the absolute path of the developer's own dump β€” machine-local, never committed | `.claude/settings.local.json`-style machine-local notes in ops-setup only | -| `{{CONTAINER_LAYOUT}}` | one line: how code is packaged on the medium (archives, overlays, compression) | the intake, the docs README | -| `{{SDK_EVIDENCE}}` | the SDK/compiler-era evidence found (library version stamps, strings, a loader's detection) | the intake, the cookbook overlay's pinned context | +| `{{DUMP_PATH}}` | the absolute path of the developer's own dump β€” machine-local, never committed | the intake; the machine-local line of the ops-setup overlay only | +| `{{CONTAINER_LAYOUT}}` | one line: how code is packaged on the medium (archives, overlays, compression) | the intake, the docs README, the ops-setup overlay | +| `{{SDK_EVIDENCE}}` | the SDK/compiler-era evidence found (library version stamps, strings, a loader's detection) | the intake, the ops-setup overlay, the cookbook overlay's pinned context | | `{{COMPILER_FAMILY}}` | the compiler family the evidence suggests β€” a candidate set, not the pin | the intake, the ops-setup version-pins row (`TODO` until Phase 4 replaces it with `{{TOOLCHAIN_TRIPLE}}`) | | `{{COMMUNITY_WORK}}` | prior public work found, or "none found on " | the intake, the README skeleton's acknowledgements | | `{{PROJECT_GOALS}}` | the developer's stated goals, one paragraph | the intake, the README skeleton | diff --git a/decomp-architect/templates/README.skeleton.md b/decomp-architect/templates/README.skeleton.md new file mode 100644 index 000000000..b5ea43e70 --- /dev/null +++ b/decomp-architect/templates/README.skeleton.md @@ -0,0 +1,60 @@ +# {{PROJECT_NAME}} + +A matching decompilation of **{{GAME_TITLE}}** ({{PLATFORM}}, {{GAME_SERIAL}}): C source that, compiled with the original +era's toolchain and linked in the original order, reproduces every shipped binary byte for byte, verified by a hash check +inside every build. + +{{PROJECT_GOALS}} + +## Status + + +*Progress numbers are generated by `tools/progress.py` and appear here once Phase 5 wires it (TODO(phase-5)).* + +## Building from your own copy of the game + +This repository contains **no byte of the game**: no executable, no disc or cartridge data, no disassembly listing, no +memory image, no vendor SDK. To build, you need your own legally obtained copy. + +1. TODO(phase-1): stage your dump under `disks/` (ignored by git) and run the extractor; it verifies your extraction + against the committed manifest of hashes. +2. TODO(phase-3): `make build` reproduces each binary and checks its hash; the clean fleet verification is in + `docs/ops-setup.md`. +3. TODO(phase-9): the fresh-clone proof β€” a stranger with their own copy can extract, build and verify from this README alone. + +## The no-ROM policy + +Nothing derived from the game enters git β€” from the first commit, whether the repository is public or private. The +`.gitignore` is the firewall; `tools/audit_public.py` (its sources in `config/firewall.txt`) is the gate, run in CI on every +push; a pasted listing of the target's instructions counts too. See `docs/wiki/The-ROM-firewall.md` (TODO: created when +the wiki is set up) and `src/NOTICE.md`. + +## How this project is made + +{{AI_DISCLOSURE}} + +The standard it holds itself to: a function is "matched" only when the whole binary still hashes with it compiled from +source; names and types are evidence-based, never guessed (an address-named placeholder is honest); every outward message +to another project is written by a person; the project makes no automated requests to community services. The record β€” +the phase syntheses, the decision log, the cookbook of byte-proven idioms β€” is what makes the work auditable. + +## Repository visibility + +{{PUBLIC_OR_PRIVATE}} + +## Acknowledgements + +Prior community work this project builds on or was checked against: {{COMMUNITY_WORK}}. The governance and method come +from ProjectArchitect 2.0 and the decomp-architect kit distilled from a finished matching decompilation. + +## License + +The project's own work β€” the tooling, the build system, the documentation β€” is licensed under {{LICENSE_CHOICE}} +(`LICENSE`). **No license is asserted over the reimplemented sources under `src/`** (`src/NOTICE.md`). Third-party +components keep their own licenses (`THIRD_PARTY.md`). *{{GAME_TITLE}}* is the property of its rights holder. + +## Contributing + +See `CONTRIBUTING.md`: bring your own copy of the game, never commit anything derived from it, and read the AI-use +conduct section before your first pull request. diff --git a/decomp-architect/templates/audit_public.template.py b/decomp-architect/templates/audit_public.template.py new file mode 100644 index 000000000..5baefdc19 --- /dev/null +++ b/decomp-architect/templates/audit_public.template.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""audit_public.py β€” the first-push gate and the CI job: no game-derived bytes among the tracked files. +(Installed by decomp-architect Step 3 as tools/audit_public.py; a generalisation of the source project's audit.) + + tools/audit_public.py # every git-tracked file (git ls-files) + tools/audit_public.py --paths a b … # an explicit file list (the planted-fixture control; a pre-commit hook) + +Everything it forbids is DERIVED from config/firewall.txt β€” purge rules and hash sources β€” never from a typed list: + 1. PURGE PATHS β€” no tracked file lies under a `purge:` prefix or a `glob:` rule. + 2. CONTENT HASH β€” no tracked file's SHA1 appears in the hash set built from the `required:` / `fixture:` / resolvable + `pending:` sources (a `.jsonl` manifest with a `sha1` key per line, or sha1sum-format checksum files, globs allowed). + A renamed copy is caught by content. Zero-length files are exempt by content (an empty payload shares the empty + file's hash). + 3. SIZE β€” no tracked file over the host's warning threshold (50 MiB). + 4. DISASSEMBLY-SHAPED CONTENT β€” no tracked TEXT file carries a long contiguous run of lines shaped like an assembler + listing, an objdump, a splitter's address/word comment or a label block: the case a path-and-hash audit cannot + see β€” a notes file that pastes a function's instructions is game-derived even when the tracked C reproduces the + bytes. The criterion is the LONGEST CONTIGUOUS run per file (cap 64 lines), so a short quoted diff passes. +Coverage is asserted: a missing `required:` source, zero resolvable sources, or zero purge rules is a FAILURE, never a +pass; a missing `pending:` source is warned loudly. Every count prints with its denominator; exit 1 names every offender. +Negative control: the installer plants the fixture blob and asserts this audit FAILS on it before trusting a PASS. +CI runs it without the game: it reads only tracked text and hashes tracked files. +""" +import fnmatch +import glob as globmod +import hashlib +import json +import pathlib +import re +import subprocess +import sys + +REPO = pathlib.Path(__file__).resolve().parent.parent +CONFIG = REPO / "config" / "firewall.txt" +SIZE_CAP = 50 * 1024 * 1024 +RUN_CAP = 64 # contiguous disassembly-shaped lines that make a text file an offender +DISASM_RES = [ + # asm-differ-style: `12: addiu $sp, $sp, -0x40`, `44: nop`, `50: j .Llabel` β€” an operand-less mnemonic + # and a `.L` label operand both continue the run + re.compile(r"^\s*\d+:\s+[a-z]{2,8}(?:\.[a-z]+)?(?:\s+(?:\$|-?0x|-?\d|[a-z_.])|\s*$)"), + re.compile(r"^\s*[0-9a-f]+:\s+[0-9a-f]{8}\s+\w+"), # objdump: `0: 27bdffc0 addiu sp,sp,-64` + re.compile(r"^\s*/\* [0-9A-F]{4,} [0-9A-F]{8} [0-9A-F]{8} \*/"), # splitter: `/* OFFSET VRAM WORD */` + re.compile(r"^(?:glabel|dlabel)\s"), +] + + +def read_config(): + """Returns (prefixes, globs, required, pending, fixtures). Refuses an absent or empty config (a tool must refuse).""" + if not CONFIG.exists(): + sys.exit(f"audit_public: {CONFIG.relative_to(REPO)} is missing β€” nothing to derive the forbidden set from") + prefixes, globs, required, pending, fixtures = [], [], [], [], [] + for raw in CONFIG.read_text(encoding="utf-8").splitlines(): + ln = raw.split("#", 1)[0].strip() + if not ln: + continue + kind, _, value = ln.partition(":") + kind, value = kind.strip(), value.strip() + if kind == "purge": + prefixes.append(value) + elif kind == "glob": + globs.append(value) + elif kind == "required": + required.append(value) + elif kind == "pending": + pending.append(value) + elif kind == "fixture": + fixtures.append(value) + else: + sys.exit(f"audit_public: unknown rule kind {kind!r} in {CONFIG.relative_to(REPO)} β€” refusing to guess") + if not prefixes and not globs: + sys.exit("audit_public: no purge rules in config/firewall.txt β€” refusing to pass on an empty forbidden set") + return prefixes, globs, required, pending, fixtures + + +def under_rules(path, prefixes, globs): + for p in prefixes: + if path == p or path.startswith(p if p.endswith("/") else p + "/"): + return p + for g in globs: + if fnmatch.fnmatchcase(path, g): + return "glob:" + g + return None + + +def load_hash_source(rel): + """Every SHA1 a source declares. `.jsonl` = one object per line with a `sha1` key; anything else = sha1sum lines.""" + out = {} + p = REPO / rel + if p.suffix == ".jsonl": + for ln in p.read_text(encoding="utf-8").splitlines(): + if ln.strip(): + o = json.loads(ln) + out[o["sha1"].lower()] = f"{rel}:{o.get('path', '?')}" + else: + for ln in p.read_text(encoding="utf-8").splitlines(): + parts = ln.split() + if len(parts) >= 2 and len(parts[0]) == 40: + out[parts[0].lower()] = f"{rel}:{parts[1]}" + return out + + +def rom_hashes(required, pending, fixtures): + """The forbidden hash set, coverage-asserted: every required/fixture source exists and is non-empty; a pending one + that exists is used, a missing one is warned; zero resolvable sources is a failure.""" + hashes, n_sources = {}, 0 + for kind, sources in (("required", required), ("fixture", fixtures), ("pending", pending)): + for pattern in sources: + matches = sorted(globmod.glob(str(REPO / pattern))) if any(c in pattern for c in "*?[") else [str(REPO / pattern)] + matches = [m for m in matches if pathlib.Path(m).is_file()] + if not matches: + if kind == "pending": + print(f"audit_public: WARNING pending hash source {pattern} does not exist yet β€” promote it when its phase creates it") + continue + sys.exit(f"audit_public: {kind} hash source {pattern} is missing β€” the forbidden set cannot be derived") + for m in matches: + rel = pathlib.Path(m).relative_to(REPO).as_posix() + found = load_hash_source(rel) + if not found and kind != "pending": + sys.exit(f"audit_public: {kind} hash source {rel} holds no hashes β€” refusing to pass on an empty source") + hashes.update(found) + n_sources += 1 + if n_sources == 0: + sys.exit("audit_public: zero resolvable hash sources β€” refusing to pass vacuously") + return hashes, n_sources + + +def tracked_files(): + r = subprocess.run(["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True) + return [p for p in r.stdout.decode("utf-8", "surrogateescape").split("\0") if p] + + +def sha1_of(path): + h = hashlib.sha1() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def longest_disasm_run(path): + """The longest contiguous run of disassembly-shaped lines in a text file; (0, False) for a binary file.""" + with open(path, "rb") as f: + data = f.read() + if b"\0" in data[:8192]: + return 0, False + text = data.decode("utf-8", errors="replace") + best = run = 0 + for line in text.splitlines(): + if any(r.match(line) for r in DISASM_RES): + run += 1 + best = max(best, run) + else: + run = 0 + return best, True + + +def main(argv): + files = argv[argv.index("--paths") + 1:] if "--paths" in argv else tracked_files() + prefixes, globs, required, pending, fixtures = read_config() + hashes, n_sources = rom_hashes(required, pending, fixtures) + offenders, top_runs = [], [] + n_hashed = n_text = 0 + for rel in files: + p = REPO / rel + rule = under_rules(rel, prefixes, globs) + if rule: + offenders.append((rel, f"purge path ({rule})")) + if not p.is_file(): # a submodule gitlink or a file deleted in the worktree β€” nothing to hash + continue + size = p.stat().st_size + if size > SIZE_CAP: + offenders.append((rel, f"{size:,} bytes > 50 MiB")) + if size == 0: + continue + h = sha1_of(p) + n_hashed += 1 + if h in hashes: + offenders.append((rel, f"game-derived content: sha1 == {hashes[h]}")) + run, is_text = longest_disasm_run(p) + if is_text: + n_text += 1 + if run: + top_runs.append((run, rel)) + if run >= RUN_CAP: + offenders.append((rel, f"disassembly-shaped content: {run} contiguous lines (cap {RUN_CAP})")) + top_runs.sort(reverse=True) + print(f"audit_public: {len(files)} paths, {n_hashed} files hashed against {len(hashes)} forbidden hashes from " + f"{n_sources} source(s), {len(prefixes) + len(globs)} purge rules, cap 50 MiB; {n_text} text files scanned for " + f"disassembly runs (cap {RUN_CAP} lines), longest runs: {', '.join(f'{r} {p}' for r, p in top_runs[:3]) or 'none'}") + if offenders: + for rel, why in offenders: + print(f" OFFENDER {rel}: {why}") + print(f"audit_public: FAIL β€” {len(offenders)} offender(s) among {len(files)} paths") + return 1 + print(f"audit_public: OK β€” 0 offenders among {len(files)} paths") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/decomp-architect/templates/bootstrap.template.sh b/decomp-architect/templates/bootstrap.template.sh new file mode 100644 index 000000000..0e003b7a1 --- /dev/null +++ b/decomp-architect/templates/bootstrap.template.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# tools/bootstrap.sh β€” fresh-clone setup (installed by decomp-architect Step 5 as a SKELETON; each TODO is a phase task). +# Idempotent; never sudo. What a build needs, in order: +# 1. system packages β€” only CHECKED here: the missing ones are printed as one install line. +# 2. the Python venv β€” created from the pinned requirements file. +# 3. the submodules β€” the assembler shim, the differ, the decompiler, the permuter (Phase 3/4 pin them). +# 4. the vintage compiler β€” fetched or extracted from a tracked, checksum-verified archive (Phase 4). +# 5. the preflight β€” `make check-env`; its exit status is this script's (Phase 3). +# Then: stage your own dump under disks/ and run the extract + fleet-check commands from docs/ops-setup.md. +set -euo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" +say() { printf 'bootstrap: %s\n' "$*"; } + +# 1) system packages β€” presence only; print the install line, never run it. +# TODO(phase-3): fill the list once the toolchain is chosen (binutils for the target, a C preprocessor, clang-format, make, +# the archive tools the extractor needs, python3-venv). TODO(platform): the package names differ per target and per distro. +PKGS="git make python3-venv clang-format" +if command -v dpkg >/dev/null 2>&1; then + missing=() + for p in $PKGS; do dpkg -s "$p" >/dev/null 2>&1 || missing+=("$p"); done + if ((${#missing[@]})); then + say "MISSING packages (${#missing[@]}) β€” run this, then re-run bootstrap:" + printf ' sudo apt-get install -y %s\n' "${missing[*]}" + else + say "packages: all present" + fi +else + say "no dpkg on this system β€” install the equivalents of: $PKGS" +fi + +# 2) the venv (pinned requirements; the file is created in Phase 1 with the extractor's dependencies) +if [ ! -x .venv/bin/python ]; then + say "creating .venv" + python3 -m venv .venv +fi +if [ -f requirements-python.txt ]; then + say "installing pinned Python requirements (no-op when satisfied)" + .venv/bin/pip install -q -r requirements-python.txt +else + say "requirements-python.txt not present yet β€” TODO(phase-1)" +fi + +# 3) submodules (no-op when populated; none until Phase 3 adds the shim, the differ, the decompiler and the permuter) +if [ -f .gitmodules ]; then + say "submodules: git submodule update --init" + git submodule update --init +else + say "no submodules yet β€” TODO(phase-3)" +fi + +# 4) the vintage compiler β€” TODO(phase-4): verify the tracked archive's checksum (`sha256sum --check`), extract each +# candidate into its OWN directory, and print the pinned triple from docs/ops-setup.md. Never download without a +# checksum to verify against; never vendor a compiler whose license forbids it (keep a fetch step + checksum instead). +say "vintage compiler: TODO(phase-4) β€” nothing to fetch until the candidate ladder exists" + +# 5) the preflight (its exit status is ours) β€” TODO(phase-3): `make check-env` asserts the toolchain executes, the +# assembler version is the pinned one and the dump's hash matches. +if grep -q '^check-env:' Makefile 2>/dev/null; then + say "make check-env" + make --no-print-directory check-env +else + say "make check-env not present yet β€” TODO(phase-3); bootstrap ends here" +fi diff --git a/decomp-architect/templates/docs-README.md b/decomp-architect/templates/docs-README.md new file mode 100644 index 000000000..b86fc8f59 --- /dev/null +++ b/decomp-architect/templates/docs-README.md @@ -0,0 +1,33 @@ +# `docs/` β€” what goes where + +*Installed by decomp-architect (Step 4) on {{INSTALL_DATE}} for {{PROJECT_NAME}}. This folder holds the records and references the +project's wiki summarises and links; knowledge has a home by KIND, not by the session that produced it. A note written for +one session is scratch (`.run/`); the durable result it produced goes into one of the files below, and the note is archived.* + +| Kind of knowledge | Where it goes | The rule behind it | +|---|---|---| +| A compiler idiom proven on the bytes (the residual, the mechanism, the lever, the byte proof) | a numbered section of the cookbook, and its symptom index (regenerated by a tool, never edited) | the flywheel: consult before a match, feed back after | +| A strategic pivot: what was believed, what failed, the measurement, the hindsight | an entry in `docs/decision-log.md`, written while fresh | capture the why while it hurts | +| A late discovery that would have sped up an earlier phase β€” what it is, when it was found, when it *could* have been found, what it would have saved | `docs/accelerators.md` | the ledger a future project starts from | +| A procedure people run (the wave, the publication) | a runbook under `docs/`; a superseded runbook carries a banner at the top and is then archived | one runbook is the procedure | +| An environment or tool fact (a version, a flag, a hook, a row per tool) | `docs/ops-setup.md`, in the same change as the tool | keep the ops reference current | +| An address, with its source and its verification status | `docs/memory-map.md` | address provenance and region tags | +| A file or container format | `docs/formats.md` (the medium's layout: {{CONTAINER_LAYOUT}}) | β€” | +| The state of the open phase; then the phase's synthesis; then the one-page digest every session starts from | `phase-ends/CURRENT_PHASE.md` β†’ `phase-ends/PhaseEnd_Phase.md` β†’ `phase-ends/DIGEST.md` | the replayable checkpoint; the digest | +| How the project is used: building, verifying, contributing, its layout and conventions | the wiki (`docs/wiki/`), the source of truth for documentation | β€” | +| A design note, a frontier analysis, a triage ladder, a worklist for one session | `.run//` while live; once its result is in the record above, the note moves to the archive β€” it is never a reference | see the archive | + +**Authored versus generated.** Numbers are generated, never typed: every progress figure, badge, timeline row and census +in a published document is produced by a tool from the tree, and the tool asserts the published copy is fresh; a +generated file says so on its first line; edits go to the generator. A number that has to appear in prose is a dated +snapshot with the command that produced it. Snapshots of a state that no longer exists are frozen, not regenerated. + +**Links.** A document links the wiki page for a topic, not the `docs/` file behind it; a wiki page links into `docs/` only +through its Reference index; nothing links into the archive (`docs/sunset/`), whose index names files as backticked paths +with the version they were archived at. A backticked path is a citation, not a link; a link checker classifies cited +paths as TRACKED or UNTRACKED by asking git, never the disk. + +**The archive.** A document leaves `docs/` when its purpose is fulfilled and its information lives elsewhere: `git mv` into +`docs/sunset/` under the same relative path, one row in the archive index (what it was, what came of it, where it lives +now), and a review row for the owner; deletion is the owner's decision, never part of the move; every referrer is +re-pointed first (a `git grep` over the tree must return only the two index files). diff --git a/decomp-architect/templates/firewall-fixture/README.md b/decomp-architect/templates/firewall-fixture/README.md new file mode 100644 index 000000000..eac7969f7 --- /dev/null +++ b/decomp-architect/templates/firewall-fixture/README.md @@ -0,0 +1,10 @@ +# The firewall fixture β€” the audit's negative control + +`blob.bin` is sixteen synthetic bytes (`printf 'DECOMP-FIXTURE!!'`), derived from no game; `blob.sha1` is its SHA1 in +`sha1sum` format. The kit's installer (Step 3) copies **only the `.sha1` file** into the new repository as +`config/firewall-fixture.sha1` and lists it in `config/firewall.txt` as a `fixture:` hash source, so the audit has at least +one resolvable source before the extraction manifest exists. The control then **plants** a copy of the blob under scratch +(`.run/firewall-control/planted.bin`), runs the audit on that path alone, asserts it FAILS naming the planted file, removes +the copy, and asserts the tree PASSES. An audit that has not failed on the fixture is not trusted to pass (the source +project's negative-control rule). The blob itself is never tracked in the new repository β€” the kit's package folder is +gitignored after install. diff --git a/decomp-architect/templates/firewall-fixture/blob.bin b/decomp-architect/templates/firewall-fixture/blob.bin new file mode 100644 index 000000000..71a42bf56 --- /dev/null +++ b/decomp-architect/templates/firewall-fixture/blob.bin @@ -0,0 +1 @@ +DECOMP-FIXTURE!! \ No newline at end of file diff --git a/decomp-architect/templates/firewall-fixture/blob.sha1 b/decomp-architect/templates/firewall-fixture/blob.sha1 new file mode 100644 index 000000000..8a8e88ce0 --- /dev/null +++ b/decomp-architect/templates/firewall-fixture/blob.sha1 @@ -0,0 +1 @@ +d4bc7b5d67878461ceac039b458b0f9db0e6adaa blob.bin diff --git a/decomp-architect/templates/firewall.txt b/decomp-architect/templates/firewall.txt new file mode 100644 index 000000000..54ff141ac --- /dev/null +++ b/decomp-architect/templates/firewall.txt @@ -0,0 +1,37 @@ +# config/firewall.txt β€” the ROM audit's sources, in ONE file (installed by decomp-architect Step 3). +# The audit (tools/audit_public.py) derives everything it forbids from the lines below; it never carries a typed list. +# +# purge: a path that may never be tracked (the rewrite's purge set reads the same lines, so the +# audit and any future history rewrite can never disagree about what is forbidden) +# required: a hash source that MUST exist: a `.jsonl` manifest (one object per line with a `sha1` +# key) or a checksum file (` ` lines, sha1sum format). Missing = the audit FAILS. +# pending: a hash source a later phase creates; missing = a loud warning, never a pass on its own. +# Promote it to `required:` in the phase that creates it (the phase ladder names which). +# fixture: the planted-fixture control's sha1 (Step 3); counts as a required source. +# +# The audit refuses to run with zero resolvable hash sources or zero purge rules (a tool must refuse, never pass vacuously). + +# ---- purge rules: the nine classes of the ROM firewall (mirror .gitignore's paths) ---- +purge: disks/ +purge: {{TARGET_BINARY}} +purge: extracted/ +glob: extracted/** +purge: asm/ +purge: assets/ +purge: build/ +purge: expected/ +purge: dumps/ +glob: dumps/**/*.bin +purge: ghidra/ +purge: tools/psyq/ +purge: session-archive/ +purge: datasets/ +purge: models/ +glob: **/*.gguf +glob: **/*.safetensors + +# ---- hash sources ---- +fixture: config/firewall-fixture.sha1 +pending: extracted/retail/manifest.jsonl # TODO(phase-1): the extraction manifest β€” promote to required: +pending: config/medium.sha1 # TODO(phase-1): the medium's own track/image hash β€” promote to required: +pending: config/check.*.sha # TODO(phase-3): the per-binary contracts (a glob) β€” promote to required: diff --git a/decomp-architect/templates/gitignore.decomp b/decomp-architect/templates/gitignore.decomp new file mode 100644 index 000000000..d8e973b80 --- /dev/null +++ b/decomp-architect/templates/gitignore.decomp @@ -0,0 +1,71 @@ +# ============================================================ +# ROM firewall β€” in force from the FIRST commit; no private-repository exemption, ever. +# The repository ships NO game-derived bytes and NO vendor SDK. Symbol names, addresses, +# hashes, configuration and the decompiled C are what a decompilation publishes. +# ============================================================ + +# 1. The disc / ROM dump β€” never committed; your own copy goes here +/disks/ + +# 2. Extracted payloads β€” regenerated from YOUR dump by the extractor and verified against the +# committed manifest of hashes, the only thing tracked under extracted/ +/extracted/* +!/extracted/retail/ +/extracted/retail/* +!/extracted/retail/manifest.jsonl +!/extracted/retail/manifest.sha1 + +# 3. Generated disassembly, assets and build output β€” `make extract` / `make build` recreate them; +# the durable record is the symbol file. (A pasted listing of target instructions is class 3 too.) +/asm/ +/assets/ +/build/ +/expected/ +/undefined_syms_auto.txt +/undefined_funcs_auto.txt + +# 4. Memory images β€” local only; INDEX.md + CHECKSUMS.sha1 stay tracked and describe them +/dumps/*.bin + +# 5. The reverse-engineering database β€” embeds the program's bytes; tracked as a TEXT export +# (config/ghidra/) with a rebuild script that proves the round trip. NEVER `git clean -x` here. +/ghidra/ + +# 6. The vendor SDK β€” user-supplied, optional, never distributed; its checksums are tracked +/tools/psyq/ + +# 7. Session transcripts β€” they quote the target's disassembly; keep them out of any public tree +/session-archive/ + +# 8. Training data and weights derived from the target +/datasets/ +/models/ +*.gguf +*.safetensors + +# 9. Re-downloadable binaries β€” a sha256 in the ops reference and a fetch step, not a commit. +# (A vendored compiler tarball may be tracked only if its license allows; keep its checksum beside it.) +/tools/bin/* +!/tools/bin/*.sha256 +/tools/ghidra-ext/*.zip +*.tar.gz + +# ---- Scratch: ignored by CONTENTS (/.run/* not /.run/) so that dated `!` exceptions can re-include ---- +# ---- irreplaceable work. Rule of thumb: commit what a rerun CANNOT reproduce (hand analysis, ---- +# ---- harnesses, ledgers, the recorded contract run); leave what a script regenerates ignored. ---- +/.run/* +# One dated block per exception β€” re-include the directory, re-exclude its contents, re-include +# the wanted files β€” under a comment naming the phase, the session and the rule. For example: +# # P (, ): the recorded contract run's per-step logs. Evidence, tracked. +# !/.run/P/ +# /.run/P/* +# !/.run/P/verify/ +# /.run/P/verify/* +# !/.run/P/verify/*.log + +# ---- Toolchain caches, research clones and editor noise ---- +/.venv/ +__pycache__/ +/tools/reference/ +.vscode/ +Thumbs.db diff --git a/decomp-architect/templates/make-format.snippet.mk b/decomp-architect/templates/make-format.snippet.mk new file mode 100644 index 000000000..3d0ed856b --- /dev/null +++ b/decomp-architect/templates/make-format.snippet.mk @@ -0,0 +1,12 @@ +# make-format.snippet.mk β€” appended to the project's Makefile by decomp-architect (Step 8). +# `make format` rewrites every C source and header under src/ with the tracked .clang-format; `make format-check` +# is the same run in dry mode with warnings as errors (a CI-able check, no game bytes needed). Dotfiles under src/ +# are excluded so a tool's live probe file is never formatted into the tree. + +.PHONY: format format-check + +format: + find src -type f \( -name '*.c' -o -name '*.h' \) -not -name '.*' -print0 | xargs -0 -r clang-format -i --style=file + +format-check: + find src -type f \( -name '*.c' -o -name '*.h' \) -not -name '.*' -print0 | xargs -0 -r clang-format --dry-run --Werror --style=file diff --git a/decomp-architect/templates/no-rom.template.yml b/decomp-architect/templates/no-rom.template.yml new file mode 100644 index 000000000..9ace5b9d7 --- /dev/null +++ b/decomp-architect/templates/no-rom.template.yml @@ -0,0 +1,44 @@ +# .github/workflows/no-rom.yml β€” the ROM-free CI (installed by decomp-architect Step 3). +# +# What CI can and cannot prove here: the project's contract is BYTE-IDENTITY of every binary against the original +# medium, which needs the game and is therefore never in CI. CI proves everything that does NOT need the game: +# * audits β€” the tracked tree is public-clean (tools/audit_public.py: no game-derived bytes, no purge path, +# nothing over the size threshold, no pasted disassembly), and any text-only self-checks the +# project adds (a cookbook index freshness check, a symbol-reference lint, decoder unit tests). +# * compile-only β€” the committed C still compiles with the pinned vintage toolchain and the build's exact flags. +# TODO(phase-3): enable once the toolchain is pinned and a compile-only driver exists; scope a PR run +# to a few representative binaries and the whole fleet to the weekly schedule. +# Byte-identity is verified locally with the game and RECORDED in the tracked verification log. +name: no-rom + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: "17 6 * * 1" # weekly: the full-fleet compile-only run, once it exists + workflow_dispatch: + +permissions: + contents: read + +jobs: + audits: + name: public-clean audit (no game bytes) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: audit_public β€” no game-derived bytes, no purge path, nothing > 50 MiB, no pasted disassembly + run: python tools/audit_public.py + # TODO(phase-5): add the project's text-only self-checks here as they are built (index freshness, symbol lint, + # decoder tests) β€” each one a derived check that asserts its own coverage. + + # TODO(phase-3): compile-only β€” every eligible translation unit with the pinned toolchain (no game bytes needed). + # steps: checkout with submodules β†’ install the assembler/binutils β†’ verify the vintage compiler's checksum and + # extract it β†’ run the compile-only driver (PR scope on push/PR; the whole fleet on schedule / dispatch). diff --git a/decomp-architect/templates/ops-setup.decomp.md b/decomp-architect/templates/ops-setup.decomp.md new file mode 100644 index 000000000..60ebbf479 --- /dev/null +++ b/decomp-architect/templates/ops-setup.decomp.md @@ -0,0 +1,59 @@ + + +## Decomp environment (decomp-architect, Phase 0.5 β€” installed {{INSTALL_DATE}}) + +### Version pins (decomp) + +| Component | Version | Notes / why pinned | +|---|---|---| +| The pinned toolchain triple (compiler β†’ assembler shim β†’ binutils, with flags) | {{TOOLCHAIN_TRIPLE}} | TODO(phase-4): pinned by fingerprint evidence down the candidate ladder; the assembler's compatibility version is always passed explicitly β€” a shim's default is not "latest" | +| Candidate compiler family (from the SDK evidence) | {{COMPILER_FAMILY}} | the candidate set the pin phase runs down; never a sibling project's triple | +| The splitter / disassembler and its config | TODO(phase-3) | version pinned in the bootstrap script | +| The disassembler database and its agent server | TODO(phase-2) | the static oracle; the database is tracked as a TEXT export with a rebuild script | +| The emulator and its scripting bridge | TODO(phase-2) | the runtime oracle | + +### The game and the medium + +- **Title / platform / serial:** {{GAME_TITLE}} Β· {{PLATFORM}} Β· {{GAME_SERIAL}} +- **The main executable on the medium:** `{{TARGET_BINARY}}` (its hash is the first per-binary contract, Phase 3) +- **Container layout:** {{CONTAINER_LAYOUT}} +- **SDK / compiler-era evidence:** {{SDK_EVIDENCE}} +- **The dump (machine-local, never committed):** `{{DUMP_PATH}}` β€” copied once onto a fast local filesystem under `disks/` + (ignored); the extractor reads it, nothing builds against it. + +### Build / extract / verify (decomp) + +``` +# extract the medium and verify against the committed manifest +{{EXTRACT_CMD}} +# the clean fleet verification β€” every binary from clean β†’ extract β†’ build, exit code read +{{FLEET_CHECK_CMD}} +``` + +- **The gate:** a binary is green only when its hash check inside `make build` passes; a match is verified from a CLEAN + rebuild, never incremental; the executable is gated only by a clean rebuild; a build is verified by its exit code. + +### The oracles (decomp) + +- **Disassembler MCP:** {{DISASSEMBLER_MCP}} β€” verify with one cheap call before any reverse-engineering task; after a + restart or a program switch, pause and ask the developer to reconnect the client. +- **Emulator bridge:** {{EMULATOR_BRIDGE}} β€” a live-memory finding is verified only with three or more consistent datapoints + or a controlled before/after diff. + +### Git posture (decomp) + +- **Visibility at day one:** {{PUBLIC_OR_PRIVATE}} β€” the ROM firewall applies either way (`config/firewall.txt`, + `tools/audit_public.py`, the CI workflow). If ever private, a later flip is gated on the host's object store, never on + a clean tree. +- Never `git clean -x` in this tree (the game-derived data is ignored-but-present); the backup of the reverse-engineering + work is the text export + the checksum files + a private archive repository, not the ignored directories. + +### Tooling inventory (decomp) + +| Tool | Location | Purpose | +|---|---|---| +| `tools/audit_public.py` | `tools/` | the ROM audit (purge paths, the derived hash set, the size cap, the pasted-disassembly check); the first-push gate and the CI job; its sources are `config/firewall.txt` | +| `make format` | `Makefile` | clang-format over `src/` with the tracked `.clang-format` (the community style) | +| TODO(phase-1): the extractor, the manifest | `tools/` | β€” | diff --git a/decomp-architect/templates/pa-overlays.md b/decomp-architect/templates/pa-overlays.md new file mode 100644 index 000000000..f828ca683 --- /dev/null +++ b/decomp-architect/templates/pa-overlays.md @@ -0,0 +1,235 @@ +# pa-overlays.md β€” the fenced blocks decomp-architect appends to ProjectArchitect's stamped files + +> **How SETUP uses this file.** Each block below is headed by its **target file**, its **marker** (the heading line the +> installer greps for to make the step idempotent) and the **step** that applies it. The installer appends the block's +> body verbatim to the target (or creates the target when the block says so), fills the copy-time placeholders, and +> writes literal `TODO(phase-N)` text where a generation-time value does not exist yet. Nothing already in a target is +> edited β€” ProjectArchitect's stamped content stays byte-identical above the marker. The blocks are the shape the source +> project's governance converged on over thirty-three phases, de-specialised. + +--- + +## Block 1 β€” `phase-ends/DIGEST.md` (CREATE at Step 8; the session-start digest) + +Marker: `# phase-ends/DIGEST.md β€” the session-start digest` + +````markdown +# phase-ends/DIGEST.md β€” the session-start digest (every phase in one page) + +> **Purpose.** The Session Start Protocol reads the constitution, the registry (every rule in full), THIS file, the +> three most recent PhaseEnds in full, and `CURRENT_PHASE.md`; it does not read every PhaseEnd once more than three +> exist. This file therefore carries a synopsis of every phase and the corrections that supersede parts of the +> permanent constitution. The PhaseEnd files stay the durable record β€” read an older one on demand when a synopsis +> is not enough. **Maintenance:** at every PhaseEnd, append that phase's synopsis here (a phase-close checkbox). +> This file is DERIVED from the PhaseEnds and may be corrected; the constitution is never edited. +> Installed by decomp-architect on {{INSTALL_DATE}}. + +## 0. Where the project stands (at the latest PhaseEnd β€” the live state is in CURRENT_PHASE.md) +(one paragraph, rewritten at every PhaseEnd: the last milestone met, the numbers as generated, what is next) + +## 1. Corrections and supersessions of the constitution recorded in PhaseEnds +(one bullet per correction: what the constitution says, what is true now, which PhaseEnd recorded it) + +## 2. Phase synopses (what each phase delivered, its key finding, the rules it added) +**P0 (date, version) Governance install + the decomp kit.** … +(one paragraph per phase, appended at its close) + +## 3. Where things live (the document map a session needs) +(the wiki's reference index first; then the files a session touches most: the ops reference, the cookbook and its +index, the codegen map, the wave runbook, the effort map, the decision log, the accelerators, the address ledger, +the formats, the pinned-walls list, the archived worklogs β€” on demand only) +```` + +--- + +## Block 2 β€” `phase-ends/CURRENT_PHASE.md` (APPEND to ProjectArchitect's template at Step 8; the replayable checkpoint) + +Marker: `## πŸ›‘ SESSION CHECKPOINT` + +*Appended to `phase-ends/CURRENT_PHASE.template.md` so every phase's working file ends with it; the block is REWRITTEN +after every task (the per-task log entry above it is appended, the block is replaced). A fresh session replays it +verbatim and inherits nothing else.* + +````markdown +## πŸ›‘ SESSION CHECKPOINT β€” Tasks [done] βœ“; NEXT = task [N] ([one line]; **[effort]** β€” prompt the developer) + +### 0. How to use this block +You are a FRESH SESSION that has read the constitution, the registry, `phase-ends/DIGEST.md`, the three most recent +PhaseEnds and this file, and nothing else. Replay this block verbatim, state phase / done / NEXT / effort, recite the +rules, then WAIT for the developer. Rebuild the harness task list from the checklist above, marking the done tasks +completed and task [N] in progress. + +### 1. Where we are +[Phase, version, gate-1 date and effort setting; baseline HEAD and every task's commit hash; the fleet's green count +from the last clean run ("tree-clean" and "fleet-green" are different invariants β€” quote both); what is tracked and +what is scratch; what is ALREADY banked for later tasks (so nothing is redone); what is running in the background +(or "nothing"); the session number; the developer's effort at the end of the last session and the effort the next +task needs.] + +### 2. What NEXT does (task [N], **[effort]**) β€” exact steps +Read first: [every file, with the section to read and why]. +Write / run: [numbered steps with the exact commands, their gotchas, and the expected outputs]. +Verify: [the exact checks and their expected literal results]. Log + checkpoint; commit by explicit path. [The effort +transition after this task, if any β€” prompt the developer.] + +### 3. Standing facts for every task of this phase +- One commit per task, after this file's log line; commit by explicit path; no trailers; the developer pushes. +- [the phase's invariants: what is never edited, what is never run, the checks that must stay green, the vocabulary] +```` + +--- + +## Block 3 β€” `phase-ends/PhaseEnd.template.md` (APPEND at Step 8; the narrative axis) + +Marker: `## What we believed, what failed and why it looked right, what it cost, and what we would do sooner` + +*Appended to ProjectArchitect's PhaseEnd template between "Rules Added This Phase" and "PhaseEnd Changelog". The +source project's retrospective was rebuilt from exactly this section of every PhaseEnd because the transcripts of its +first month were lost; the terse "what changed" record survives a context boundary, the WHY does not.* + +````markdown +## What we believed, what failed and why it looked right, what it cost, and what we would do sooner + +*(For the retrospective. Narrative, not a table; every cost with its denominator; the detail is in the decision log.)* + +- **Believed:** [what the phase assumed at its plan]. **True / false:** [what was actually the case]. **Sooner:** [what + would have found it earlier β€” a control, a tool, an order of work]. +- **Failed, and why it looked right:** [the failure; the reason it was not caught the same day]. +- **Cost:** [tokens, sessions, rework β€” of what]. +```` + +--- + +## Block 4 β€” `docs/effort-map.md` (APPEND at Step 8; the decomp per-phase rows) + +Marker: `## Per-phase effort map (decomp ladder β€” decomp-architect)` + +````markdown +## Per-phase effort map (decomp ladder β€” decomp-architect, installed {{INSTALL_DATE}}) + +*The Max shortlist is the set of judgments whose silent error would poison everything downstream; the rest of a phase +runs at the baseline; breadth is the same analysis across many independent items.* + +| Phase | Mandatory-Max tasks | Fine at xHigh | Breadth (fan-out) | +|---|---|---|---| +| Phase Start (any) | the plan itself (always Max) | β€” | wide surveys feeding the plan | +| 1 extraction + manifest | the container/compression semantics when they are ambiguous | the extractor, the manifest, the cross-validation | a fleet-wide format audit | +| 2 oracles + load map | every load-address derivation; the segmentation decision (the forced boundaries) | the database import, the text export, the emulator bridge | a survey of every payload's loader route | +| 3 the all-assembly baseline | the linker-script/layout diagnosis when the first link is red | the build pipeline, the contracts | β€” | +| 4 the compiler pinned | **the fingerprint verdict** (the triple, the flags, per-module variation) | the probe wrappers, the differ wiring | the candidate ladder run as parallel probes | +| 5 census + harness | the census's shape reading (what the strategy will be built on) | the scanners, the reports | the fleet-wide census; the differential harness's pairs | +| 6 the multipliers | the reconcile ladder's design; any "class is dead" verdict | signatures, propagation, families, carves | mass propagation and remaps | +| 7 the map + the permuter | **reading the compiler's source into the map**; the plateau classifier's classes | the dump scripts, the permuter wiring | probes across many constructs | +| 8 the campaign | the routing cliff; **every wall verdict**; the harvest distillation's vocabulary | cards, packs, lanes, gates, recovery | bulk drafting; the harvest over a wave's reports | +| 9 publish | the contract run's design; any irreversible repository operation (rehearsed) | the publisher, the badges, the README | the fresh-clone proof on a second machine | +| 10 readability | struct unification decisions; every name that asserts meaning | pins-off by family; formatting | family-batched pin removal, gated | +```` + +--- + +## Block 5 β€” the cookbook `docs/{{COOKBOOK_NAME}}` (APPEND at Step 8; the entry shape and the triage table) + +Marker: `## The decomp entry shape (decomp-architect)` + +````markdown +## The decomp entry shape (decomp-architect, installed {{INSTALL_DATE}}) + +Every idiom entry is byte-proven on a named function and carries four parts: + +1. **The residual** β€” what the diff looks like (the tell): the instruction, its position, the register, the count. +2. **The mechanism** β€” which compiler pass produces it and why, with the dump line that shows it (a verdict without a + pass and a dump line is a hypothesis). +3. **The lever** β€” the C-level change that moves it, stated as a shape (never a register pin as the answer; a pin is a + symptom). Before a lever enters this file, strip it from the accepted body and recompile: about one credited lever in + three is inert. +4. **The byte proof** β€” the function, the diff before and after, the compilation the claim survived (standalone / + real translation unit / whole-binary). + +Pinned context for every entry: the toolchain triple `{{TOOLCHAIN_TRIPLE}}` (TODO until Phase 4); the SDK evidence +({{SDK_EVIDENCE}}); the target's calling convention and register roles (TODO(phase-4)). + +### The triage table (symptom β†’ section) β€” the index a worker greps + +| Tell in the diff | Likely mechanism | Lever family | Section | +|---|---|---|---| +| a temporary in a spill slot | a reload artifact | reshape the live range; declaration order | TODO | +| every small edit moves 20+ instructions | a register-pressure lock | hand to the permuter | TODO | +| a load stuck below a store while other loads float | an aliasing decision | the source's aliasing facts | TODO | +| a phantom callee-saved register | cross-call value caching / the allocation order | count the uses; re-shape | TODO | +| a shift off by Γ—4 against the target | the scaffold's pointer arithmetic | cast through a byte pointer | TODO | +| a branch's arms swapped | branch polarity | put the fall-through block in the `if` | TODO | +| a cross-jumped tail merged or not merged | the cross-jump count law | shape the shared block count | TODO | + +*(Grow this table as the cookbook grows; a tool asserts every section is indexed and every index entry resolves.)* +```` + +--- + +## Block 6 β€” `docs/wave-playbook.md` (CREATE at Step 6; the campaign procedure skeleton) + +Marker: `# The wave playbook β€” the operating procedure for the matching campaign` + +````markdown +# The wave playbook β€” the operating procedure for the matching campaign + +> ONE document is the procedure: read it before every run, keep it current in the same change as any tool it names; +> a superseded procedure gets a banner. Each guard below is paired with the MEASUREMENT that earned it β€” fill the +> measurement column from your own waves; the source project's are in the kit's methodology (the calibration fences). +> Created by decomp-architect on {{INSTALL_DATE}}. + +| # | Step | The guard | The measurement that earned it | +|---|---|---|---| +| 0 | Preconditions | tree clean; the last clean fleet run green | (fill) | +| 1 | Draw | exclude what cannot bank; audit the exclude list first (it records what the TOOLING could not do); rank by open templatable instructions; know face mass vs delivered mass | (fill) | +| 2 | Cards | the banked twin found over the WHOLE world goes on the card; a card names only levers the base contains | (fill) | +| 3 | Packs | one per target + the shared laws file, at the paths the drafting prompt reads | (fill) | +| 4 | Validate | never hand-type a target; assert every target is still open at draw time; near-misses to the permuter first | (fill) | +| 5 | Draft | one agent per target, model by measured size, streaming not batched; the wave's difficulty is a draw-time knob | (fill) | +| 6 | Gate | everything in parallel (worktrees; the executable by a clean rebuild); gate the DIRECTORY, never the verdict list; assert banked + failed + no-verdict == drafts | (fill) | +| 6b | Reconcile | one deterministic pass over each gate group's slate BEFORE the rebuild | (fill) | +| 7 | After ANY bank | the twin rescan, then commit | (fill) | +| 8 | Harvest β€” a hard gate | every idiom into the cookbook + index; every mechanical idiom into a sweep that banks the free functions; the widening review | (fill) | +| 9 | Verify the fleet | a clean rebuild of everything; read the exit code | (fill) | +| 10 | Checkpoint | refresh the replayable block; stale is worse than absent | (fill) | + +Then **recover** before re-drawing: triage the gate's failures into body and plumbing; bank the plumbing rejects through +the reconcile ladder without a redraft; the gate number is not the close rate until recovery has run. +```` + +--- + +## Block 7 β€” `.claude/settings.json` (MERGE at Step 8; add only absent keys) and `.mcp.json` (CREATE at Step 8) + +*ProjectArchitect already wrote the `SessionEnd` backup hook; the kit adds a `SessionStart` slot for the disassembler +server (filled at Phase 2) and the MCP client entry. Merge rule: add a key only if absent; never overwrite; validate +the JSON afterwards.* + +````json +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/disassembler_mcp_start.sh", + "_comment": "TODO(phase-2): the script that starts the disassembler's headless server and prints 'serving'; until it exists this hook is a no-op stub the kit installs" + } + ] + } + ] + } +} +```` + +````json +{ + "mcpServers": { + "disassembler": { + "type": "sse", + "url": "{{DISASSEMBLER_MCP}}", + "_comment": "TODO(phase-2): the loopback SSE endpoint of the disassembler's MCP server; after any restart, the developer reconnects the client" + } + } +} +```` diff --git a/decomp-architect/templates/run-README.md b/decomp-architect/templates/run-README.md new file mode 100644 index 000000000..c28642ad6 --- /dev/null +++ b/decomp-architect/templates/run-README.md @@ -0,0 +1,35 @@ +# `.run/` β€” scratch, with dated exceptions + +*Installed by decomp-architect (Step 4) on {{INSTALL_DATE}} for {{PROJECT_NAME}}. This folder is the project-local replacement for +the system temp directory: no project data lives outside the repository, ever. Build and extract logs, signature dumps, +permuter and compile scratch, agent work directories, per-session probes β€” everything a rerun can reproduce lives here and +is never committed. `make clean` never touches it; pruning it is a hand decision.* + +**It is ignored by contents, not as a directory.** The `.gitignore` rule is `/.run/*` rather than `/.run/`, because git will +not look inside an excluded directory and no `!` re-include could then work. That form lets the irreplaceable part be +tracked by exception. Each exception is a three-line idiom under a dated comment naming the phase, the session and the rule +that justified it: + +``` +# P (, ): the recorded contract run's per-step logs. Evidence, tracked. +!/.run/P/ +/.run/P/* +!/.run/P/verify/ +/.run/P/verify/* +!/.run/P/verify/*.log +``` + +**The test for an exception:** commit what a rerun CANNOT reproduce β€” hand or frontier-model analysis, the harness that +produced a verdict, a ledger, the recorded contract run β€” and leave what a script regenerates (compiler dumps, build logs, +drafts, compile directories) ignored. Work that turns out to be irreplaceable gets its dated `!` block the day it is +recognised as such, not at the phase close. + +**Per-session layout.** A session or wave gets one directory, `.run//`; each agent works in its own +`work/-
/` under it and may clean only that; deliverables (drafts, verdict lines, reports) go to a +`drafts/`, `verdicts/` or `reports/` directory no agent owns; an agent never runs `find`, `rm` or `mv` outside its own work +directory; an agent's final message is one JSON line, with the prose in `reports/.md`. + +**Two fail-safes.** Never `git clean -x` or `git clean -fdx` in this tree: the game-derived data (the dump, memory images, +the reverse-engineering database, the vendor SDK) is ignored-but-present on the maintainer's disk, and a `-x` clean deletes +it. And a tracked `.run/` file is PUBLISHED: it is subject to the ROM firewall like anything else β€” a listing of the target's +instructions is game-derived even inside a notes file, and the audit's content check looks for exactly that. diff --git a/decomp-architect/tools/MANIFEST.md b/decomp-architect/tools/MANIFEST.md new file mode 100644 index 000000000..cf91dbb00 --- /dev/null +++ b/decomp-architect/tools/MANIFEST.md @@ -0,0 +1,404 @@ +# tools/MANIFEST.md β€” the source project's portable tools, by ladder phase, as tasks + +> **What this is, and what it is not.** The kit installs no tools (its README says so). This manifest lists every tool the +> source project built, grouped by the phase of the ladder in `intake.decomp.md` Part B that needs it, with one line on +> what it does and one on what it hard-codes. **Until the source project's tools are de-specialised and split out, each +> row is a task for that phase:** build the tool for your target from its description, using the source project's file +> (named by its file name) as the reference implementation. Rows whose hard-codes read "none" or name only the +> instruction set are copy-after-the-split candidates; rows that hard-code the source repository's layout, its compiler +> triple or its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code +> (their *shape* is a Phase-1/2 task; their code does not transfer). +> +> **Coverage (a read-only survey on 2026-09-07):** 325 tool files classified of 325 found under the source project's +> `tools/` (top-level 266, lanes 19, disassembler scripts 12, the extractor package 11, the history-rewrite package 10, +> the permuter wrappers 3, the signature tool 3, an oracle 1); submodules, vendored third-party code, the vendor SDK, the +> reference compiler sources and downloaded binaries are excluded and not listed. Per phase: P1 2 Β· P2 26 Β· P3 17 Β· +> P4 9 Β· P5 32 Β· P6 59 Β· P7 25 Β· P8 102 Β· P9 25 Β· P10 12 Β· project-only 16. *TODO(platform): the MIPS and PlayStation +> SDK hard-codes are the ones another platform replaces first.* + +## P1 β€” extraction + manifest + +| Tool | What it does | Hard-codes | +|---|---|---| +| `iso9660.py` | Reusable reader for a raw 2352-byte-sector data track; walks the filesystem and extracts files by name | none | +| `manifest.py` | Builds and verifies a deterministic sorted SHA1 manifest of an extraction tree | the extraction output root | + +*The extractor itself, the container and compression decoders and the disc-completeness partition are project-only in code +(last table) and Phase-1 tasks in shape: a deterministic walker over the medium, a decoder implementing the GAME's +semantics with a length cross-check, a committed manifest, and a partition that accounts for every byte of the medium.* + +## P2 β€” the oracles + the load map + +| Tool | What it does | Hard-codes | +|---|---|---| +| `ghidra_mcp_start.sh` | Starts the headless disassembler MCP server detached on a fixed local port | repo paths, project name, port | +| `ghidra_mcp_stop.sh` | Clean save-and-close via a sentinel file; the only persistence event | repo paths, project name | +| `ghidra_mcp_verify.sh` | Read-only re-open confirming a symbol edit actually persisted after save-shutdown | repo paths, project name | +| `ghidra_import.sh` | Headless import and auto-analysis of a console executable with the platform loader | PS1 executable loader, project name | +| `ghidra_import_raw.sh` | Headless import of a flat headerless blob at a given base with the right processor spec | PS1 processor spec, project name | +| `ghidra_rebuild.sh` | Rebuilds one analysis program from committed text plus extracted bytes, and proves equality | repo paths, project layout | +| `ghidra_export_annotations.sh` | Read-only export of a program's annotations to byte-stable line-delimited JSON | repo paths | +| `ghidra_annotations_delta.py` | Derives the hand-authored annotation rows by subtracting a fresh rebuild baseline | repo paths | +| `ghidra_roster.py` | Generates and checks a roster of committed analysis programs from the build registry | repo config paths | +| `prefetch_fleet.py` | Batch headless decompilation of one representative per distinct open class into a cache | repo paths, binary registry | +| `ram_probe.py` | Captures, diffs and reads emulator main RAM over its web API for field typing | emulator host/port, 2 MB console RAM map | +| `find_addr_refs.py` | Register-tracked scan for code that materializes an absolute address; never window-paired | MIPS encodings, repo binary registry | +| `payload_base_evidence.py` | Ranks candidate load addresses for an unonboarded payload from pointers, self-calls and upper-half reach | the source game's payload map | +| `split_indicator.py` | Names code segments that must be split before their switch functions can be sectioned | repo config layout | +| `main_seed_ends.py` | Emits function start/length seeds for the executable's game-code objects, derived from the build | repo build paths | +| `BfmMcpServer.java` | The headless MCP server itself; holds an open transaction and saves on a sentinel stop | repo sentinel path, server port | +| `DecompileAt.java` | Headless script decompiling the function at one address and printing its C | default address | +| `DecompileFunctions.java` | Headless batch decompile of an address list into one C file per function | none | +| `DefineFunctions.java` | Disassembles and creates functions at externally validated entry points listed in a file | repo scratch path | +| `DumpFunctionSignatures.java` | Read-only per-function fingerprint dump in three hash tiers for cross-binary correspondence | none | +| `DumpProgramInfo.java` | Prints program metadata: language, compiler spec, image base, function count, properties | none | +| `ExportAnnotations.java` | Read-only serialization of types, signatures, data, comments, bookmarks and labels to stable JSONL | none | +| `ExportSymbols.java` | Exports user-defined symbols to a committable text file so annotations are version-controlled | none | +| `GetSymbolAt.java` | Prints the function or symbol name at one address for scripted persistence checks | none | +| `ImportAnnotations.java` | Idempotent compare-before-write import of the annotation JSONL back into a program | none | +| `ImportPsyqGdt.java` | Headlessly resolves a vendor SDK type archive into the program's type manager | PS1 SDK type archive name | + +## P3 β€” the all-assembly baseline + +| Tool | What it does | Hard-codes | +|---|---|---| +| `new_binary.sh` | One-command onboarding of any flat blob: config from template, registry entry, first build | repo config/template paths, the source game's payload classes | +| `mk_write.py` | The only safe writer of the generated binary-registry makefile; validates before replacing | repo config path | +| `gen_lib_subsegs.py` | Generates segment lines and a stub list for a multi-block vendor library region | repo config layout, PS1 SDK libs | +| `ld_interleave.py` | Reorders a generated linker script to reproduce the original section interleaving | repo build paths | +| `split_src_region.py` | Splits a source file at object boundaries, preserving matched code and stub blocks | repo src/config layout | +| `reorder_passthrough.py` | Restores the assembler-reorder build path for the objects originally assembled that way | pinned assembler flags, repo build paths | +| `verify_binary.py` | The correct hand verification of one binary: full re-extract plus rebuild, then hash compare | repo make targets | +| `psyq_lib_split.py` | Splits a vendor linker-format library archive into its member objects | PS1 SDK archive format | +| `psyq_build_libs.sh` | Converts vendor library members to ELF and archives them per library | PS1 SDK, repo scratch paths | +| `psyq_identify.py` | Locates where vendor library objects are linked in a target image via relocation-masked patterns | PS1 SDK objects | +| `psyq_link.py` | Links one vendor object at a fixed address so its code is byte-identical to the target | PS1 SDK, repo build paths | +| `psyq_link_lib.py` | Driver linking every used object of a library and byte-verifying each | PS1 SDK, repo paths | +| `psyq_link_region.py` | Links a whole library's objects in place of stubs, placing non-code sections as no-load | PS1 SDK, repo paths | +| `psyq_integrate.py` | Wires real library objects into the split build, replacing stub subsegments | repo config/build paths | +| `psyq_bss_probe.py` | Asks whether an object's scattered zero-init section can be split and placed byte-exactly | PS1 SDK object shape | +| `psyq_bss_split.py` | Rewrites an ELF object, splitting one zero-init section into per-base no-bits pieces | ELF32 REL layout | +| `psyq_libs_from_disc.py` | Extracts vendor SDK library files from the vendor's runtime-library disc image | vendor disc layout, PS1 SDK | + +## P4 β€” the compiler pinned; the probes + +| Tool | What it does | Hard-codes | +|---|---|---| +| `match_one.py` | Compiles one function standalone with the pinned toolchain, masks relocations, compares to target bytes | compiler triple, repo build flags | +| `masked_diff.py` | Shared relocation-masked instruction comparison used by the matcher and the permuter scorer | MIPS relocation encodings | +| `rtu_match.py` | Splices a candidate into a copy of the real translation unit and checks the same bytes | repo src layout, compiler triple | +| `rtu_second_chance.py` | Re-judges standalone compile-failures against their real translation unit before dropping them | repo scratch paths | +| `decompile.py` | Wrapper locating a function's disassembly and running the C-scaffold generator on it | repo asm paths, decompiler target name | +| `decompme_replica.sh` | Runs a function through an external reference toolchain build and compares words against the target | pinned toolchain versions, network fetch | +| `cc1_dumps.sh` | Dumps every compiler pass file for a self-contained draft into a private directory | compiler triple, repo scratch paths | +| `cc1_dumps_tu.sh` | Same pass dumps for the spliced real translation unit, the faithful compile | absolute repo path, compiler triple | +| `draft_prechecks.py` | Static pre-checks that skip a draft doomed to fail before any compile is spent | repo symbol/config sources | + +## P5 β€” the census, the harness, the reports + +| Tool | What it does | Hard-codes | +|---|---|---| +| `corpus.py` | The single derived model of the source tree: open, matched or shared, per function | repo src/config layout | +| `audit_digest.py` | Recomputes headline metrics from the current tree and fails if the committed digest disagrees | repo docs paths | +| `audit_frontier.py` | Checks that the independent "what remains" views agree with the corpus oracle | repo paths | +| `audit_binaries.py` | Gate asserting every consumer knows about each newly onboarded binary | repo config/registry paths | +| `audit_header_sigs.py` | Finds shared-header declarations that contradict the banked definition | repo shared-header paths | +| `audit_text_sources.py` | Every tracked C source must be plain text, or text searches silently skip it | repo src paths | +| `difficulty.py` | Ranks unmatched functions easiest-first by size, control flow, table presence and call count | repo asm layout | +| `dup_report.py` | Reports byte-identical and structurally identical function groups within and across binaries | repo signature files | +| `burndown.py` | Tracks per-session yield and velocity so a diminishing-returns close is visible | repo ledger paths | +| `backlog.py` | Near-miss ledger: every close-but-not-matching attempt, ranked for hand sessions | repo scratch/doc paths | +| `worklist.py` | Joins the target pool and the near-miss ledger into one byte-weighted ranked queue | repo scratch paths | +| `strand_census.py` | Lists every draft already on disk whose function is still open, with its blocker | repo scratch paths | +| `frontier_classify.py` | Classifies every remaining open function by its true blocker, deterministically | repo paths | +| `atlas.py` | Partitions all open functions into exactly one lever-labelled work group | repo scratch paths | +| `atlas_features.py` | Extracts one deterministic feature record per function across all registered binaries | repo signature files | +| `wall_taxonomy.py` | Census-classifies every unmatched residual by blocker class to size each avenue | canonical source binary name | +| `wall_sweep.py` | Enumerates fleet-wide instances of one known assembler-level blocker class | repo asm layout, toolchain quirk | +| `plumbing_groups.py` | Groups recorded declaration-conflict failures into sweepable work groups from the ledgers | repo ledger paths | +| `blocker_probe.py` | Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate | repo build paths | +| `symcheck.py` | Pre-gate guard diffing the symbol set a draft references against the target's | repo build paths | +| `reloc_identity.py` | Disagreeing oracle checking the symbol identity the masked comparison deliberately hides | repo build paths | +| `reloc_verify.py` | Resolves every relocation in a draft and compares the resolved words to the target | repo build paths | +| `oracle_reorder.py` | Decides whether a near-miss residual is a source defect or an assembler artifact | pinned assembler flags | +| `rtu_shadow.py` | Runs the real-translation-unit check in shadow beside the gate to measure gate inversion | repo scratch paths | +| `ab_score.py` | Re-scores every draft of two experiment arms with the standalone matcher as ground truth | repo scratch paths | +| `p16_improve.py` | Known-answer loop: reverts matched functions to stubs and measures the real pipeline | repo src/build paths | +| `p16_known_answer.py` | Graduated known-answer validation of the scaffold pipeline across difficulty bands | repo paths | +| `stub_invariant_audit.py` | Negative control: for every stub the built object must equal the target assembly exactly | repo build paths | +| `test_jtbl_parse_config.py` | Regression plus negative control for the table-carve configuration parser | repo config fixtures | +| `test_o0_detect.py` | Negative control for the matcher's optimization-level auto-detection | repo paths | +| `test_reconcile_ledger.py` | Targeted proof of the propagation ledger guard the full control never exercised | repo paths | +| `test_residual_class.py` | Synthetic hand-encoded unit tests for the deterministic residual classifier | MIPS encodings | + +## P6 β€” the multipliers: signatures, dedup, families, the reconcile ladder, the carve chain + +| Tool | What it does | Hard-codes | +|---|---|---| +| `sig_image.py` | Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper | MIPS encodings, repo scratch paths | +| `sig_unify.py` | Unifies a draft's callee externs and its own definition signature to the banked-canonical set | repo shared-header paths | +| `xsig.py` | Relocation-masked per-function signatures for cross-project code identification | MIPS/relocation model only | +| `test_xsig.py` | Property tests for the signature tool on committed fixtures, needing no compiler | fixture paths | +| `make_fixtures.sh` | Regenerates the signature-tool fixtures with the pinned toolchain | compiler triple, repo tool paths | +| `match_protos.py` | Joins per-function signature dumps of two related builds into a function correspondence | repo signature files | +| `dedup_integrate.py` | Byte-honesty validator for the code-share registry; fails closed on signature drift | repo config registry path | +| `dedup_propagate.py` | Lifts one matched body into a shared macro and instantiates it at every duplicate site | repo shared-header/src layout | +| `dedup_extend.py` | Extends the existing share registry to newly onboarded binaries | repo config/src layout | +| `inject_capped_externs.py` | Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs | repo shared-header layout | +| `restore_dropped_decls.py` | Puts back file-scope declarations a share propagation deleted with the body | repo src layout | +| `macro_draft.py` | Materializes a shared macro body back into a compilable standalone draft | repo shared-header layout | +| `demacroize.py` | Per-binary local escape from a shared-header declaration conflict by unmacroizing one body | repo shared-header layout | +| `exemplar_miner.py` | Routes every residual stub to its lever and ranks the highest-reach work | census file paths, canonical source binary | +| `exclude_audit.py` | Classifies every exclude-list entry by its current blocker and regenerates the list | repo config path | +| `family_hseq.py` | Ranked clustering of the unmatched frontier by mnemonic-skeleton hash | repo signature files | +| `family_manifest.py` | Ranked structural-family target manifest naming the one exemplar to draft per family | repo signature files | +| `family_cousins.py` | Similarity clustering one tier looser than exact skeleton hashing over the open frontier | repo signature files | +| `family_remap.py` | Mechanically remaps a matched exemplar's C onto a structural sibling by positional symbol pairing | repo src layout | +| `family_align.py` | Length-tolerant aligned classifier plus the mechanical constant-expansion engine for drifted members | repo scratch paths | +| `family_sweep.py` | Sweeps a matched exemplar across every same-structure sibling, gate-arbitrated | repo src/config layout | +| `twin_sweep.py` | Banks every open stub that has an already-banked structural twin, for near-zero tokens | repo signature files | +| `twin_rescan.py` | After a bank, reports which open stubs just became mechanical remaps instead of drafts | repo signature files | +| `bank_exemplar.py` | Banks one cracked exemplar through the carve, splice and staged-recovery ladder | repo src/config layout | +| `aprop_autodraft.py` | Mechanically drafts a family member from the seed body plus a positional symbol rebase | repo scratch paths | +| `aprop_symfix.py` | Rebases stale seed symbols in a mechanically adapted draft onto the target's own symbols | repo symbol/config sources | +| `weave_sweep.py` | Applies one known prologue-scheduling lever everywhere the bytes say it belongs | repo src layout, compiler behaviour | +| `cdecl.py` | The single recursive-descent C declarator parser every declaration tool shares | C89 grammar only | +| `canon_draft_decls.py` | Canonicalizes a draft's extern and data declarations to the banked-consistent set | repo shared-header paths | +| `canon_resident_calls.py` | Rewrites address-named calls in a draft to the curated symbol name when one exists | repo symbol file path | +| `conform_decls.py` | Conforms every declaration of a function fleet-wide to its byte-true definition | repo src/shared-header layout | +| `gen_engine_decls.py` | Declares every shared function and data symbol once, canonically, in a generated header | repo shared-header path | +| `decl_from_use.py` | Infers a minimal extern for a data symbol a draft uses but its destination does not declare | repo src layout | +| `decl_prior.py` | Computes the fleet's consensus declaration for every symbol, as card fuel | repo src layout | +| `sync_tu_decls.py` | Banks a refused draft by copying the destination unit's own declarations into it | repo src layout | +| `normalize_self_decls.py` | Normalizes a function's own declaration when a templated definition lands in a new unit | repo src layout | +| `reconcile_slate.py` | Drives a whole batch to declaration-compatible before any rebuild is spent | repo src layout | +| `reconcile_tu.py` | Conforms a draft's data declarations to what the destination translation unit can actually see | repo src layout | +| `fix_arity_callers.py` | Repairs the shared-caller argument-count conflict for the no-prototype failure class | repo shared-header layout | +| `fix_header_decl.py` | Rewrites a shared-header caller declaration to a draft's byte-true signature | repo shared-header path | +| `fix_decl_mirror.py` | Binds a void definition to its symbol when the destination declares a value return | repo src layout | +| `fix_tu_ret_decls.py` | Retypes a binary's own stale forward declarations to a draft's byte-true return, revert-on-fail | repo src layout | +| `cast_self_callers.py` | Lets a unit keep calling, through a per-site cast, the function it is about to define | repo src layout | +| `scope_data_externs.py` | Places a templated body's data externs at the scope the destination unit can accept | repo src layout | +| `scope_tu_externs.py` | Moves a unit's own file-scope data externs down into their consumers to legalize a block-scope type | repo src layout | +| `scope_demote_drafts.py` | The scope-demote step, wired as a rung of the recovery ladder | repo src layout | +| `jtbl_carve.py` | Sets up the read-only-data carve for a binary's matched jump-table functions | repo config/build layout | +| `jtbl_rodata_pads.py` | Post-assembler filter reproducing the original inter-table padding in a multi-table carve | compiler/assembler behaviour, repo build paths | +| `jtbl_pads_fix.py` | Repairs a stale padding spec by search plus byte proof, never by guessing | repo config layout | +| `jtbl_family_bank.py` | Banks a matched jump-table exemplar across its structural siblings, revert-on-fail | repo config/src layout | +| `pads_audit.py` | Derives each object's padding spec from the bytes instead of searching for it | repo build/config layout | +| `interleave_check.py` | Asserts the carve interleave order equals the segment sequence, position by position | repo config layout | +| `jr_isolate.py` | Isolates one switch function into its own code segment for an independent table carve | repo config/src layout | +| `jr_isolate_all.py` | One-shot multi-cut resegment isolating every switch function in a binary | repo config/src layout | +| `overlay_src_split.py` | Comment-state-aware partition of a source file, the oracle for the split chain | repo src layout | +| `o0_detect.py` | The single place that answers whether a function was built unoptimized, from its prologue | compiler prologue shape | +| `o0_boundary.py` | Finds and banks unoptimized functions stranded at the end of an unoptimized region | repo config layout | +| `o0_subsplit.py` | Routes an unoptimized address range that sits inside an otherwise optimized object | repo config layout | +| `rollout_o0.py` | Generalized two-file atomic rollout of an unoptimized definition across every binary carrying it | repo src/config layout | + +## P7 β€” the codegen map, the dumps, the permuter + +| Tool | What it does | Hard-codes | +|---|---|---| +| `gccmap_cites.py` | Tags every compiler-source citation in the codegen-map docs with the tree it refers to | repo doc/reference paths | +| `sweep_citations.py` | Deterministically localizes every compiler-source citation in a document against the pinned sources | repo reference tree paths | +| `verify_map_findings.py` | Machine-checks a codegen-map audit's findings against the actual compiler sources | repo reference tree paths | +| `reg_renumber_swap.sh` | Mechanized oracle separating a register-allocation swap from a scheduling reorder | compiler triple, repo scratch paths | +| `alloc_table.py` | Prints each pseudo's refs, live length, block and allocation priority from compiler pass dumps | compiler pass-dump format, repo dump root | +| `ghost_census.py` | Frame-residue oracle: finds pseudos with references but no remaining occurrences, and their slots | compiler pass-dump format | +| `autopsy.py` | Materializes the residual corpus by recompiling every open draft and classifying its failure | repo scratch/build paths | +| `residual_class.py` | Deterministic decoder-based classifier mapping a residual to a named codegen class | MIPS encodings | +| `residual_rules.py` | Turns a measured residual into the specific documented lever that addresses it | repo doc paths | +| `residual_rules_b.py` | Independent second implementation of residual-shape to rule classification, for head-to-head | repo doc paths | +| `diff_regions.py` | Classifies where a remapped member's compiled bytes diverge from its target | repo build paths | +| `diff_autopsy.sh` | Reproduces exactly what the gate saw, then decodes the diverging words and restores the tree | repo src/build paths | +| `main_diff_locate.py` | Turns a red whole-binary result into a named list of divergent symbols | repo build/config layout | +| `len_tells.py` | Classifies a length-drift near-miss against its own target and builds the routing card | repo scratch paths | +| `lenmiss_route.py` | Routes the length-drift near-miss pile through the documented lenses, re-verified against the tree | repo scratch paths | +| `masked_scorer.py` | Drop-in permuter scorer scoring true relocation-masked code closeness instead of mnemonic diff | permuter internals, repo module paths | +| `run_masked.py` | Runs the permuter with the masked scorer rebound before its entry point, without patching it | permuter internals, repo module paths | +| `compile.sh` (permuter) | The permuter's compile command, mirroring the build rule exactly so objects are build-faithful | compiler triple, repo build flags | +| `compile_o0.sh` (permuter) | Unoptimized variant of the same permuter compile command | compiler triple, repo build flags | +| `p16_permute.py` | Per-function permuter driver building base, target and settings, then reporting closeness | repo scratch paths, permuter layout | +| `permuter_weights.py` | Directs permuter mutation toward the lever family the residual class implies | permuter settings format | +| `permuter_ils.py` | Iterated-local-search wrapper warm-restarting the permuter from the best waypoint each cycle | permuter layout, repo scratch paths | +| `permuter_sweep.py` | Hands a batch's near-misses to the permuter, but only the ones worth the CPU | repo scratch paths | +| `grinder.py` | Token-free permuter daemon grinding the closest near-misses and banking through the gate | repo scratch paths | +| `warmstart.py` | Feeds the permuter warm-start drafts built from banked exemplars and routed fuel | repo scratch paths | + +## P8 β€” the campaign: cards, lanes, gates, recovery, harvest + +| Tool | What it does | Hard-codes | +|---|---|---| +| `claude_wave_packs.py` | Builds the per-function prompt packs an agent wave drafts from | repo scratch paths, agent harness | +| `journal_notes.py` | Mines per-function past-attempt notes out of the agent journals into card fuel | agent transcript paths | +| `seed_ref.py` | For an open stub, finds the already-banked body that matches it, including near and contained tiers | repo signature files | +| `neighbor_ref.py` | For an open stub, ranks already-matched functions worth reading as worked examples | repo signature files | +| `wave_card_fuel.py` | The single library computing the per-target card fuel a pack carries | repo scratch paths | +| `t5_cards.py` | Builds the card fuel for one wave's own targets, keyed correctly per binary | repo scratch paths | +| `t5_targets.py` | Draws one routed slate from the frontier classes | repo scratch paths | +| `t5_distill_args.py` | Builds the post-wave distillation slate from a wave's directories | repo scratch paths | +| `t5_bank.sh` | Post-draft half of an agent wave: refuse if unsafe, judge each arm, bank the union | repo paths, lane conventions | +| `t7_bank.py` | Batch banking driver applying reconcile-at-bank-time plus the whole-binary gate | repo src/config layout | +| `draw_waves.py` | Draws several drafting waves off the open frontier, cheapest-first | repo scratch paths | +| `build_wave.py` | Builds the next wave from a card pool, ranked by fleet leverage | repo scratch paths | +| `build_wave_atlas.py` | Builds a wave from the frontier atlas, optimized for gate throughput | repo scratch paths | +| `build_wave_args.py` | Emits workflow arguments for a wave from the target manifest, resolving each path | repo asm/scratch layout | +| `wave_args.py` | Emits the exact workflow arguments for a wave, asserting every target is open | repo scratch paths | +| `wave_targets.py` | Selects a worker-wave target batch from the target manifest with cache filters | repo scratch paths | +| `wave_snapshot.py` | Gives a wave its own immutable copy of the disassembly files its agents read | repo asm paths | +| `wave_judge.py` | Per-arm whole-binary gate with tree reset between arms, then gates the best-of union | repo scratch/src layout | +| `build_fuel_manifest.py` | Unifies live stubs into one ranked, class-tagged, cache-annotated target pool | canonical source binary, repo scratch paths | +| `gen_harvest_targets.py` | Generates a callee-signature-aware target manifest with per-function fleet reach | canonical source binary, repo paths | +| `validate_targets.py` | The validity gate every target list must pass before agents are spawned | repo scratch paths | +| `launch_check.py` | Refuses to launch an agent at a target that is already banked | repo src layout | +| `lane_inflight.py` | The authoritative ledger of which drafting agents are currently live | repo scratch paths | +| `work_evidence.py` | Assertions that a tool actually did the work it reports | repo scratch paths | +| `campaign_status.py` | One status view covering every lane, not only the loud one | lane names, repo scratch paths | +| `triage_ladder.py` | Zero-token pass deciding whether a target needs an agent at all, before and after drafting | repo scratch paths | +| `gap_triage.py` | Pre-filters a wave's knowledge-gap reports against the existing documentation | repo doc paths | +| `fragment_check.py` | Refuses drafts that subsume a neighbouring symbol, the enclosing-function trap | repo asm layout | +| `gate_stage.py` | The shared deterministic recovery, gate and log ladder every producer calls | repo src/config layout | +| `gate_lane.py` | The wave gate driver, accepting either result shape and an explicit draft path | repo scratch paths | +| `gate_wave.py` | Gates a whole wave by splitting table-bearing drafts off and running both lanes concurrently | repo scratch/config layout | +| `gate_triage.py` | Routes a gate result set to the repair tool its own verdict names | repo scratch paths | +| `gate_main.py` | Gates a batch of executable drafts the only trustworthy way: a clean rebuild with bisect | repo build/make targets | +| `gate_main_parallel.py` | Discovers the executable's bankable drafts concurrently, then banks them once, serially | repo build/make targets | +| `parallel_gate.py` | Gates many binaries concurrently in isolated worktrees and merges only the passers | repo worktree/build layout | +| `sweep_parallel.py` | Gates pre-staged draft directories across distinct binaries in parallel | repo scratch paths | +| `gater_lane.py` | Continuous gater draining a wave's finished drafts into the parallel gate, grouped by binary | repo scratch paths | +| `harvest_verify.py` | The whole-binary byte gate: substitute, build, keep only if identical, else revert | repo src/build layout | +| `pregate_check.py` | Validates a batch without building it, in seconds instead of minutes | repo src layout | +| `restage_matching.py` | Re-stages only the drafts that compile and match in their real unit, so one failure is isolated | repo src layout | +| `bisect_slate.py` | Isolates the byte-wrong drafts in a batch, with a null control first | repo build paths | +| `blast_radius.py` | Measures a change's write set and names the verification it actually requires | repo tree layout | +| `r22_verify.sh` | Exclusive clean-tree fleet verification that also clears deferred-check debt | repo make targets | +| `verify_worktree.py` | Checks a commit into its own worktree, provisions build deps, and verifies it there | repo worktree/build layout | +| `shared_lock.py` | One reader/writer lock over the shared source state for concurrent lanes | repo lock paths | +| `treelock.sh` | The mutex wrapper for tree-writing campaigns, with status and stale detection | repo lock paths | +| `jtbl_lane.py` | Carve, draft and bank lane for jump-table functions, with the carve deliberately outside it | repo config/src layout | +| `main_lane.py` | The executable's own draft, gate and commit cadence beside the other lanes | repo build/make targets | +| `main_queue_rebuild.py` | Rebuilds the executable lane's queue from the tree, never from a stale list | repo src layout | +| `integration_resolver.py` | Zero-token lane re-judging drafts that are already byte-correct and banking them | repo scratch paths | +| `scan_leftovers.py` | Re-verifies every draft already on disk and banks the free wins | repo scratch paths | +| `recover_drafts.py` | Recovers agent-written draft files from a run's transcripts after they were deleted | agent transcript paths | +| `agent_drafts_restore.py` | Replays an agent's writes from its transcript to rebuild its final deliverable | agent transcript paths | +| `agent_reports.py` | Saves each subagent's full final prose report to one file per target | agent transcript paths | +| `agent_verdicts.py` | Pulls the final structured verdict out of subagent transcripts | agent transcript paths | +| `transcript_dump.py` | Condenses a session transcript into readable text for a successor session | agent transcript paths | +| `recover_rejects.py` | Turns pre-gate rejects back into bankable drafts for zero model tokens | repo scratch paths | +| `recover_route.py` | Given a drop verdict, names the one recovery tool that actually applies | repo scratch paths | +| `recover_giant.py` | Canonical-extern recovery for a large draft rejected only on declaration plumbing | repo shared-header layout | +| `recover_integration.py` | Batch integration recovery driver over a wave directory of correct-but-rejected drafts | repo scratch/src layout | +| `bulk_harvest.py` | Phase-separated bulk drafting plus a parallel gate farm, for throughput | repo scratch paths, GPU endpoint | +| `lora_grind.py` | Mass-run driver rotating binaries, drafting small stubs with a served model, gating, propagating | repo scratch paths, model endpoint | +| `orchestrator.py` | The deterministic half of the unattended loop: pick the pool, emit the batch, record the cycle | repo scratch paths | +| `auto_driver.py` | Autonomous model-free driver looping a stub worklist through scaffold, draft and gate | canonical source binary, repo paths | +| `auto_supervisor.sh` | Keeps an unattended driver alive across crashes until a clean or requested exit | repo scratch paths | +| `auto_stop.sh` | Requests a safe exit of the unattended run via a stop sentinel | repo scratch paths | +| `auto_status.sh` | Prints the unattended run's heartbeats and recent progress for a remote check-in | repo scratch paths | +| `ox_campaign.py` | The unattended wave loop: draw, draft, pre-filter, gate, commit, ledger, repeat | repo scratch paths, provider config | +| `api_agent.py` | Gives an external chat model the same multi-turn tool harness an agent gets | provider endpoint, repo doc paths | +| `api_draft.py` | Provider-agnostic single-shot drafting worker against any chat completions endpoint | provider endpoint, repo doc paths | +| `api_rate.py` | Reports measured request rate and rate-limit attribution from append-only telemetry | repo telemetry path | +| `glm_parallel.sh` | Splits a target list into slices and runs concurrent drafting workers against one endpoint | provider endpoint, repo paths | +| `glm_reconcile.py` | Aims a reasoning model at the declaration wall and captures its reasoning for reuse | provider endpoint, repo paths | +| `idiom_harvest.py` | Pulls the newly learned idioms out of a wave's shard logs into candidate notes | repo log paths | +| `idiom_hunt.py` | Uses a reasoning model on grouped near-misses to discover new compiler idioms | provider endpoint, repo paths | +| `idiom_loop.py` | Never-ending meta-loop: drain the easy fuel, learn the next idiom, hand off | repo scratch paths | +| `idiom_serial.py` | The serial idiom-learning lane where each step inherits what the previous learned | repo scratch paths | +| `distill_scan.py` | One pass of the distillation lane deciding which harvested waves still need mining | repo scratch paths | +| `export_pairs.py` | Mines gate-verified assembly-to-source pairs into a fine-tuning corpus | repo src/build layout | +| `format_finetune.py` | Turns the mined pairs into a chat-template instruction dataset | dataset paths, model chat template | +| `train_lora.py` | Fine-tunes a code model into a matching specialist and saves the adapter | dataset paths, training venv | +| `eval_lora.py` | Gate-true evaluation of a fine-tuned model on held-out banked functions | dataset paths, repo corpus | +| `serve_local.py` | Serves the fine-tuned model locally behind a chat-completions endpoint | training venv, GPU assumptions | +| `bounce_drafter_on_queue.sh` | Waits for the in-flight wave to queue, then restarts the drafting process | lane script paths | +| `campaign.sh` | Dual-pool campaign lane: two drafting pools, one serial gate | provider config, repo paths | +| `distill.sh` | The distillation lane, running beside drafting and never in front of it | repo scratch paths | +| `drafter.sh` | The drafting lane that must never stop; touches only scratch state | repo scratch paths, provider config | +| `elastic.sh` | Spends spare provider capacity on extra work when the main lane cannot | provider config, repo paths | +| `forever.sh` | Keeps the wave campaign running indefinitely across caps and crashes | repo scratch paths | +| `gater.sh` | The restartable gating lane, safe to kill and relaunch at any time | job count, repo paths | +| `grinder_lane.sh` | Runs the model-free permuter grinder beside a saturated gater | repo scratch paths | +| `launch_ox.sh` | Launcher for a sharded campaign run with a tag and shard count | provider key file, repo paths | +| `main.sh` | The executable's own lane loop, on its clean-rebuild cadence | repo make targets | +| `maintenance.sh` | The maintenance lane running the free recovery and re-judgement passes | repo scratch paths | +| `relaunch_drafter_shell.sh` | Waits for a wave to queue, then restarts the drafting lane's shell so new arguments apply | lane script paths | +| `resolver_lane.sh` | The zero-token resolver lane: re-judge, verify twice, stage, gate, commit | repo scratch paths | +| `restart_gater_when_idle.sh` | Restarts the gating lane's shell once the current gate finishes | lane script paths | +| `restart_main_lane_when_idle.sh` | Restarts the executable lane's shell at its one safe boundary | lane script paths | +| `sibling_lane.sh` | The free sibling lane driving structural sweeps with synthesized declarations | repo paths | +| `stallguard.sh` | Detects and repairs lane stalls on measured conditions every minute, logging each action | lane names, repo paths | +| `toolwork.sh` | Runs tooling-design briefs against an external model with build config as context | provider config, repo paths | +| `wait_distill.sh` | Blocks until the distillation lane raises the next batch, then exits | repo scratch paths | + +## P9 β€” publish + +| Tool | What it does | Hard-codes | +|---|---|---| +| `verify_contract.sh` | The recorded end-to-end verification run of the byte-identity contract, step by step | repo make targets, output dir | +| `bootstrap.sh` | Idempotent fresh-clone setup: check packages, create environments, fetch pinned tools | repo layout, package names | +| `fetch_psyq.sh` | Optional acquisition and checksum verification of vendor SDK pieces for the linked build | PS1 SDK sources, repo paths | +| `compile_only.py` | Compiles every eligible translation unit with the pinned toolchain, without any game bytes | compiler triple, repo makefile parsing | +| `audit_public.py` | The first-push gate: no derived bytes among tracked files, four independent checks | repo purge-set and path lists (the kit's template reads `config/firewall.txt` instead) | +| `progress.py` | Computes and publishes the progress metrics as report, JSON and per-binary breakdowns | repo src/config layout | +| `objdiff_report.py` | Converts the progress JSON into the external progress-report schema | external report schema | +| `frogress_upload.py` | Posts the progress JSON to a hosted progress service, dry-run by default | service URL, project/version names | +| `timeline.py` | Builds a progress timeline from the repository's own committed digests, with a self-check | repo doc paths | +| `mine_hindsight.py` | Gathers every recorded hindsight with file and line anchors into one document | repo doc paths | +| `cookbook_index.py` | Generates a symptom-keyed index of the large technique document | repo doc paths | +| `doc_links.py` | Checks that every relative link in the public docs resolves and the link policy holds | repo doc paths | +| `gitignore_template_check.py` | Asserts the ignore-file template in the docs equals the kit's template byte for byte | repo doc and template paths | +| `wiki_render.py` | Renders the doc trees into wiki page names with every relative link rewritten deterministically | repo doc paths | +| `wiki_sync.sh` | Renders and publishes the docs to the hosted wiki, replacing its pages | wiki remote URL, repo paths | +| `common.py` (history rewrite) | Shared purge rules, derived hash sets and helpers for the history-rewrite package | repo scratch paths | +| `hash_dict.py` | Builds the private dictionary mapping every old commit hash to an inert token | repo scratch paths | +| `scrub.py` | The single scrub function of the history rewrite, plus its sample and self-test | purge rules, repo scratch paths | +| `run_filter.py` | Composes and runs the history rewrite inside a bare clone, refusing to run elsewhere | rewrite tool version, repo scratch paths | +| `gate_scan.py` | History gate: no derived blob reachable from the given refs, four derived checks | repo declaration files | +| `verify_rewrite.py` | Pairwise proof that the rewrite changed only what it was told to change | repo scratch paths | +| `build_commit_map.py` | Emits the public ordinal-to-new-hash map and the private old-to-new map | repo doc/scratch paths | +| `resolve_tokens.py` | Turns ordinal tokens back into shortest-unique hashes at the rewritten tip | repo tracked text files | +| `absent_scan.py` | Asserts nothing purged remains in any object, message or ref | repo scratch paths | +| `probe_github.sh` | Probes whether old commit hashes still resolve on the hosting service after a purge | hosting service API, repo name | + +*The history-rewrite package exists because the source project committed game-derived files while private; a project that +holds the firewall from commit one never needs it. It is listed so the procedure is known, not as a task.* + +## P10 β€” readability + +| Tool | What it does | Hard-codes | +|---|---|---| +| `lift_types.py` | Lifts a named list of types fleet-wide into the shared type header | repo shared-header path | +| `build_engine_types.py` | Extracts inline-defined named types and typedefs from a source file into a shared header | repo shared-header path | +| `uniquify_type.py` | Gives each conflicting camp of a same-named type its own name so every camp becomes liftable | repo src layout | +| `cast_call_sites.py` | Adds per-site function-pointer casts so a draft can call a differently typed callee | repo src layout | +| `canon_sig_reconcile.py` | Reconciles a definition's typed signature with what its destination unit already declares | repo src layout | +| `ghidra_apply_symbols.sh` | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name | +| `ApplySymbols.java` | The in-tool half of that mirror: apply curated names and signatures, save on exit | none | +| `lint_symbol_refs.py` | Flags address-named references in committed sources whose address now has a curated name | repo symbol/src paths | +| `verbatim_check.py` | Regression guard that every inline-assembly body still reproduces its target bytes | repo src layout | +| `asm_verbatim.py` | Emits the file-scope inline-assembly body form from a disassembly file | repo asm layout | +| `verbatim_target_s.py` | Regenerates a splitter-format target disassembly for a function that is no longer a stub | repo build/asm layout | +| `verbatim_to_stub.py` | Turns an inline-assembly body back into an include-assembly stub | repo src/asm layout | + +## Project-only in code (the shape is a task; the code does not transfer) + +| Tool | Why it does not transfer | +|---|---| +| `bfm_extract/__init__.py` | Package entry for one title's disc pipeline; names its own container stages | +| `bfm_extract/extract.py` | Full-disc extractor bound to one disc's root file list and audio-track handling | +| `bfm_extract/extract_exe.py` | Hard-validates one executable's exact name, size, hash and header field values | +| `bfm_extract/extract_proto_exe.py` | Companion for one title's prototype discs and their specific deviations | +| `bfm_extract/cd_archive.py` | Reader for one game's bespoke container table-of-contents format | +| `bfm_extract/pac.py` | Parser for one game's bespoke archive-chain header format | +| `bfm_extract/lzss.py` | Decoder for one game's non-generic compression semantics, verified against its own decompressor | +| `bfm_extract/test_lzss.py` | Unit tests for that game-specific compression variant | +| `bfm_extract/crosscheck.py` | Cross-validates against a third-party tool written for that specific game | +| `disc_audit.py` | Byte partition over one disc, importing the game-specific container and compression modules β€” the partition itself (every byte of the medium accounted for, residue zero) is a Phase-1 task | +| `idxtab_map.py` | Decodes one game's on-disc index table and its payload-to-binary naming convention β€” the load map itself is a Phase-2 task | +| `cdtrace.py` | Reads one game's loader request structures at fixed RAM addresses | +| `new_overlay.sh` | Takes one game's container and file-index naming as its command-line contract | +| `make_libgs.sh` | Hard-codes one executable's exact vendor object list and address blocks | +| `make_snd_used.py` | Hard-codes one executable's sound-library address window and per-address exclusions | +| `make_apicard_used.py` | Hard-codes one executable's interface-library address window and object roster | diff --git a/docs/SETUP.md b/docs/SETUP.md index 78a2791d0..8ca9f2e17 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -769,6 +769,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/mine_hindsight.py [--out …]` | **(P33 F2)** Gathers every recorded hindsight with `file:line` anchors β€” the decision-log's `Hindsight` bullets and `### Hindsight` sections (19 over 79 entries), the PhaseEnds' "What we believed…" sections (2), every Deviations table (237 rows / 32 PhaseEnds) β†’ `.run/P33/hindsight.md` (scratch; `docs/retrospective.md` cites the sources, never the working set). Prints the census. | | **Docs** | `tools/doc_links.py [--strict] [--disk] [FILE…]` | **(P33 D5; extended P33.5 task 7)** Six checks, each printed with its denominator: (1) every relative Markdown link in the public-facing docs (the DEFAULT set + every wiki page and how-to chapter) resolves; targets listed in `docs/doc_links_pending.txt` (`pathcreating task`) count as PENDING, not broken β€” `--strict` (gate 2) refuses any pending entry; (2) nothing links into `docs/sunset/`; (3) a wiki page/chapter links into `docs/` only at a target the Reference index or the README links (the allow-list is DERIVED from those two pages; the two index pages are exempt); (4) every tracked `docs/` file outside wiki/how-to/sunset is covered the same way; (5) backticked `docs/…` / `.run/…` citations are TRACKED or UNTRACKED by `git ls-files` (never the disk) β€” a wiki page citing an UNTRACKED path fails, elsewhere it is counted; `--disk` adds the PRIVATE/DANGLING split for the maintainer; (6) wiki-first WARNINGS (exit 0) for a non-wiki document linking a `docs/` file whose topic has a wiki page. In `tools-health`. Controls: a broken link β†’ rc 1; the P33.5 cookbook control (`--disk docs/matching-cookbook.md` listed the stale `.run/` citations before they were fixed). | | | `tools/gitignore_template_check.py` | **(P33.5 task 7)** The ```` ```gitignore ```` fence of `docs/wiki/The-ROM-firewall.md` must equal `decomp-architect/templates/gitignore.decomp` byte for byte (one source, two copies; R75-shaped). rc 1 on drift, rc 2 when the template does not exist yet ("nothing to compare" β€” never a pass, R43); refuses a page with β‰  1 fence. In `tools-health` behind an existence test that skips loudly until the kit lands (task 11). | +| | `tools/kit_lint.py [--selftest] [--paths …]` | **(P33.5 task 11)** The day-one decomp kit (`decomp-architect/`) stays de-specialised: (1) LEAK β€” no line outside a ```` ```calibration ```` fence and off a `provenance:` line matches `SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\.17\.|\bR[0-9]{1,2}\b|Β§[0-9]+` (fence-aware: `grep -v calibration` would drop only lines containing the word); (2) PLACEHOLDERS β€” the `{{NAME}}` set used under the package equals the backticked set in `templates/PLACEHOLDERS.md` (the contract); (3) SYNTAX β€” `bash -n` / `py_compile` (no bytecode written) / JSON+YAML parse; (4) the gitignore template diff (delegated); (5) `TODO(platform)` / `TODO(phase-N)` counts; (6) coverage β€” zero files is a failure. rc 1 findings, rc 2 package absent (R43). `--selftest` = the R39 control (a planted leak line + a planted unlisted placeholder must be caught, fenced and provenance lines must not). In `tools-health` (selftest, then the real run). | | **Verification** | `tools/verify_contract.sh` | **(P33 A5/C8) THE recorded contract run**: 00 tree Β· 01 check-env Β· 02 family_hseq Β· 03 `make clean && extract-all && check-all` Β· 04 sdk-dual (or a recorded SKIP) Β· 05 tools-health (zero `[warn]`) Β· 06 audit-frontier Β· 07 audit-disc Β· 08 report; one log per step ending `EXIT=`, abort on the first red (R53), every step asserted by its contract line (R49), `SUMMARY.md` generated β†’ `.run/P33/verify/` (tracked evidence, quoted by `docs/verification.md` Β§2); step 00 ignores its own output dir. β‰ˆ14 min on 32 CPUs. | | **Publishing** | `tools/progress.py --json \| --readme [--check]` | **(P33 D1/D3)** The same numbers as DATA: `--json` β†’ `docs/progress.json` (schema 1: the four metrics with numerator/denominator/pct, the counts, 218 per-binary rows incl. instruction totals; no run date) + `docs/badges/{fleet_instr,fleet_fn,distinct,binaries}.json` (shields endpoint format; the README references `fleet_instr` + `binaries` by name); `--readme` rewrites the README's `` block (refuses a README without the markers); `--check` asserts JSON + block + badges are fresh (in `make audit-digest`). Run by `make report BINARY=main`. | | | `tools/wiki_render.py OUT_DIR \| --list \| --selftest` | **(P33 F3)** Render `docs/wiki/*.md` + `docs/how-to-ai-decomp/*.md` into GitHub-wiki page names with every relative link rewritten deterministically (wiki page β†’ its name; a chapter β†’ `How-to-AI-decomp-NN-name`; any other repo path β†’ a `blob/main` / `tree/main` / raw URL; URLs, mailto and anchors untouched; **a dead link is an error**, R43). `--selftest` = the 12-case fixture incl. the dead-link negative control **plus (P33.5 task 7) the reachability assertion: every `docs/wiki/*.md` except `_Sidebar`/`_Footer`/`Home` is linked from `_Sidebar.md`, every chapter from `_Sidebar.md` AND `How-to-AI-decomp.md`** β€” a published page nobody can navigate to fails here; in `make tools-health`. | diff --git a/docs/story-timeline.md b/docs/story-timeline.md index 925f0b733..4607eb021 100644 --- a/docs/story-timeline.md +++ b/docs/story-timeline.md @@ -88,6 +88,6 @@ | 2026-09-04 | 57 | | 213 | 100.0% | 100.0% | 99.9% | 95.1% | 21 | 777 / 2091 | | 2026-09-05 | 83 | P31 (v1.30.0) | 218 | 100.0% | 100.0% | 100.0% | 97.7% | 4 | 787 / 2091 | | 2026-09-06 | 36 | P32 (v1.31.0) | 218 | 100.0% | 100.0% | 100.0% | 100.0% | 0 | 789 / 2091 | -| 2026-09-07 | 43 | P33 (v1.32.0) | 218 | 100.0% | 100.0% | 100.0% | 100.0% | 0 | β€” | +| 2026-09-07 | 48 | P33 (v1.32.0) | 218 | 100.0% | 100.0% | 100.0% | 100.0% | 0 | β€” | 73 dated rows Β· phase ticks from the 33 PhaseEnds Β· the chart: `docs/story-timeline.svg`. diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 88ca8f9d6..f6dad04d9 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -49,7 +49,7 @@ in-tree links to `docs/wiki/.md`. 8. `.run/`: only what git tracks; no sca - [x] **8** Tracked `.run/` prune (218 paths untracked: the 172 inertia files + 44 finished logs + the 2 firewall listings; `untracked_after_rewrite.txt`; `audit_public` check 4 with both controls; runbook Β§11; the four tool notes) β€” xHigh β€” see Log 2026-09-07 Task 8 β€” **P6 rules check done after it** - [x] **9** Memory reconciliation (the off-project file parked; the 7 stale updated in place; `bfm-decomp-context-system` refreshed; the seed set β€” 16 files β€” under `decomp-architect/memory-seed/` with `upstream: PA` tags) β€” xHigh β€” see Log 2026-09-07 Task 9 - [x] **10** Kit part 1: `README.md`, `intake.decomp.md`, `decomp-architect.md`, `templates/registry-E.decomp.md`, `corpus/decomp-kernels.md`, `templates/PLACEHOLDERS.md` β€” Max β€” see Log 2026-09-07 Task 10 -- [ ] **11** Kit part 2: the firewall pack, docs/run READMEs, ops-setup, bootstrap, CLAUDE overlay, `pa-overlays.md`, LICENSE/NOTICE/README/CONTRIBUTING skeletons, `.clang-format` + format snippet, `tools/MANIFEST.md`; `tools/kit_lint.py` in tools-health β€” xHigh +- [x] **11** Kit part 2: the firewall pack, docs/run READMEs, ops-setup, bootstrap, CLAUDE overlay, `pa-overlays.md`, LICENSE/NOTICE/README/CONTRIBUTING skeletons, `.clang-format` + format snippet, `tools/MANIFEST.md`; `tools/kit_lint.py` in tools-health β€” xHigh β€” see Log 2026-09-07 Task 11 - [ ] **12** Kit part 3: `SETUP.md` (Β§0–§10, `--answers`, the PA-2.0 version pin, the honesty section, Path A only) β€” Max β€” **then P6 rules check** - [ ] **13** Dry-run install under D6's guardrails; fix; re-run to green; fixture + expected manifest + logs tracked under `.run/P33.5/kit-dryrun/` β€” xHigh - [ ] **14** Wiki `Start-a-new-decomp-project.md` + README/`Tools-from-this-project.md` rows + SETUP rows (R21) + decision-log entry (R31) + accelerators entry if earned β€” xHigh @@ -483,100 +483,134 @@ ANY untrack, not only after a docs edit. script above was used instead and is what `kit_lint.py` implements. The kit's `docs/` cross-references are by chapter title and kernel/rule id, never by this repository's paths, so the split can lift the folder unchanged. -## πŸ›‘ SESSION CHECKPOINT β€” Tasks 0–10 βœ“; NEXT = task 11 (kit part 2: the firewall pack, READMEs, overlays, skeletons, MANIFEST, `kit_lint`; **xHigh** β€” prompt Drew, R27) +### 2026-09-07 β€” Task 11 β€” Kit part 2: the firewall pack, READMEs, overlays, skeletons, MANIFEST, `kit_lint` (xHigh; S91) +**Written under `decomp-architect/templates/` (18 new files):** `gitignore.decomp` (EXTRACTED by script from the wiki page's +```` ```gitignore ```` fence, 71 lines β€” `gitignore_template_check` now runs in tools-health and reads OK); `firewall.txt` (the audit's ONE +config: `purge:`/`glob:` rules for the nine classes with `{{TARGET_BINARY}}` as the first purge path, hash sources tagged +`required:` / `pending:` / `fixture:` β€” the manifest, the medium hash and the contracts are `pending:` with `TODO(phase-1/3)` promotion +notes); `audit_public.template.py` (a de-BFM'd generalisation of `tools/audit_public.py`: sources from the config, four checks incl. +the disassembly-shaped-content run β‰₯ 64, refuses zero resolvable sources / a missing required source / an unknown rule kind, warns +loudly on a missing pending source, `--paths` for the control; compiles); `firewall-fixture/` (`blob.bin` = the 16 synthetic bytes +`DECOMP-FIXTURE!!`, `blob.sha1` = `d4bc7b5d…`, README: only the `.sha1` is installed into the new repo as `config/firewall-fixture.sha1`; +the control PLANTS the blob under scratch, asserts FAIL by `--paths`, removes it, asserts PASS); `no-rom.template.yml` (the audit job; +compile-only as `TODO(phase-3)`); `docs-README.md` + `run-README.md` (the conventions page's two tables); `ops-setup.decomp.md` (the +decomp rows: pins with `{{TOOLCHAIN_TRIPLE}}`/`{{COMPILER_FAMILY}}`, the game/medium/dump lines, `{{EXTRACT_CMD}}`/`{{FLEET_CHECK_CMD}}`, +the two oracles, the git posture, the tooling rows); `bootstrap.template.sh` (skeleton, every phase-gated step a `TODO(phase-N)`; +`bash -n` clean); `CLAUDE.decomp-overlay.md` (the four fail-safes; the session-start extras: the digest replaces the full PhaseEnd +read, the checkpoint replayed verbatim, the oracle ping only for RE tasks, the flywheel via `{{COOKBOOK_NAME}}`, the effort line); +`pa-overlays.md` (SEVEN fenced blocks, each headed by target + marker + step: the DIGEST template (CREATE), the πŸ›‘ SESSION CHECKPOINT +block for CURRENT_PHASE (APPEND, sections 0–3), the PhaseEnd narrative axis (APPEND), the decomp per-phase effort rows (APPEND), the +cookbook entry shape + the symptom-keyed triage table (APPEND), the wave-playbook skeleton with an empty measurement column (CREATE), +`settings.json` SessionStart slot + `mcp.json` (MERGE/CREATE)); the skeletons `LICENSE.skeleton.md` (replaced by the license's verbatim +text), `NOTICE.src.md` (no license over `src/`; "clean-room" appears nowhere), `README.skeleton.md`, `CONTRIBUTING.skeleton.md` (the +five conduct rules); `.clang-format` (the community style per gen3-standards: 4 spaces, 80 cols, Attach, pointer left, InsertBraces; +provenance comment, not fetched) + `make-format.snippet.mk` (`format` / `format-check`, dotfiles excluded). **`tools/MANIFEST.md`** +(45 KB): 325 tool files by ladder phase from ONE read-only Explore agent's survey (coverage asserted 325/325 of the find; P1 2 Β· P2 26 +Β· P3 17 Β· P4 9 Β· P5 32 Β· P6 59 Β· P7 25 Β· P8 102 Β· P9 25 Β· P10 12 Β· project-only 16), each row = what it does + what it hard-codes, +framed as Phase-N TASKS until the split; the agent's ten "unsure" placements kept as given (the agent could not write the deliverable +file β€” read-only mode β€” so its inline result was transcribed; the seven `TODO(platform)` markers now include the manifest's). +**In the repo:** `tools/kit_lint.py` (six checks with denominators: fence-aware LEAK; the PLACEHOLDERS set-diff; SYNTAX by `bash -n` / +an in-memory `compile()` (the first cut used `py_compile`, which wrote a `__pycache__` INTO the kit, then refused `os.devnull`) / +JSON+YAML parse; the gitignore diff delegated; TODO counts; coverage β€” zero files refuses; `--selftest` = the R39 control, a planted +leak + a planted unlisted placeholder must be caught while a fenced line and a provenance line must not; rc 1/2), wired into +`tools-health` after the gitignore block; `decomp-architect/README.md` added to `doc_links` DEFAULT; the SETUP row (R21). The +PLACEHOLDERS "Used in" cells reconciled against a script listing where each placeholder actually landed (6 cells corrected; no +placeholder added β€” the 22-set held). **Verify:** `kit_lint --selftest` OK; `kit_lint` OK over 43 files (leak 0; placeholders used 22 == +listed 22; syntax 0 failures over 3 scripts; gitignore 71 lines identical; `TODO(platform)` 7, `TODO(phase-N)` 31); the audit template +`py_compile`s; the CI template parses; `bash -n` clean; `doc_links --strict` rc 0 (with the kit README in the set); **`make tools-health` +OK** (run DETACHED via `setsid nohup` into `.run/P33.5/tools_health_t11.log`, 5,903 lines β€” the foreground cap is 10 min and the chain +runs longer β€” with a background pid waiter; the first full run of the phase: sigs fresh, sdk-dual both legs, corpus/cdecl/binaries/text +audits, report + audit-digest, cookbook/gccmap/roster checks, doc_links, wiki_render, gitignore check, kit_lint selftest + run, xsig 8, +work_evidence, split_indicator 218/218). The report step regenerated `docs/story-timeline.md` (the 2026-09-07 row's commit count +43 β†’ 48; a generated file, committed β€” R75). **Deviations:** none from the plan's file list; the checkpoint's "audit template + +`config/firewall.txt`" landed as `templates/firewall.txt` + `templates/audit_public.template.py` + `firewall-fixture/` (three files, the +config separate from the code so a project edits the config only). + +## πŸ›‘ SESSION CHECKPOINT β€” Tasks 0–11 βœ“; NEXT = task 12 (kit part 3: `decomp-architect/SETUP.md`; **Max** β€” prompt Drew, R27; then the P6 rules check) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase31/32/33.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest (R1–R83), then -WAIT for Drew. Rebuild the harness task list (16 rows, R28) marking tasks 0–10 completed and task 11 in progress. +WAIT for Drew. Rebuild the harness task list (16 rows, R28) marking tasks 0–11 completed and task 12 in progress. ### 1. Where we are **Phase 33.5** (sub-phase; v1.32.0 β†’ v1.32.1), gate 1 approved 2026-09-07 by Drew in plan mode at Max; effort follows the plan's -column (Max for tasks 12 and 15 β€” prompt at each transition, R27; task 11 is xHigh). Baseline HEAD `80d45b29b`; task 0 = -`39d524991`; task 1 = `a0cf302e5`; task 2 = `d06923a06`; task 3 = `5d10a0d12`; task 4 = `9970f1e62`; task 5 = `a0d4ae836`; task 6 = -`6ec4786bd`; task 7 = `21c98ed5a`; task 8 = `ae71efe56`; task 9 = `caefe9872`; S90 checkpoint = `e1463d430`; task 10 = the commit -after this block (session S91, 2026-09-07). No build input changed; the fleet is 218/218 at the Phase-33 close. Tracked `.run/` = -868; `audit_public` OK; `gate_scan` PASS; **`doc_links --strict` rc 0 again** (it was rc 1 at `e1463d430` β€” see the task-10 log: -the firewall page's residue citation went UNTRACKED at task 8; fixed in task 10); pending list empty. **The kit directory now -holds:** `README.md`, `intake.decomp.md`, `decomp-architect.md`, `templates/registry-E.decomp.md` (G1–G65), -`templates/PLACEHOLDERS.md` (22 placeholders), `corpus/decomp-kernels.md` (DK-1…DK-64 + the museum), `memory-seed/` (16 seeds + -MEMORY.md, task 9). Tasks 11–13 build the rest; 14–15 close. **Already banked for task 14 (R31):** the `docs/decision-log.md` -entry "P33.5 S90" β€” task 14 must NOT write a second one; it still owes the wiki page `Start-a-new-decomp-project.md`, the -README/`Tools-from-this-project.md` rows, the SETUP rows for the kit and the checkers, and an accelerators entry if the dry-run -earns one. Nothing runs in the background; the SessionStart hook's headless Ghidra MCP was never used (no RE work; nothing under -`ghidra/` is tracked). The plan file `~/.claude/plans/max-effort-set-plan-encapsulated-muffin.md` is a copy of the "Approved -plan" section at the end of this file β€” this file is the one that counts. +column (Max for tasks 12 and 15 β€” prompt at each transition, R27). Baseline HEAD `80d45b29b`; task 0 = `39d524991`; task 1 = +`a0cf302e5`; task 2 = `d06923a06`; task 3 = `5d10a0d12`; task 4 = `9970f1e62`; task 5 = `a0d4ae836`; task 6 = `6ec4786bd`; task 7 = +`21c98ed5a`; task 8 = `ae71efe56`; task 9 = `caefe9872`; S90 checkpoint = `e1463d430`; task 10 = `83de29c02`; task 11 = the commit +after this block (session S91, 2026-09-07). No build input changed; the fleet is 218/218 at the Phase-33 close and +**`make tools-health` is OK on this tree (S91, `.run/P33.5/tools_health_t11.log`)**. Tracked `.run/` = 868; `audit_public` OK; +`gate_scan` PASS; `doc_links --strict` rc 0; pending list empty. **The kit directory holds everything except `SETUP.md`:** `README.md`, +`intake.decomp.md`, `decomp-architect.md`, `corpus/decomp-kernels.md` (DK-1…DK-64), `memory-seed/` (16 + MEMORY.md), +`tools/MANIFEST.md` (325 rows by phase), and `templates/` β€” `registry-E.decomp.md` (G1–G65), `PLACEHOLDERS.md` (the 22-set), +`gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, `firewall-fixture/{blob.bin,blob.sha1,README.md}`, +`no-rom.template.yml`, `docs-README.md`, `run-README.md`, `ops-setup.decomp.md`, `bootstrap.template.sh`, `CLAUDE.decomp-overlay.md`, +`pa-overlays.md` (7 blocks), `LICENSE.skeleton.md`, `NOTICE.src.md`, `README.skeleton.md`, `CONTRIBUTING.skeleton.md`, `.clang-format`, +`make-format.snippet.mk`. `tools/kit_lint.py` runs in tools-health (selftest + real). Tasks 12–13 finish the kit; 14–15 close. +**Already banked for task 14 (R31):** the `docs/decision-log.md` entry "P33.5 S90" β€” task 14 must NOT write a second one; it still owes +the wiki page `Start-a-new-decomp-project.md`, the README/`Tools-from-this-project.md` rows, the SETUP rows for the kit (the kit_lint +row exists), and an accelerators entry if the dry-run earns one. Nothing runs in the background; the headless Ghidra MCP was never +used (no RE work; nothing under `ghidra/` is tracked). The plan file `~/.claude/plans/max-effort-set-plan-encapsulated-muffin.md` is a +copy of the "Approved plan" section at the end of this file β€” this file is the one that counts. -### 2. What NEXT does (task 11, **xHigh**) β€” exact steps (plan D5, "Kit part 2") -**Read first:** `decomp-architect/templates/PLACEHOLDERS.md` (THE CONTRACT β€” every template below uses only the 22 placeholders -listed there, and every "Used in" cell must come true or be amended in the same change), `decomp-architect/README.md` ("What it -installs" is the file list), `decomp-architect/intake.decomp.md` Part D (the four fail-safes the CLAUDE overlay states) and Part C, -`docs/wiki/The-ROM-firewall.md` (the ```gitignore fence = `templates/gitignore.decomp` BYTE FOR BYTE β€” `tools/gitignore_template_check.py` -from task 7 already exists and currently SKIPS LOUDLY in tools-health until the template lands; rc 1 = drift, rc 2 = no template), -`docs/wiki/Docs-and-scratch-conventions.md` (the docs/scratch READMEs' content), `/mnt/z/Storage/git/ProjectArchitect/project-architect-2.0/templates/` -(`CURRENT_PHASE.template.md`, `PhaseEnd.template.md`, `effort-map.template.md`, `cookbook.template.md`, `ops-setup.template.md`, -`CLAUDE.template.md` β€” the overlays APPEND marked sections to what PA stamped from these; PA's Β§5.1 marked-section precedent is -`## Pre-existing instructions (merged at PA 2.0 install)`), `tools/audit_public.py` (the template is a de-BFM'd generalisation of -it: sources from `config/firewall.txt`, checks 1–4, the planted-fixture control), `.github/workflows/no-rom.yml`, -`tools/bootstrap.sh`, `docs/gen3-standards.md` Β§2 (the formatter settings: 4 spaces, 80 columns, braces on the same line, pointer on -the type β€” author `.clang-format` from these; cite the style guide as provenance, do not fetch it), `LICENSE`, `src/NOTICE.md`, -`THIRD_PARTY.md`, `CONTRIBUTING`-equivalent wiki page (`docs/wiki/Contributing.md` "AI use β€” conduct"), and `phase-ends/DIGEST.md`'s -shape (for the digest template) + this file's πŸ›‘ block (for the checkpoint template) + `PhaseEnd_Phase33.md`'s "What we believed" -section (for the PhaseEnd narrative axis). -**Write (all under `decomp-architect/`, de-BFM'd; the lint pattern is -`SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\.17\.|\bR[0-9]{1,2}\b|Β§[0-9]+` outside ```` ```calibration ```` fences and -`provenance:` lines β€” remember `Β§`+digit is banned, so no `Β§8`-style references to PA's sections; write "step 8"): -1. `templates/gitignore.decomp` β€” the wiki fence, byte for byte (extract it with a script, do not retype; then run - `tools/gitignore_template_check.py` β€” it must exit 0). -2. `templates/audit_public.template.py` β€” the audit: hash sources from `config/firewall.txt` (lines `required: ` / - `pending: ` / `fixture: `; refuse if zero sources resolve; fail on a missing required; warn loudly on a - missing pending), the four checks (purged paths, hash set, size threshold, the disassembly-shaped-content run β‰₯ 64 lines with the - asm-differ / objdump / splat-comment / label-block shapes from task 8), counts with denominators, `--paths` for the control; - `templates/firewall.txt` (the config with the fixture line and `pending:` manifest/contract lines tagged `TODO(phase-1)`/ - `TODO(phase-3)`); `templates/firewall-fixture/` (a 16-byte SYNTHETIC blob + its `.sha1`; the SETUP's control plants a copy in the - tree, asserts the audit FAILS, removes it, asserts PASS). -3. `templates/no-rom.template.yml` β€” the CI: the audit on push/PR/weekly; a compile-only job left as `TODO(phase-3)`. -4. `templates/docs-README.md`, `templates/run-README.md` β€” the conventions page's two tables (kind of knowledge β†’ home; the - `.run/` rules: ignored by contents, the dated three-line `!` idiom, the R20 test "commit what a rerun cannot reproduce", the - per-session layout, never `git clean -x`, a tracked scratch file is published). -5. `templates/ops-setup.decomp.md` β€” the rows appended to PA's ops-setup: version pins (`{{TOOLCHAIN_TRIPLE}}` as `TODO(phase-4)`, - `{{COMPILER_FAMILY}}`), the extract/fleet-check commands (`{{EXTRACT_CMD}}`, `{{FLEET_CHECK_CMD}}`), the oracles - (`{{DISASSEMBLER_MCP}}`, `{{EMULATOR_BRIDGE}}`), the machine-local dump path (`{{DUMP_PATH}}`), the git posture - (`{{PUBLIC_OR_PRIVATE}}`), the tooling-inventory rows for the audit and the format target. -6. `templates/bootstrap.template.sh` β€” the fresh-clone bootstrap skeleton (toolchain fetch + verify by checksum, venv, submodules, - the fingerprint ladder as a Phase-4 task); `bash -n` clean. -7. `templates/CLAUDE.decomp-overlay.md` β€” the marked section appended to CLAUDE.md: the four fail-safes (never commit game-derived - bytes; a match is byte-for-byte AND the whole-binary hash; never `git clean -x` here; the byte gate is the only claim), the - session-start extras (the oracle ping only when the next task is RE; the cookbook by symptom; the checkpoint replayed - verbatim), the flywheel line naming `{{COOKBOOK_NAME}}`. -8. `templates/pa-overlays.md` β€” ONE file of fenced blocks, each headed by the target file and the marker text: the digest - (synopses-only) template; the πŸ›‘ SESSION CHECKPOINT block for CURRENT_PHASE (sections 0–3 as in this file, de-BFM'd); the - PhaseEnd narrative axis ("What we believed, what failed and why it looked right, what it cost, what we would do sooner"); - the effort-map rows (Max on the plan, the PhaseEnd, the compiler pin, the segmentation decision, any wall verdict; breadth for - audits/bulk drafting); the cookbook entry shape (residual Β· mechanism Β· lever Β· byte proof) + the triage-table skeleton keyed by - symptom; the wave-playbook skeleton (the ten-step spine from `decomp-architect.md` part 3 with an empty "measurement" column per - guard); `settings.json` (the SessionEnd hook PA already set + a SessionStart hook slot for the disassembler server) and - `mcp.json` (a loopback SSE server entry with `{{DISASSEMBLER_MCP}}`). -9. The skeletons: `templates/LICENSE.skeleton.md` (`{{LICENSE_CHOICE}}` + the no-license-over-src statement), `templates/NOTICE.src.md` - (no license asserted over the decompiled source; "clean-room" appears nowhere), `templates/README.skeleton.md` - (`{{PROJECT_NAME}}`, `{{GAME_TITLE}}`, `{{PLATFORM}}`, `{{GAME_SERIAL}}`, `{{PROJECT_GOALS}}`, `{{COMMUNITY_WORK}}`, `{{AI_DISCLOSURE}}`, - `{{PUBLIC_OR_PRIVATE}}`, a generated-numbers block marked `TODO(phase-5)`), `templates/CONTRIBUTING.skeleton.md` (bring your own - dump; the AI-conduct section = G61–G65; the target's own AI policy line). -10. `templates/.clang-format` + `templates/make-format.snippet.mk` (a `format` target over `src/`). -11. `tools/MANIFEST.md` β€” the portable tools of the source project BY LADDER PHASE (extract/manifest; oracles/load map; baseline; - pin/probe; census/harness/reports; signatures/twin band/dedup/families/reconcile/carves/draw filter; map/dumps/permuter/ - reproducers; cards/lanes/wave/gates/recovery/harvest; contract/bootstrap/progress/audit; the readability levers) β€” one row per - tool: what it does, what it hard-codes (paths, the compiler, the platform), "copy after the split"; written as Phase-1 TASKS. - Breadth-shaped: survey `tools/` with ONE read-only Explore agent (the SETUP rows in `docs/SETUP.md` are the index) β€” the - manifest names tools by their file name (allowed) but never cites this repo's paths. -12. `tools/kit_lint.py` (in the REPO's `tools/`, wired into `make tools-health`): fence-aware leak grep (strip ```` ```calibration ```` - blocks and `provenance:` lines, then the pattern) with `--paths`; the placeholder set-diff against `PLACEHOLDERS.md`; `bash -n` - on every `.sh`; `py_compile` on every `.py`; the gitignore template diff (delegate to `gitignore_template_check.py`); the - `TODO(platform)`/`TODO(phase-N)` counts printed with denominators; an R39 control: a planted leak line must FAIL it. Add - `decomp-architect/README.md` to `doc_links.py`'s DEFAULT set. SETUP rows for `kit_lint` (R21). -**Verify:** `kit_lint` clean; `gitignore_template_check` rc 0; `bash -n`/`py_compile` clean; the planted-leak control fails; -`doc_links --strict` rc 0; `make tools-health` (foreground, ~15-min timeout β€” the FIRST full run since task 7's wiring; read it). -Log + checkpoint; commit by explicit path. Task 12 is Max β€” prompt Drew to raise (R27). +### 2. What NEXT does (task 12, **Max**) β€” exact steps (plan D5, "Kit part 3 β€” SETUP.md") +**Read first:** `/mnt/z/Storage/git/ProjectArchitect/project-architect-2.0/SETUP.md` in full (THE SHAPE TO MIRROR: the execution +contract, `βœ“ Verify` per step, idempotent via CURRENT_PHASE.md checkboxes, commit by explicit path, never push, the install run as a +phase, the hard stop) and its `templates/CURRENT_PHASE.template.md` + `PhaseEnd.template.md` (the kit's Phase 0.5 uses them); +`decomp-architect/README.md` (the promises SETUP must keep: what it installs / does NOT, the version pin, Path A only); +`decomp-architect/templates/PLACEHOLDERS.md` (which step fills which placeholder β€” SETUP's steps must match its "Filled" column); +`decomp-architect/templates/pa-overlays.md` (each block's target/marker/step β€” SETUP applies them by marker, idempotently); +`decomp-architect/templates/firewall-fixture/README.md` (the control's exact procedure); `decomp-architect/intake.decomp.md` Part D +(the Β§E-empty precondition SETUP Step 7 asserts) and Part C; the plan's D5 + D6 in the "Approved plan" section below (the dry-run's +guardrails shape what SETUP may write: never under `~/.claude`, every git call `git -C `, no `git clean`, no relative `rm -rf`). +**Write `decomp-architect/SETUP.md`** β€” headings `## Step 0` … `## Step 10` (NOT `Β§N`: the lint bans `Β§`+digit; refer to PA's sections as +"step N" too). Contents per step: **Step 0** the execution contract (Claude Code from the project root with `KIT=./decomp-architect`; +in plan mode the plan is "execute Steps 1–10 in order", no redesign; templates copied verbatim with `cp`, placeholders filled with the +Edit tool, never retyped; every step self-verifies, STOP on a failed verify; idempotent β€” re-run resumes from the first unticked +checkbox of the Phase-0.5 CURRENT_PHASE; commits by explicit path, no trailers, never push; the developer answers Step 2 once β€” or +`--answers `: "if the file exists every question is answered from it; an unanswered question STOPs, never defaults"); +**Step 1** prerequisites β€” assert `grep -q '^> \*\*Version:\*\* 2\.0' docs/project-architect.md`, `RULES_REGISTRY.md` and +`phase-ends/` exist, `PROJECT_CONTEXT.md` exists and carries the ladder (grep "Phase 0.5" or the intake's phase names), the PA install +phase is closed (a `PhaseEnd_Phase0.md`), registry Β§E holds only the one-line pointer (else STOP with the renumber-by-offset fallback +spelled out), and the repo is a PA Path-A project β€” an existing decomp repository (any `src/*.c`, a splitter config, a `Makefile` with a +build) is an explicit STOP; then create the Phase-0.5 `CURRENT_PHASE.md` from PA's template with Steps 2–10 as checkboxes; +**Step 2** the game interview β†’ the 13 copy-time placeholders (one compact round; `--answers` unattended; record the answers into the +Phase-0.5 log); **Step 3** the firewall β€” append `templates/gitignore.decomp` to `.gitignore` under a dated marker (NEVER replace PA's +block), copy `firewall.txt` β†’ `config/firewall.txt` (fill `{{TARGET_BINARY}}`), copy `firewall-fixture/blob.sha1` β†’ +`config/firewall-fixture.sha1`, copy `audit_public.template.py` β†’ `tools/audit_public.py` (+x), copy `no-rom.template.yml` β†’ +`.github/workflows/no-rom.yml`; **the control**: `mkdir -p .run/firewall-control && cp $KIT/templates/firewall-fixture/blob.bin +.run/firewall-control/planted.bin && python3 tools/audit_public.py --paths .run/firewall-control/planted.bin` must exit 1 naming the +planted file; `rm` it; `python3 tools/audit_public.py` must exit 0 (`git check-ignore` positive probes `disks/x.bin asm/x.s +extracted/x` and negative probes `src/main.c config/x.yaml`); **Step 4** layout β€” `mkdir -p src config tools docs .run disks +extracted dumps` + `docs/README.md` from `docs-README.md`, `.run/README.md` from `run-README.md` (fill PROJECT_NAME/INSTALL_DATE/ +CONTAINER_LAYOUT); **Step 5** `bootstrap.template.sh` β†’ `tools/bootstrap.sh` (+x, `bash -n`); **Step 6** the flywheel skeleton β€” +`pa-overlays.md` block 6 β†’ `docs/wave-playbook.md` (CREATE), `tools/MANIFEST.md` β†’ `docs/tools-manifest.md` (the Phase-1 task list), +`corpus/decomp-kernels.md` β†’ `docs/decomp-kernels.md` (the intake's ladder cites DK ids β€” they must survive package deletion); +**Step 7** registry β€” append `registry-E.decomp.md`'s body under Β§E after asserting the pointer line (marker = the seed's `## Β§E β€”` line; +idempotent); **Step 8** overlays + skeletons β€” `CLAUDE.decomp-overlay.md` appended to `CLAUDE.md` (marker `## Decomp fail-safes`); +`pa-overlays.md` blocks 1 (CREATE `phase-ends/DIGEST.md`), 2 (APPEND to `phase-ends/CURRENT_PHASE.template.md`), 3 (APPEND to +`phase-ends/PhaseEnd.template.md`), 4 (APPEND to `docs/effort-map.md`), 5 (APPEND to `docs/`), 7 (MERGE `.claude/settings.json` +adding only absent keys, CREATE `.mcp.json`); `ops-setup.decomp.md` appended to `docs/ops-setup.md`; `decomp-architect.md` β†’ +`docs/decomp-architect.md`; the skeletons β†’ `LICENSE` (only if absent β€” PA does not create one; else a `LICENSE.decomp-note.md`?) NO: +write `LICENSE` from the skeleton only when no LICENSE exists, otherwise STOP and ask; `src/NOTICE.md`, `README.md` (PA creates none β€” +create; if one exists, append under a marker), `CONTRIBUTING.md`, `.clang-format`, `make-format.snippet.mk` appended to `Makefile` +(create a minimal Makefile if absent); generation-time placeholders written as literal `TODO(phase-N)` lines; **Step 9** memory seed +(`memory-seed/*.md` β†’ `.claude-state/memory/`, MEMORY.md rows APPENDED β€” PA's 16 + the kit's 16; the `upstream: PA` files included) + +"seeded memories activate from the NEXT session"; **Step 10** verify β€” the leftover-placeholder audit `grep -rn "{{" --include="*.md" +--include="*.txt" --include="*.json" --include="*.sh" --include="*.py" .` must hit ONLY `decomp-architect/` and `docs/decomp-architect.md`; +`tools/audit_public.py` OK; `bash -n tools/bootstrap.sh`; JSON parses; the registry recites G1–G65; the install MANIFEST (the file list) +printed and written to `.run/decomp-architect-install-manifest.txt`; append `decomp-architect/` to `.gitignore`; write +`phase-ends/PhaseEnd_Phase0.5.md` (PA's template + the narrative axis) and `git mv` the Phase-0.5 CURRENT_PHASE to +`phase-ends/logs/PhaseLog_0.5.md`; commit by explicit path (`chore: decomp-architect Phase 0.5 β€” the decomp overlay installed`); the +final message: the manifest, "you may delete decomp-architect/", "start a fresh session and say Begin Phase 1", the plain-English +recap; **πŸ›‘ HARD STOP**. **The honesty section** (near the top): the installer verifies documents, configuration and the firewall audit; +it does not install a byte gate, a splitter config, a permuter harness, a decompiler context or a differential harness β€” those are +`docs/tools-manifest.md`'s Phase-N tasks. **Verify (task 12):** `kit_lint` clean (SETUP.md is in the package β€” the lint runs over it: +no `Β§`+digit, no `R`+digits, no source-project literals; every `{{…}}` it names is in the 22-set); a structural read-through against +PA's SETUP.md (every PA mechanism has its counterpart: contract, verify lines, idempotence, explicit-path commits, the phase file, +the hard stop); every "Filled" column of PLACEHOLDERS.md names a step that exists. Log + checkpoint; commit by explicit path. +**Then the P6 rules check** (re-read CLAUDE.md's Mandatory Behavior + the fail-safes + DIGEST Β§3, state "Rules check β€” re-read +complete. Continuing with task 13"). Task 13 is xHigh β€” prompt Drew to drop back (R27). ### 3. Standing facts for every task of this phase - One commit per task, after this file's log line (R8/R42 form); commit by explicit path; no trailers (R5); Drew pushes (R6). @@ -588,7 +622,10 @@ Log + checkpoint; commit by explicit path. Task 12 is Max β€” prompt Drew to rai - `purge_set.txt` is never edited in this phase. Never `git clean -x`. The untracked 29 GB of `.run/` is out of scope. - Effort transitions are prompted and waited for (R27): Max for tasks 12 and 15; xHigh otherwise. - The kit's vocabulary (task 10): rules by G-id, kernels by DK-id, PA's sections as "step N", the kit's SETUP as "Step N"; source - numbers only in ```` ```calibration ```` fences; the placeholder set is `PLACEHOLDERS.md`'s 22 and nothing else. + numbers only in ```` ```calibration ```` fences; the placeholder set is `PLACEHOLDERS.md`'s 22 and nothing else (`kit_lint` enforces). +- **`make tools-health` runs longer than the 10-minute foreground cap:** launch it detached (`setsid nohup make tools-health > + .run/P33.5/ 2>&1 < /dev/null &`), then wait on the pid from a background Bash (`while kill -0 ; do sleep 10; done`); never + as a harness background task (the memory guard killed one in S88), never sleep-poll in the foreground. --- diff --git a/tools/doc_links.py b/tools/doc_links.py index 4609292b7..b45d34af9 100644 --- a/tools/doc_links.py +++ b/tools/doc_links.py @@ -41,7 +41,8 @@ DEFAULT = ["README.md", "THIRD_PARTY.md", "CLAUDE.md", "src/NOTICE.md", "tools/R "docs/gcc-2.7.2-map/README.md", "tools/xsig/README.md", "docs/permuter-ils.md", "docs/matching-drafter-pipeline.md", "docs/gen3-handoff.md", "docs/decompme-preset.md", "docs/outreach/archipelago.md", "docs/outreach/tools-announcement.md", - "docs/gen3-standards.md", "docs/phase34-seed.md"] + "docs/gen3-standards.md", "docs/phase34-seed.md", + "decomp-architect/README.md"] # P33.5 task 11: the kit's README is public-facing # whole directories in the default set (P33 F3): the wiki pages and the how-to chapters β€” every file, so a new page is # checked the moment it exists (the glob is expanded at run time; the count is printed with the rest, R41) DEFAULT_GLOBS = ["docs/wiki/*.md", "docs/how-to-ai-decomp/*.md"] diff --git a/tools/kit_lint.py b/tools/kit_lint.py new file mode 100644 index 000000000..1ff63197e --- /dev/null +++ b/tools/kit_lint.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""kit_lint.py β€” the day-one decomp kit (decomp-architect/) stays free of the source project's names, paths, addresses and +rule numbers, honours its own placeholder contract, and ships scripts that parse (P33.5 task 11; in tools-health). + + tools/kit_lint.py # the whole package: every file under decomp-architect/ + tools/kit_lint.py --paths a b … # an explicit file list (the checks that make sense per file) + tools/kit_lint.py --selftest # the R39 control: a planted leak line and a planted unlisted placeholder MUST fail + +Six checks, each printed with its denominator (R41), exit 1 on any finding, exit 2 when the package is absent (R43): + 1. LEAK β€” no line outside a ```calibration fence and off a `provenance:` line matches the pattern + `SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\\.17\\.|\\bR[0-9]{1,2}\\b|Β§[0-9]+` (the kit refers to rules + by G-id, to kernels by DK-id, to sections by "step N"; source-project figures live only inside calibration fences). + The fence-aware form matters: `grep -v calibration` drops only the lines containing that WORD, not the fence contents. + 2. PLACEHOLDERS β€” the set of `{{NAME}}` tokens used anywhere under the package equals the set listed (backticked) in + templates/PLACEHOLDERS.md; a used-but-unlisted or listed-but-unused placeholder fails (the contract tasks 11/12 honour). + 3. SYNTAX β€” every `*.sh` passes `bash -n`; every `*.py` compiles (the `compile` builtin, in memory β€” no bytecode is written into the kit); every `*.yml`/`*.json` block parses + (JSON only for files whose whole content is JSON). + 4. GITIGNORE β€” templates/gitignore.decomp equals the ROM-firewall wiki page's fence (delegated to + tools/gitignore_template_check.py; rc 0 required). + 5. TODO counts β€” `TODO(platform)` and `TODO(phase-N)` occurrences, informational, with the file count as denominator + (the README says how many remain). + 6. COVERAGE β€” the number of files scanned is printed; zero files is a failure, never a pass. +Controls (R39): `--selftest` writes a leak line, a fenced (allowed) line, a provenance (allowed) line and an unlisted +placeholder into a scratch file under .run/ and asserts checks 1 and 2 catch exactly the two planted defects. +""" +import json +import pathlib +import re +import subprocess +import sys + +REPO = pathlib.Path(__file__).resolve().parent.parent +KIT = REPO / "decomp-architect" +PLACEHOLDERS_MD = KIT / "templates" / "PLACEHOLDERS.md" +LEAK_RE = re.compile(r"SLUS|Musashi|BFM|Druthulu|func_80|ov_SC|/home/musashi|/mnt/z|172\.17\.|\bR[0-9]{1,2}\b|Β§[0-9]+") +PH_RE = re.compile(r"\{\{[A-Z_0-9]+\}\}") +PH_LISTED_RE = re.compile(r"`(\{\{[A-Z_0-9]+\}\})`") + + +def kit_files(): + return sorted(p for p in KIT.rglob("*") if p.is_file() and "__pycache__" not in p.parts) + + +def leak_lines(text): + """(lineno, line) for every leak outside a calibration fence and off a provenance line.""" + out, in_cal = [], False + for i, line in enumerate(text.splitlines(), 1): + if line.startswith("```"): + if in_cal: + in_cal = False + continue + if line.strip().startswith("```calibration"): + in_cal = True + continue + if in_cal or "provenance:" in line: + continue + if LEAK_RE.search(line): + out.append((i, line)) + return out + + +def placeholders_used(files): + used = {} + for p in files: + for m in PH_RE.findall(p.read_text(encoding="utf-8", errors="replace")): + used.setdefault(m, set()).add(p) + return used + + +def placeholders_listed(): + if not PLACEHOLDERS_MD.exists(): + sys.exit(f"kit_lint: {PLACEHOLDERS_MD.relative_to(REPO)} missing β€” the placeholder contract cannot be checked (R43)") + return set(PH_LISTED_RE.findall(PLACEHOLDERS_MD.read_text(encoding="utf-8"))) + + +def check_syntax(files): + bad = [] + for p in files: + try: + if p.suffix == ".sh": + r = subprocess.run(["bash", "-n", str(p)], capture_output=True, text=True) + if r.returncode: + bad.append((p, r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "bash -n failed")) + elif p.suffix == ".py": + compile(p.read_text(encoding="utf-8"), str(p), "exec") # in memory: never writes bytecode into the kit + elif p.suffix == ".json": + json.loads(p.read_text(encoding="utf-8")) + elif p.suffix in (".yml", ".yaml"): + try: + import yaml # noqa: F401 + except ImportError: + continue + yaml.safe_load(p.read_text(encoding="utf-8")) + except Exception as e: # noqa: BLE001 β€” the message is the finding + bad.append((p, str(e).splitlines()[0])) + return bad + + +def run_checks(files, do_gitignore=True): + findings = 0 + n = len(files) + if n == 0: + print("kit_lint: 0 files to check β€” refusing to pass on an empty package (R43)") + return 2 + # 1 LEAK + leaks = 0 + for p in files: + for ln, line in leak_lines(p.read_text(encoding="utf-8", errors="replace")): + leaks += 1 + print(f" LEAK {p.relative_to(REPO)}:{ln}: {line.strip()[:110]}") + print(f"kit_lint: leak check β€” {leaks} finding(s) over {n} files (calibration fences + provenance lines exempt)") + findings += leaks + # 2 PLACEHOLDERS + used = placeholders_used(files) + listed = placeholders_listed() + extra, unused = sorted(set(used) - listed), sorted(listed - set(used)) + for ph in extra: + print(f" PLACEHOLDER used but not listed in PLACEHOLDERS.md: {ph} ({', '.join(str(q.relative_to(REPO)) for q in sorted(used[ph])[:3])})") + for ph in unused: + print(f" PLACEHOLDER listed but used nowhere: {ph}") + print(f"kit_lint: placeholders β€” used {len(used)}, listed {len(listed)}, unlisted {len(extra)}, unused {len(unused)}") + findings += len(extra) + len(unused) + # 3 SYNTAX + bad = check_syntax(files) + for p, why in bad: + print(f" SYNTAX {p.relative_to(REPO)}: {why}") + n_scripts = sum(1 for p in files if p.suffix in (".sh", ".py", ".json", ".yml", ".yaml")) + print(f"kit_lint: syntax β€” {len(bad)} failure(s) over {n_scripts} script/config files") + findings += len(bad) + # 4 GITIGNORE + if do_gitignore: + r = subprocess.run([sys.executable, str(REPO / "tools" / "gitignore_template_check.py")], capture_output=True, text=True) + print(f"kit_lint: gitignore template β€” {(r.stdout.strip().splitlines() or ['(no output)'])[-1]}") + if r.returncode: + findings += 1 + # 5 TODO counts (informational) + text = "\n".join(p.read_text(encoding="utf-8", errors="replace") for p in files) + print(f"kit_lint: TODO(platform) {text.count('TODO(platform)')}, TODO(phase-N) {len(re.findall(r'TODO\(phase-\d+\)', text))} over {n} files") + return 1 if findings else 0 + + +def selftest(): + scratch = REPO / ".run" / "kit_lint_selftest" + scratch.mkdir(parents=True, exist_ok=True) + f = scratch / "planted.md" + f.write_text( + "# planted control\n" + "a leak line that names the source project: BFM and its rule R32\n" # MUST be caught + "```calibration\nsource project: BFM 218/218, R22 after every batch\n```\n" # allowed + "provenance: BFM R32 (Phase 26)\n" # allowed + "an unlisted placeholder {{NOT_IN_THE_CONTRACT}}\n", # MUST be caught + encoding="utf-8") + leaks = leak_lines(f.read_text(encoding="utf-8")) + used = placeholders_used([f]) + extra = set(used) - placeholders_listed() + ok = len(leaks) == 1 and leaks[0][0] == 2 and extra == {"{{NOT_IN_THE_CONTRACT}}"} + print(f"kit_lint --selftest: leak lines caught {len(leaks)} (expected 1, at line 2: {leaks[0][0] if leaks else '-'}), " + f"unlisted placeholders caught {sorted(extra)} (expected ['{{{{NOT_IN_THE_CONTRACT}}}}']) β†’ {'OK' if ok else 'FAIL'}") + f.unlink() + return 0 if ok else 1 + + +def main(argv): + if "--selftest" in argv: + return selftest() + if not KIT.is_dir(): + print(f"kit_lint: {KIT.relative_to(REPO)} does not exist β€” nothing to lint (R43)") + return 2 + if "--paths" in argv: + files = [pathlib.Path(a).resolve() for a in argv[argv.index("--paths") + 1:]] + return run_checks(files, do_gitignore=False) + rc = run_checks(kit_files()) + print("kit_lint: OK" if rc == 0 else f"kit_lint: FAIL (rc {rc})") + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))