diff --git a/Makefile b/Makefile index 3ace65418f..e14711fd48 100644 --- a/Makefile +++ b/Makefile @@ -144,12 +144,13 @@ CC1_SMOKE_FLAGS := -quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-lin BINUTILS_WARN_MAJOR := 2 BINUTILS_WARN_MINOR := 38 -.PHONY: help check-env disc-extract extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health +.PHONY: help bootstrap check-env disc-extract extract build check expected clean report sig-refresh sig-overlays sig-resident sig-main sdk-dual build-all check-all audit-corpus audit-cdecl audit-binaries audit-text-sources audit-digest audit-frontier tools-health # ----------------------------------------------------------------------------- help: @echo "BFM-decomp — make targets (218 binaries: main + resident + 141 overlays + 75 modules; BINARY= scopes a target)" - echo " make check-env toolchain preflight (python/venv/cc1/maspsx/binutils; the extracted EXE if present)" + echo " make bootstrap fresh-clone setup: apt check (printed), venv, submodules, the two cc1 tarballs, then check-env" + echo " make check-env toolchain preflight (python/venv/cc1/maspsx/binutils/headers; the extracted EXE + payload census)" echo " make disc-extract regenerate extracted/ from YOUR redump dump in disks/ and verify it against the committed manifest (H1: no ROM in the repo)" echo " make extract splat split one binary -> asm/, the linker script (runs disc-extract if its payload is absent)" echo " make extract-all disc-extract + splat split every binary (main first, then parallel)" @@ -518,6 +519,11 @@ sig-modules: echo "sig-modules: signed $$n modules (of $(words $(MODULE_BINARIES)) onboarded)" if [ "$$bad" -ne 0 ]; then echo "[FAIL] sig-modules: $$bad module(s) failed to sign"; exit 1; fi +# bootstrap (P33 B3): the fresh-clone setup — apt presence (printed, never run), the venv from +# requirements-python.txt, the submodules, the two cc1 tarballs verified + extracted, then check-env. +bootstrap: + @tools/bootstrap.sh + # ----------------------------------------------------------------------------- # check-env: assert every Phase-4 toolchain component. Runs ALL checks (does not # stop at the first failure) so the report is complete, then exits nonzero if any @@ -564,6 +570,22 @@ check-env: else echo "[FAIL] $(MASPSX) missing (run: git submodule update --init)"; fail=1 fi + # 4b) the other three submodules (P33 B3) — matching tooling, not build inputs: WARN, not FAIL + for sub in tools/asm-differ tools/m2c tools/decomp-permuter; do + if [ -n "$$(ls -A "$$sub" 2>/dev/null)" ]; then + echo "[PASS] submodule populated: $$sub" + else + echo "[WARN] submodule empty: $$sub (matching tooling only; run: git submodule update --init)" + fi + done + # 4c) the four TRACKED splat preset headers (P33 B1) — assembled into every object + for h in include/include_asm.h include/macro.inc include/labels.inc include/gte_macros.inc; do + if [ -f "$$h" ]; then + echo "[PASS] preset header present: $$h" + else + echo "[FAIL] $$h missing (tracked since P33 B1; 'make extract' regenerates it)"; fail=1 + fi + done # 5) mipsel binutils on PATH (as / ld / objcopy) for t in $(AS) $(LD) $(OBJCOPY); do if command -v $$t >/dev/null 2>&1; then @@ -609,6 +631,9 @@ check-env: echo "[FAIL] $(EXTRACT_ROOT)/manifest.jsonl + manifest.sha1 missing (the committed extraction oracle)"; fail=1 fi if [ -f "$(DISC_TRACK1)" ]; then echo "[INFO] disc dump present: '$(DISC_TRACK1)'"; else echo "[INFO] no disc dump under $(DISC_DIR)/ (needed only to (re)generate extracted/)"; fi + # 8) the extracted payload census (P33 B3): how many of the fleet's inputs exist (INFO — disc-extract makes them) + present=0; for p in $(foreach b,$(BINARIES),$($(b)_EXE)); do [ -f "$$p" ] && present=$$((present+1)); done + echo "[INFO] extracted payloads present: $$present / $(words $(BINARIES)) binaries$$( [ "$$present" -eq $(words $(BINARIES)) ] || echo ' — run: make disc-extract')" echo if [ "$$fail" -ne 0 ]; then echo "check-env: FAIL — see the [FAIL] lines above." diff --git a/docs/SETUP.md b/docs/SETUP.md index dda3ad767d..3bcd8e29b8 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1069,6 +1069,19 @@ fills fast). Nothing is leaking — but the host does not get the memory back on * `gate_main` **REFUSES a draft containing its own `INCLUDE_ASM`** (substituting it restores the stub, so the build passes for free and the function counts as banked), and counts banks from the SOURCE. +### P33 B3 (S86, 2026-09-06) — `tools/bootstrap.sh` / `make bootstrap`: the fresh-clone setup +- **`tools/bootstrap.sh`** (idempotent, never sudo): apt packages of §4.5 checked with `dpkg -s` and the missing ones printed + as ONE `sudo apt-get install` line (never run); `.venv` created from `requirements-python.txt` (pinned; a no-op when + satisfied); `git submodule update --init`; `sha256sum --check tools/bin/CHECKSUMS.sha256` then each cc1 tarball extracted + into its own `tools/bin/gcc-2.7.2-{psx,cdk}/` unless `cc1` already exists; finally `make check-env` (its exit status is + the script's). `make bootstrap` is the alias. Idempotent run on a set-up tree: 1.4 s. +- `make check-env` additions: 4b) the other three submodules populated (WARN — matching tooling, not build inputs), + 4c) the four tracked splat preset headers present (FAIL), 8) `[INFO] extracted payloads present: N / 218 binaries` + (the census of the fleet's inputs; the hint names `make disc-extract`). +- The **fresh-clone proof** (the "stranger with their own dump" criterion, gen2-roadmap Phase-14 milestone (b)): `git clone + --no-local` → `tools/bootstrap.sh` → `ln -s disks` → `make disc-extract && make extract-all && make + check-all` → `check-all: 218 passed, 0 failed of 218`. Recorded in `phase-ends/CURRENT_PHASE.md` (B3) with timings. + ### P33 B2 (S86, 2026-09-06) — portable includes, audited - 34 lines in 19 `src/ov_*/…_jr_*.c` files (15 binaries) carried `#include "/home/musashi/bfm-decomp/src/shared/ engine_core.h"` (30) / `engine_types.h` (4) — the absolute spelling the jr-isolation tooling once wrote; byte-neutral diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 700f74ca22..4026bcba27 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -52,7 +52,7 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo - [x] **B1** `make disc-extract` (extract.py `--expect-manifest` / `--allow-missing-audio`; check-env; `.gitignore` re-tightened; the 4 splat preset headers tracked; `clean` fixed) — Max — see Log 2026-09-06 B1 - [x] **B2** 34 absolute includes → `../shared/` + portable-include audit + 15-binary re-gate — xHigh — see Log 2026-09-06 B2 -- [ ] **B3** `tools/bootstrap.sh` / `make bootstrap` + check-env extensions + fresh-clone proof 218/218 — xHigh +- [x] **B3** `tools/bootstrap.sh` / `make bootstrap` + check-env extensions + fresh-clone proof 218/218 — xHigh — see Log 2026-09-06 B3 - [ ] **B4** `tools/fetch_psyq.sh` + CHECKSUMS rows (20 lib40 LIBs, psyq-obj-parser) BEFORE `tools/psyq/` leaves git — xHigh - [ ] **B5** Ghidra regenerability (ExportAnnotations/ImportAnnotations/ghidra_rebuild.sh `--proof` on SLUS_007.26 + resident; roster; ExportSymbols R15 fix; path hardcodes; hooks) — Max, MCP stopped @@ -148,19 +148,26 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact angle-bracket / unresolvable / outside-repo `#include`); positive control BEFORE the fix: 34 offenders in 19 files, all ABSOLUTE (4,299 sources scanned); the sed → `../shared/engine_core.h` ×30 + `../shared/engine_types.h` ×4; `grep -rn /home/musashi src include` → 0; negative control → OK. Re-gate of the 15 binaries (derived from the diff, 5 in - parallel): 15/15 `[ OK ]` BYTE-IDENTICAL in 29 s wall (`.run/P33/b2_gate/.log`). SETUP: P33 B2 section (R21). + parallel): 15/15 `[ OK ]` BYTE-IDENTICAL in 29 s wall (`.run/P33/b2_gate/.log`). SETUP: P33 B2 section (R21). Commit + `commit:4017`. +- **2026-09-06 (S86) — B3 (bootstrap + the fresh-clone proof).** `tools/bootstrap.sh` (apt presence → printed install line; + venv from `requirements-python.txt`; submodules; cc1 tarballs sha256-checked + extracted into their own dirs; `make + check-env`), `make bootstrap`; check-env gained 4b (the other three submodules, WARN), 4c (the four tracked preset headers, + FAIL) and 8 (`[INFO] extracted payloads present: N / 218`). Idempotent run here: 1.4 s, OK. **The proof** (`.run/P33/ + b3_fresh.log`): `git clone --no-local` into `.run/P33/fresh` (the uncommitted Makefile + bootstrap.sh copied in) → + `tools/bootstrap.sh` created its own `.venv`, extracted both cc1s, fetched the submodules, `check-env: OK` → `disks` symlinked + → `disc-extract: OK` → `extract-all: 217 extracted, 0 failed of 217` → **`check-all: 218 passed, 0 failed of 218`**, EXIT=0, + **4 m 18 s wall** (user 45 m 48 s) — with NO SDK objects in the clone (`.run/obj40` absent), i.e. every binary built the way a + public user builds it. Clone deleted afterwards (3.1 GB). SETUP: P33 B3 section (R21). -## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1 ✓, B2 ✓; NEXT = B3 (`tools/bootstrap.sh` + `make bootstrap` + check-env extensions + the fresh-clone proof, xHigh) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B3 ✓; NEXT = B4 (`tools/fetch_psyq.sh` + the SDK checksums, xHigh) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **NEXT = B3** (xHigh: `tools/bootstrap.sh` / `make bootstrap` — apt presence check, venv + -requirements, submodules, cc1 tarball checksums + untar, `make check-env`; check-env gains submodule state, the 4 tracked -headers, disc/extracted state; the fresh-clone proof: `git clone --no-local` into `.run/P33/fresh/` → bootstrap → disc -symlinked → `make disc-extract && make extract-all && make check-all` → 218/218). If `git log -1 --format=%s` does not start -with `feat(phase-33): B2`, B2's commit did not land: `tools/audit_text_sources.py` must print OK and the 15 gate logs under -`.run/P33/b2_gate/` must each say BYTE-IDENTICAL before committing. +(R1–R73), then WAIT for Drew. **NEXT = B4** (xHigh: `tools/fetch_psyq.sh` — see §3 step 1 for the measured facts). If +`git log -1 --format=%s` does not start with `feat(phase-33): B3`, B3's commit did not land: `tools/bootstrap.sh` must print +`check-env: OK` (1–2 s on this tree) before committing; the proof log is `.run/P33/b3_fresh.log`. ### 1. Where we are **Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at @@ -229,16 +236,27 @@ items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2. ### 3. NEXT — in order 0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) · `.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not). -1. **B3** (xHigh) per the plan's Block B: `tools/bootstrap.sh` (idempotent; prints the apt line for missing packages from - SETUP §4.3, no sudo; `python3 -m venv .venv && .venv/bin/pip install -r requirements-python.txt`; `git submodule update - --init`; `sha256sum --check tools/bin/CHECKSUMS.sha256` then `tar xzf` each cc1 tarball into `tools/bin/gcc-2.7.2-{psx,cdk}/` - unless `cc1` exists; `make check-env`), `make bootstrap`; check-env additions (submodule gitlinks populated, the 4 tracked - headers present, extracted/ state). Proof = the "stranger with their own dump" criterion: `git clone --no-local ~/bfm-decomp - .run/P33/fresh` → `tools/bootstrap.sh` → `ln -s ~/bfm-decomp/disks .run/P33/fresh/disks` → `make disc-extract && make - extract-all && make check-all` in the clone → `check-all: 218 passed, 0 failed of 218`. Disk: the clone + its extracted/ + - build ≈ 3 GB (13 GB free); delete `.run/P33/fresh` afterwards. NOTE the clone's HEAD still tracks the EXE and the - purge paths until C3 — that is fine for the proof (the EXE arrives from git AND from disc-extract, same bytes). -2. Then B4 → B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box, +1. **B4** (xHigh) — measured facts (S86): the 20 PsyQ **4.0** LIBs come from the redump disc "PlayStation - Programmer Tools - + Run-time Library 4.0 (USA) (PC-CD ROM Release 2.0)" (DTL-S2002; our copy: `tools/psyq/PlayStation - Programmer Tools - + Run-time Library 4.0 (USA) (PC-CD ROM Release 2.0) (Track 1).bin`, 270 MB, gitignored; volume `PROGTOOL`) — its ISO + directory `PSX/LIB/` holds exactly the 20 `*.LIB` (names `LIBCD.LIB;1` …), and `tools/bfm_extract/iso9660.py` + (`img.find('PSX')` → `img.find('LIB', psx)` → `img.iter_directory(lib)` / `img.extract_file(name, lib)`) extracts them + byte-identical to `tools/psyq/lib40/` (LIBCD.LIB sha256 `db4e1a71…` both ways). The RTL 4.2 7z's archive.org URL is + `https://archive.org/download/play-station-programmer-tool-runtime-library-version-4.2.7z/PlayStation_Programmer_Tool_-_Runtime_Library_Version_4.2.7z` + (383,431 B; sha256 already recorded `e4f5a678…`). `psyq-obj-parser`: `https://github.com/decompme/compilers/releases/ + download/compilers/psyq-obj-parser.tar.gz` — tarball sha256 `353495f13f6756773cd905ba3a0743843232f95719e306fc6ee56cbf7d731a3e` + (3,507,359 B), the binary inside is BYTE-IDENTICAL to our tracked `tools/psyq/psyq-obj-parser` (sha256 + `4fba623a31bb5830ceea95b5081bc13504629686b0b4904c533b7be2a5f2377a`). Plan: `git mv tools/psyq/CHECKSUMS.sha256 + tools/psyq_CHECKSUMS.sha256` NOW (the plan put it in C3; doing it in B4 keeps the script's path final) and add rows for the + 20 lib40 LIBs + the parser tarball + the parser binary; `tools/fetch_psyq.sh [--disc ] [--from DIR]`: + (1) parser: download + verify or reuse; (2) RTL 4.2 7z: download + verify → `lib42/` + `lib421/` (the J421PD.ZIP payload) + → `.run/obj42/{libpad421,libapi42}` via `psyq_lib_split.py` + the parser; (3) the 4.0 LIBs: from `--disc` via the ISO + walker (new `tools/psyq_libs_from_disc.py`) or `--from DIR`, verified against the checksum rows → `tools/psyq/lib40/`; + REFUSE with the redump title when neither is given and `lib40/` is absent; (4) the builders: `tools/psyq_build_libs.sh` + (LIBCD LIBETC LIBGPU LIBMCRD LIBC2 LIBGTE LIBGS LIBSPU LIBSND LIBCARD), `tools/make_libgs.sh`, `tools/make_snd_used.py`, + `tools/make_apicard_used.py`; (5) `make -j$(nproc) sdk-dual`. Verify on this machine by rebuilding into the real dirs + (they are regenerable) and the dual passing; the checksum rows are verified by `sha256sum --check`. +2. Then B5 (MCP stopped) → B6 → B7 → B8 → A5 → B9/C3 … per the task list. After every task: tick the box, add a Log line, refresh this checkpoint block (the 🛑 block is the ONLY in-phase context the next session inherits), commit. Next P6 rules check after B4 (8 tasks done). `extracted/proto/` (sep8 + aug31 EXEs) is regenerated from the prototype discs by `tools/bfm_extract/extract_proto_exe.py` (docstring examples) — B5 needs both files. diff --git a/tools/bootstrap.sh b/tools/bootstrap.sh new file mode 100644 index 0000000000..b12093bcc2 --- /dev/null +++ b/tools/bootstrap.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# tools/bootstrap.sh — fresh-clone setup for BFM-decomp (P33 B3). Idempotent; never sudo. +# +# What a build needs, in order (docs/SETUP.md §4): +# 1. apt packages (§4.5) — only CHECKED here: the missing ones are printed as one install line. +# 2. the Python venv (§4.6) — created from requirements-python.txt (pinned; pycparser < 3.0). +# 3. the four submodules — maspsx (a build input), asm-differ, m2c, decomp-permuter. +# 4. the two vintage cc1s (§4.7) — the tracked old-gcc 0.17 tarballs, sha256-verified, each +# extracted into its OWN subdir (they are flat and share file names). +# 5. `make check-env` — the preflight; its exit status is this script's. +# Then: stage your redump dump under disks/ and run `make disc-extract && make extract-all && make check-all`. +set -euo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" +say() { printf 'bootstrap: %s\n' "$*"; } + +# 1) apt packages — presence only (dpkg); print the install line, never run it. +PKGS="bchunk binutils-mipsel-linux-gnu bsdmainutils clang-format coreutils curl gcc-mipsel-linux-gnu git libelf-dev make ninja-build p7zip-full python3-pip python3-venv unzip wget" +if command -v dpkg >/dev/null 2>&1; then + missing=() + for p in $PKGS; do dpkg -s "$p" >/dev/null 2>&1 || missing+=("$p"); done + if ((${#missing[@]})); then + say "MISSING apt packages (${#missing[@]}) — run this, then re-run bootstrap:" + printf ' sudo apt-get install -y %s\n' "${missing[*]}" + else + say "apt packages: all present" + fi +else + say "no dpkg on this system — install the equivalents of: $PKGS" +fi + +# 2) the venv (Python >= 3.12; pinned requirements) +if [ ! -x .venv/bin/python ]; then + say "creating .venv" + python3 -m venv .venv +fi +say "installing pinned Python requirements (no-op when satisfied)" +.venv/bin/pip install -q -r requirements-python.txt + +# 3) submodules (no-op when populated) +say "submodules: git submodule update --init" +git submodule update --init + +# 4) the vintage compilers: verify the tracked tarballs, extract each into its own dir once +say "cc1 tarballs: sha256sum --check tools/bin/CHECKSUMS.sha256" +( cd tools/bin && sha256sum --check --quiet CHECKSUMS.sha256 ) +for v in psx cdk; do + if [ ! -x "tools/bin/gcc-2.7.2-$v/cc1" ]; then + say "extracting tools/bin/gcc-2.7.2-$v.tar.gz -> tools/bin/gcc-2.7.2-$v/" + mkdir -p "tools/bin/gcc-2.7.2-$v" + tar xzf "tools/bin/gcc-2.7.2-$v.tar.gz" -C "tools/bin/gcc-2.7.2-$v" + else + say "cc1 present: tools/bin/gcc-2.7.2-$v/cc1" + fi +done + +# 5) the preflight (its exit status is ours) +say "make check-env" +make --no-print-directory check-env