mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
feat(phase-30 S44 I.1e): tools/new_binary.sh — one onboarder for every flat-blob class
- Generalized from new_overlay.sh: ALIAS + PAYLOAD + VRAM + optional TEXT_LO (file offset of code). Class registry chosen by alias prefix (ov_* -> overlays.mk, md_* -> modules.mk; modules.mk created with its contract header on first use). Fixes the two places new_overlay.sh re-hardcoded the slot literal instead of $VRAM (:39 TEXTHI, :44 CODEEND). - TEXT_LO wires the §154 module-id law end-to-end: sig bootstrap gets --text-lo (else 0 functions on 75/78 payloads), and the splat yaml gets [0x0, rodata, hdr] + shifted code start (the resident's leading-word trick — ONE shared template, no second file, R33). _TEXT_LO lands in the mk block as a VRAM for make sig-modules. - The sentinel-anchored 3-dict registrar + check.sha/symbols/extract/build steps reused verbatim. - new_overlay.sh is now a 30-line WRAPPER (same CLI, docs preserved, H5); exec's new_binary.sh with the overlay defaults.
This commit is contained in:
Drew T
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
# tools/new_binary.sh <ALIAS> <PAYLOAD> <VRAM> [TEXT_LO] — onboard ANY flat-blob binary (P30 S44).
|
||||
# =============================================================================
|
||||
# The generalized form of new_overlay.sh (Phase 13), for every flat-payload binary class:
|
||||
# * location overlays (ov_*, shared slot 0x80128158, payload 0.4.dec / 1.4.dec / raw 0.1 / 1.1)
|
||||
# * md_* modules (own slot: 0x800CAE08 / 0x800CCB1C / 0x801A00D8 / ..., §S44 loader table)
|
||||
# One command: instantiate config/splat.us.overlay.template.yaml, compute the build-identity hash +
|
||||
# code/data boundary, register in the class registry (config/overlays.mk for ov_*, config/modules.mk
|
||||
# for md_* — the Makefile -include's both; its body is NEVER touched) and in the report/diff tools'
|
||||
# BINARIES dicts (sentinel-anchored, idempotent, ast-checked), then extract+build to byte-verify.
|
||||
# Idempotent: re-running is a clean no-op.
|
||||
#
|
||||
# TEXT_LO (optional, default 0) = FILE OFFSET where code begins. The §154 module-id law: 75/78
|
||||
# payloads open with a module-id word (sometimes + a fn-ptr table) — code is NOT at offset 0. A
|
||||
# non-zero TEXT_LO (a) carves [0x0, rodata, hdr] ahead of the code (the resident's leading-word
|
||||
# trick, SETUP §6.7), and (b) makes the sig bootstrap start at the right vram (--text-lo), without
|
||||
# which sig_image finds 0 functions (§154-A).
|
||||
#
|
||||
# tools/new_binary.sh ov_MAIN_012 extracted/retail/MAIN.CD.dir/FILE_012.dir/1.1 0x80128158
|
||||
# tools/new_binary.sh md_MAIN_013 extracted/retail/MAIN.CD.dir/FILE_013.dir/1.1 0x800CAE08 0x4
|
||||
# tools/new_binary.sh md_SC07_004 extracted/retail/SC07.CD.dir/FILE_004.dir/1.1 0x801A00D8 0x158
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.." # repo root
|
||||
|
||||
[ $# -ge 3 ] && [ $# -le 4 ] || {
|
||||
echo "usage: tools/new_binary.sh <ALIAS> <PAYLOAD> <VRAM> [TEXT_LO_fileoff]"; exit 2; }
|
||||
ALIAS="$1"; DEC="$2"; VRAM="$3"; TLO="${4:-0x0}"
|
||||
PYV=".venv/bin/python"
|
||||
|
||||
case "$ALIAS" in
|
||||
ov_*) MK="config/overlays.mk"; MKVAR="OVERLAY_BINARIES"; KIND="location overlay" ;;
|
||||
md_*) MK="config/modules.mk"; MKVAR="MODULE_BINARIES"; KIND="module" ;;
|
||||
*) echo "ERROR: alias must start ov_ or md_ (got '$ALIAS') — the registry is chosen by class"; exit 2 ;;
|
||||
esac
|
||||
|
||||
[ -f "$DEC" ] || { echo "ERROR: payload not found: $DEC (run extraction first)"; exit 1; }
|
||||
|
||||
SHA1="$(sha1sum "$DEC" | cut -d' ' -f1)"
|
||||
EOFSZ="$(stat -c%s "$DEC")"
|
||||
EOFHEX="$(printf '0x%X' "$EOFSZ")"
|
||||
TEXTHI="$(printf '0x%X' $((VRAM + EOFSZ)))"
|
||||
TLO_VRAM="$(printf '0x%X' $((VRAM + TLO)))"
|
||||
|
||||
# sign (bootstrap honors the header via --text-lo, §154-A); code_end = last function's end fileoff
|
||||
SIG=".run/sig.${ALIAS}.jsonl"
|
||||
if [ ! -f "$SIG" ]; then
|
||||
if [ $((TLO)) -ne 0 ]; then
|
||||
$PYV tools/sig_image.py --image "$DEC" --vram-base "$VRAM" --bootstrap --text-lo "$TLO_VRAM" --name "$ALIAS" >/dev/null
|
||||
else
|
||||
$PYV tools/sig_image.py --image "$DEC" --vram-base "$VRAM" --bootstrap --name "$ALIAS" >/dev/null
|
||||
fi
|
||||
fi
|
||||
[ -s "$SIG" ] || { echo "ERROR: $SIG is empty — sig_image found no functions (wrong VRAM/TEXT_LO? §154)"; exit 1; }
|
||||
CODEEND="$(tail -1 "$SIG" | VRAM="$VRAM" $PYV -c "
|
||||
import sys, json, os
|
||||
r = json.loads(sys.stdin.readline())
|
||||
print('0x%X' % (int(r['addr'], 16) + r['nins']*4 - int(os.environ['VRAM'], 16)))")"
|
||||
|
||||
echo "Onboarding $ALIAS ($KIND, $DEC)"
|
||||
echo " sha1=$SHA1 size=$EOFHEX vram=$VRAM text_lo=$TLO code_end=$CODEEND text_hi=$TEXTHI"
|
||||
|
||||
# 1) splat config from the one shared template; a non-zero TEXT_LO swaps the code-at-0 line for
|
||||
# the hdr carve + shifted code start (the resident's rodata-head trick — no second template, R33)
|
||||
YAML="config/splat.${ALIAS}.yaml"
|
||||
sed -e "s|@ALIAS@|$ALIAS|g" -e "s|@DEC_PATH@|$DEC|g" -e "s|@SHA1@|$SHA1|g" \
|
||||
-e "s|@VRAM@|$VRAM|g" -e "s|@CODE_END@|$CODEEND|g" -e "s|@EOF@|$EOFHEX|g" \
|
||||
config/splat.us.overlay.template.yaml > "$YAML"
|
||||
if [ $((TLO)) -ne 0 ]; then
|
||||
TLOHEX="$(printf '0x%X' $((TLO)))"
|
||||
sed -i "s|^ - \[0x0, c, ${ALIAS}\].*| - [0x0, rodata, hdr] # module-id word (+ptr table) — §154-A\n - [${TLOHEX}, c, ${ALIAS}]|" "$YAML"
|
||||
fi
|
||||
if [ $((EOFSZ % 4)) -ne 0 ]; then
|
||||
FLOOR="$(printf '0x%X' $(( (EOFSZ/4)*4 )))"; NB=$((EOFSZ - (EOFSZ/4)*4))
|
||||
sed -i "s|^@TRAILING@\$| - [$FLOOR, bin, trailing] # final $NB bytes (EOF not 4-aligned; spimdisasm drops a partial word)|" "$YAML"
|
||||
else
|
||||
sed -i "/^@TRAILING@\$/d" "$YAML"
|
||||
fi
|
||||
echo " wrote $YAML"
|
||||
|
||||
# 2) build-identity hash
|
||||
echo "$SHA1 $ALIAS" > "config/check.${ALIAS}.sha"
|
||||
|
||||
# 3) binary-local symbols (don't clobber an existing curated file)
|
||||
if [ ! -f "config/symbols.${ALIAS}.txt" ]; then
|
||||
cat > "config/symbols.${ALIAS}.txt" <<EOF
|
||||
// config/symbols.${ALIAS}.txt — ${KIND} ${ALIAS} local symbols
|
||||
// R13/R15: this binary's REGION ONLY (vram ${VRAM}..${TEXTHI}); NEVER merged into config/symbols.us.txt.
|
||||
// count=0
|
||||
EOF
|
||||
fi
|
||||
|
||||
# 4) register in the class registry (generated, -include'd by the Makefile). Idempotent. Creates
|
||||
# config/modules.mk with its contract header on first module.
|
||||
if [ ! -f "$MK" ]; then
|
||||
cat > "$MK" <<EOF
|
||||
# config/modules.mk — GENERATED registry of module-class binaries (P30 S44; tools/new_binary.sh).
|
||||
# Same contract as overlays.mk: -include'd by the Makefile BEFORE 'BINARIES :=' (silent when
|
||||
# absent); the hand-maintained Makefile body is never edited to add a binary. Each md_* loads at
|
||||
# its OWN vram (the §S44 loader table: docs/memory-map.md). _TEXT_LO is a VRAM (code start), read
|
||||
# by 'make sig-modules' as --text-lo (§154-A: bootstrap past the module-id header).
|
||||
${MKVAR} :=
|
||||
EOF
|
||||
echo " + created $MK"
|
||||
fi
|
||||
if ! grep -q "^${ALIAS}_EXE " "$MK"; then
|
||||
grep -qE "^${MKVAR} :=.* ${ALIAS}( |\$)" "$MK" || sed -i "s|^${MKVAR} :=.*|& ${ALIAS}|" "$MK"
|
||||
cat >> "$MK" <<EOF
|
||||
|
||||
# --- ${ALIAS} (${KIND}, vram ${VRAM} — onboarded by new_binary.sh) ---
|
||||
${ALIAS}_EXE := ${DEC}
|
||||
${ALIAS}_NAME := ${ALIAS}
|
||||
${ALIAS}_OUT_DIR := build/${ALIAS}
|
||||
${ALIAS}_OUT := \$(${ALIAS}_OUT_DIR)/\$(${ALIAS}_NAME)
|
||||
${ALIAS}_ELF := \$(${ALIAS}_OUT).elf
|
||||
${ALIAS}_MAPFILE := \$(${ALIAS}_OUT).map
|
||||
${ALIAS}_LD_SCRIPT := \$(${ALIAS}_OUT).ld
|
||||
${ALIAS}_SPLAT_YAML := config/splat.${ALIAS}.yaml
|
||||
${ALIAS}_CHECK_SHA := config/check.${ALIAS}.sha
|
||||
${ALIAS}_SYMBOLS := config/symbols.${ALIAS}.txt
|
||||
${ALIAS}_SIG := .run/sig.${ALIAS}.jsonl
|
||||
${ALIAS}_GHIDRA_PROG := ${ALIAS}
|
||||
${ALIAS}_VRAM_BASE := ${VRAM}
|
||||
${ALIAS}_TEXT_LO := ${TLO_VRAM}
|
||||
${ALIAS}_TEXT_HI := ${TEXTHI}
|
||||
${ALIAS}_ASM_DIR := asm/${ALIAS}
|
||||
${ALIAS}_SRC_DIR := src/${ALIAS}
|
||||
${ALIAS}_UNDEF_SYMS := build/${ALIAS}/undefined_syms_auto.txt
|
||||
${ALIAS}_UNDEF_FUNCS := build/${ALIAS}/undefined_funcs_auto.txt
|
||||
EOF
|
||||
echo " + $MK"
|
||||
fi
|
||||
|
||||
# 5) register in the report/diff tools' BINARIES dicts (sentinel-anchored, idempotent, ast-checked —
|
||||
# the fully-generic step of new_overlay.sh, reused verbatim; the sentinel says ov_* but anchors
|
||||
# ALL generated entries).
|
||||
$PYV - "$ALIAS" <<'PY'
|
||||
import sys, ast
|
||||
alias = sys.argv[1]
|
||||
SENT = "inserts ov_* entries above this line"
|
||||
entries = {
|
||||
"diff_settings.py": (f' "{alias}": dict(baseimg="expected/build/{alias}/{alias}.elf",\n'
|
||||
f' myimg="build/{alias}/{alias}.elf",\n'
|
||||
f' mapfile="build/{alias}/{alias}.map"),\n'),
|
||||
"tools/progress.py": (f' "{alias}": dict(build="build/{alias}/{alias}", check="config/check.{alias}.sha",\n'
|
||||
f' src="src/{alias}", asm="asm/{alias}/nonmatchings", out="docs/progress.{alias}.md"),\n'),
|
||||
"tools/dup_report.py": (f' "{alias}": dict(sig=".run/sig.{alias}.jsonl", md="docs/duplicates.{alias}.md"),\n'),
|
||||
}
|
||||
for path, entry in entries.items():
|
||||
txt = open(path).read()
|
||||
if f'"{alias}":' in txt:
|
||||
continue # idempotent
|
||||
out, done = [], False
|
||||
for ln in txt.splitlines(keepends=True):
|
||||
if SENT in ln and not done:
|
||||
out.append(entry); done = True
|
||||
out.append(ln)
|
||||
new = "".join(out)
|
||||
ast.parse(new, path) # syntax-check before writing
|
||||
open(path, "w").write(new)
|
||||
print(f" + {path}")
|
||||
PY
|
||||
|
||||
# 6) extract + build + byte-check
|
||||
echo "Building $ALIAS ..."
|
||||
make extract BINARY="$ALIAS" >/dev/null
|
||||
make build BINARY="$ALIAS"
|
||||
echo "Done. $ALIAS onboarded; run 'make check BINARY=$ALIAS' anytime, or 'make check-all' for the fleet."
|
||||
+6
-115
@@ -21,119 +21,10 @@
|
||||
# tools/new_overlay.sh SC01 005 # -> ov_SC01_005 from 0.4.dec
|
||||
# tools/new_overlay.sh SC07 006 1.4 # -> ov_SC07_006 from 1.4.dec (index-1 overlay)
|
||||
# =============================================================================
|
||||
# P30 S44: this script is now a WRAPPER over tools/new_binary.sh (the generalized onboarder for
|
||||
# every flat-blob class — overlays AND md_* modules). All behavior above still holds; the
|
||||
# implementation lives in new_binary.sh. Kept as the overlay-shaped entry point (same CLI).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.." # repo root
|
||||
|
||||
[ $# -ge 2 ] && [ $# -le 3 ] || { echo "usage: tools/new_overlay.sh <SCxx> <FILE_nnn> [ENTRY] e.g. tools/new_overlay.sh SC01 005 | tools/new_overlay.sh SC07 006 1.4"; exit 2; }
|
||||
SC="$1"; FILE="$2"; ENTRY="${3:-0.4}"
|
||||
ALIAS="ov_${SC}_${FILE}"
|
||||
DEC="extracted/retail/${SC}.CD.dir/FILE_${FILE}.dir/${ENTRY}.dec"
|
||||
VRAM="0x80128158"
|
||||
PYV=".venv/bin/python"
|
||||
|
||||
[ -f "$DEC" ] || { echo "ERROR: payload not found: $DEC (run extraction first)"; exit 1; }
|
||||
|
||||
SHA1="$(sha1sum "$DEC" | cut -d' ' -f1)"
|
||||
EOFSZ="$(stat -c%s "$DEC")"
|
||||
EOFHEX="$(printf '0x%X' "$EOFSZ")"
|
||||
TEXTHI="$(printf '0x%X' $((0x80128158 + EOFSZ)))"
|
||||
|
||||
# code_end = end of the last sig_image function as a file offset (vram - base). Sign if absent.
|
||||
SIG=".run/sig.${ALIAS}.jsonl"
|
||||
[ -f "$SIG" ] || $PYV tools/sig_image.py --image "$DEC" --vram-base "$VRAM" --bootstrap --name "$ALIAS" >/dev/null
|
||||
CODEEND="$(tail -1 "$SIG" | $PYV -c "import sys,json; r=json.loads(sys.stdin.readline()); a=int(r['addr'],16); print('0x%X' % (a + r['nins']*4 - 0x80128158))")"
|
||||
|
||||
echo "Onboarding $ALIAS ($DEC)"
|
||||
echo " sha1=$SHA1 size=$EOFHEX code_end=$CODEEND text_hi=$TEXTHI"
|
||||
|
||||
# 1) splat config from the template (sed the @TOKENS@), then resolve @TRAILING@ (the non-4-aligned carve)
|
||||
YAML="config/splat.${ALIAS}.yaml"
|
||||
sed -e "s|@ALIAS@|$ALIAS|g" -e "s|@DEC_PATH@|$DEC|g" -e "s|@SHA1@|$SHA1|g" \
|
||||
-e "s|@VRAM@|$VRAM|g" -e "s|@CODE_END@|$CODEEND|g" -e "s|@EOF@|$EOFHEX|g" \
|
||||
config/splat.us.overlay.template.yaml > "$YAML"
|
||||
if [ $((EOFSZ % 4)) -ne 0 ]; then
|
||||
FLOOR="$(printf '0x%X' $(( (EOFSZ/4)*4 )))"; NB=$((EOFSZ - (EOFSZ/4)*4))
|
||||
sed -i "s|^@TRAILING@\$| - [$FLOOR, bin, trailing] # final $NB bytes (EOF not 4-aligned; spimdisasm drops a partial word)|" "$YAML"
|
||||
else
|
||||
sed -i "/^@TRAILING@\$/d" "$YAML"
|
||||
fi
|
||||
echo " wrote $YAML"
|
||||
|
||||
# 2) build-identity hash
|
||||
echo "$SHA1 $ALIAS" > "config/check.${ALIAS}.sha"
|
||||
|
||||
# 3) overlay-local symbols (don't clobber an existing curated file)
|
||||
if [ ! -f "config/symbols.${ALIAS}.txt" ]; then
|
||||
cat > "config/symbols.${ALIAS}.txt" <<EOF
|
||||
// config/symbols.${ALIAS}.txt — overlay ${SC}/FILE_${FILE} local symbols (Phase 13+)
|
||||
// R13/R15: overlay-REGION ONLY (vram ${VRAM}..${TEXTHI}); NEVER merged into config/symbols.us.txt.
|
||||
// count=0
|
||||
EOF
|
||||
fi
|
||||
|
||||
# 4) register in config/overlays.mk (generated registry, -include'd by the Makefile). Idempotent.
|
||||
MK="config/overlays.mk"
|
||||
if ! grep -q "^${ALIAS}_EXE " "$MK"; then
|
||||
grep -qE "^OVERLAY_BINARIES :=.* ${ALIAS}( |\$)" "$MK" || sed -i "s|^OVERLAY_BINARIES :=.*|& ${ALIAS}|" "$MK"
|
||||
cat >> "$MK" <<EOF
|
||||
|
||||
# --- ${ALIAS} (location overlay ${SC}/FILE_${FILE}, vram ${VRAM} — onboarded by new_overlay.sh) ---
|
||||
${ALIAS}_EXE := ${DEC}
|
||||
${ALIAS}_NAME := ${ALIAS}
|
||||
${ALIAS}_OUT_DIR := build/${ALIAS}
|
||||
${ALIAS}_OUT := \$(${ALIAS}_OUT_DIR)/\$(${ALIAS}_NAME)
|
||||
${ALIAS}_ELF := \$(${ALIAS}_OUT).elf
|
||||
${ALIAS}_MAPFILE := \$(${ALIAS}_OUT).map
|
||||
${ALIAS}_LD_SCRIPT := \$(${ALIAS}_OUT).ld
|
||||
${ALIAS}_SPLAT_YAML := config/splat.${ALIAS}.yaml
|
||||
${ALIAS}_CHECK_SHA := config/check.${ALIAS}.sha
|
||||
${ALIAS}_SYMBOLS := config/symbols.${ALIAS}.txt
|
||||
${ALIAS}_SIG := .run/sig.${ALIAS}.jsonl
|
||||
${ALIAS}_GHIDRA_PROG := ${ALIAS}
|
||||
${ALIAS}_VRAM_BASE := ${VRAM}
|
||||
${ALIAS}_TEXT_LO := ${VRAM}
|
||||
${ALIAS}_TEXT_HI := ${TEXTHI}
|
||||
${ALIAS}_ASM_DIR := asm/${ALIAS}
|
||||
${ALIAS}_SRC_DIR := src/${ALIAS}
|
||||
${ALIAS}_UNDEF_SYMS := build/${ALIAS}/undefined_syms_auto.txt
|
||||
${ALIAS}_UNDEF_FUNCS := build/${ALIAS}/undefined_funcs_auto.txt
|
||||
EOF
|
||||
echo " + config/overlays.mk"
|
||||
fi
|
||||
|
||||
# 5) register in the report/diff tools' BINARIES dicts (sentinel-anchored, idempotent, syntax-checked).
|
||||
# difficulty.py was DROPPED from this list (Phase-27 T6): it now DERIVES its per-binary paths from the
|
||||
# alias (cfg_for), so a new overlay needs no hand-registration there — the remaining 3 tools still carry
|
||||
# hand-lists (migrating them is future work; each is byte-gated per the audit's one-at-a-time cadence).
|
||||
$PYV - "$ALIAS" <<'PY'
|
||||
import sys, ast
|
||||
alias = sys.argv[1]
|
||||
SENT = "inserts ov_* entries above this line"
|
||||
entries = {
|
||||
"diff_settings.py": (f' "{alias}": dict(baseimg="expected/build/{alias}/{alias}.elf",\n'
|
||||
f' myimg="build/{alias}/{alias}.elf",\n'
|
||||
f' mapfile="build/{alias}/{alias}.map"),\n'),
|
||||
"tools/progress.py": (f' "{alias}": dict(build="build/{alias}/{alias}", check="config/check.{alias}.sha",\n'
|
||||
f' src="src/{alias}", asm="asm/{alias}/nonmatchings", out="docs/progress.{alias}.md"),\n'),
|
||||
"tools/dup_report.py": (f' "{alias}": dict(sig=".run/sig.{alias}.jsonl", md="docs/duplicates.{alias}.md"),\n'),
|
||||
}
|
||||
for path, entry in entries.items():
|
||||
txt = open(path).read()
|
||||
if f'"{alias}":' in txt:
|
||||
continue # idempotent
|
||||
out, done = [], False
|
||||
for ln in txt.splitlines(keepends=True):
|
||||
if SENT in ln and not done:
|
||||
out.append(entry); done = True
|
||||
out.append(ln)
|
||||
new = "".join(out)
|
||||
ast.parse(new, path) # syntax-check before writing
|
||||
open(path, "w").write(new)
|
||||
print(f" + {path}")
|
||||
PY
|
||||
|
||||
# 6) extract + build + byte-check
|
||||
echo "Building $ALIAS ..."
|
||||
make extract BINARY="$ALIAS" >/dev/null
|
||||
make build BINARY="$ALIAS"
|
||||
echo "Done. $ALIAS onboarded; run 'make check BINARY=$ALIAS' anytime, or 'make check-all' for the fleet."
|
||||
cd "$(dirname "$0")/.."
|
||||
[ $# -ge 2 ] && [ $# -le 3 ] || { echo "usage: tools/new_overlay.sh <SCxx> <FILE_nnn> [ENTRY]"; exit 2; }
|
||||
exec tools/new_binary.sh "ov_${1}_${2}" "extracted/retail/${1}.CD.dir/FILE_${2}.dir/${3:-0.4}.dec" 0x80128158
|
||||
|
||||
Reference in New Issue
Block a user