disc conversion and optimizations to generate+build flow

This commit is contained in:
Alex Vanderveen
2026-07-31 17:22:45 -04:00
parent e7f68a2ba7
commit 17c60e470d
3 changed files with 1495 additions and 0 deletions
+844
View File
@@ -0,0 +1,844 @@
#!/usr/bin/env python3
"""Headless disc → generate / rebuild / optional PGO for psxrecomp games.
Commands:
verify-disc Hash-check a dump against game.toml [prepare_disc]
generate Prepare disc (if needed) + run psxrecomp-game → generated/
rebuild cmake --build; if [pgo] enabled, instrument → train → use
pgo-train Standalone PGO train (same as rebuild's PGO phase)
Exit codes: 0 ok · 1 runtime · 2 usage · 3 disc verify fail
"""
from __future__ import annotations
import argparse
import hashlib
import os
import re
import shutil
import subprocess
import sys
import time
from pathlib import Path
from typing import Any, Optional
ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT / "tools"))
from sdk_progress import ProgressReporter # noqa: E402
EXIT_OK = 0
EXIT_ERROR = 1
EXIT_USAGE = 2
EXIT_VERIFY = 3
def _parse_array_items(inner: str) -> list[Any]:
items: list[Any] = []
if not inner.strip():
return items
for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner):
part = part.strip().rstrip(",").strip()
if not part:
continue
if part.startswith('"') and part.endswith('"'):
items.append(part[1:-1])
else:
try:
items.append(int(part, 0))
except ValueError:
items.append(part)
return items
def parse_toml_simple(text: str) -> dict[str, dict[str, Any]]:
sections: dict[str, dict[str, Any]] = {"": {}}
cur = ""
array_key: Optional[str] = None
array_buf: list[str] = []
for raw in text.splitlines():
line = raw.split("#", 1)[0].strip()
if array_key is not None:
array_buf.append(line)
joined = " ".join(array_buf)
if "]" in joined:
inner = joined.split("]", 1)[0]
if inner.startswith("["):
inner = inner[1:]
sections[cur][array_key] = _parse_array_items(inner)
array_key = None
array_buf = []
continue
if not line:
continue
if line.startswith("[") and line.endswith("]"):
cur = line[1:-1].strip()
sections.setdefault(cur, {})
continue
if "=" not in line:
continue
key, val = line.split("=", 1)
key, val = key.strip(), val.strip()
if val.startswith("["):
if val.endswith("]") and val.count("[") == val.count("]"):
sections[cur][key] = _parse_array_items(val[1:-1])
else:
array_key = key
array_buf = [val[1:]]
continue
if val.startswith('"') and val.endswith('"'):
sections[cur][key] = val[1:-1]
elif val.lower() in ("true", "false"):
sections[cur][key] = val.lower() == "true"
else:
try:
sections[cur][key] = int(val, 0)
except ValueError:
sections[cur][key] = val
return sections
def file_hashes(path: Path) -> tuple[str, str, int]:
h_md5, h_sha1 = hashlib.md5(), hashlib.sha1()
size = 0
with open(path, "rb") as f:
while True:
chunk = f.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
h_md5.update(chunk)
h_sha1.update(chunk)
return h_md5.hexdigest(), h_sha1.hexdigest(), size
def resolve_cue_bin(cue_path: Path) -> Path:
text = cue_path.read_text(encoding="utf-8", errors="replace")
m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I)
if not m:
m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I)
if not m:
raise ValueError(f"no BINARY FILE in cue: {cue_path}")
cand = Path(m.group(1))
if not cand.is_absolute():
cand = cue_path.parent / cand
if not cand.is_file():
raise ValueError(f"cue references missing bin: {cand}")
return cand
def find_psxrecomp_game(project_root: Path) -> Path:
env = os.environ.get("PSXRECOMP_GAME", "").strip()
if env:
p = Path(env).expanduser()
if p.is_file():
return p.resolve()
candidates = [
project_root / "psxrecomp" / "recompiler" / "build" / "psxrecomp-game",
project_root / "psxrecomp" / "recompiler" / "build" / "psxrecomp-game.exe",
ROOT / "recompiler" / "build" / "psxrecomp-game",
ROOT / "recompiler" / "build" / "psxrecomp-game.exe",
]
for c in candidates:
if c.is_file():
return c.resolve()
which = shutil.which("psxrecomp-game")
if which:
return Path(which).resolve()
raise FileNotFoundError(
"psxrecomp-game not found. Build it:\n"
" cmake -S psxrecomp/recompiler -B psxrecomp/recompiler/build -G Ninja\n"
" cmake --build psxrecomp/recompiler/build --target psxrecomp-game\n"
"Or set PSXRECOMP_GAME=/path/to/psxrecomp-game"
)
def load_sections(config: Path) -> dict[str, dict[str, Any]]:
return parse_toml_simple(config.read_text(encoding="utf-8"))
def verify_disc_path(
disc: Path,
prep: dict[str, Any],
*,
skip_hash: bool,
progress: ProgressReporter,
) -> dict[str, Any]:
path = disc.resolve()
if path.suffix.lower() == ".cue":
path = resolve_cue_bin(path)
md5, sha1, size = file_hashes(path)
identity = {
"path": str(path),
"md5": md5,
"sha1": sha1,
"size": size,
"verified": False,
}
progress.event("disc", **identity)
sizes = [int(s) for s in (prep.get("known_sizes") or [])]
md5s = [str(x).lower() for x in (prep.get("known_md5") or [])]
sha1s = [str(x).lower() for x in (prep.get("known_sha1") or [])]
if not md5s and not sha1s and not sizes:
identity["verified"] = True
return identity
if skip_hash:
return identity
ok = (md5 in md5s) or (sha1 in sha1s)
if not ok and sizes and size in sizes and not md5s and not sha1s:
ok = True
if not ok:
raise DiscVerifyError(
f"disc digests not in prepare_disc.known_* "
f"(size={size} md5={md5} sha1={sha1})"
)
identity["verified"] = True
return identity
class DiscVerifyError(Exception):
pass
def cmd_verify_disc(args: argparse.Namespace, progress: ProgressReporter) -> int:
config = Path(args.config).expanduser().resolve()
if not config.is_file():
progress.error(f"config not found: {config}", code=EXIT_USAGE)
return EXIT_USAGE
project_root = (
Path(args.project_root).expanduser().resolve()
if args.project_root
else config.parent
)
disc = Path(args.disc).expanduser()
if not disc.is_absolute():
disc = (project_root / disc).resolve()
else:
disc = disc.resolve()
if not disc.is_file():
progress.error(f"disc not found: {disc}", code=EXIT_USAGE)
return EXIT_USAGE
secs = load_sections(config)
prep = secs.get("prepare_disc") or {}
progress.phase("verify", pct=0.1, message=f"Verifying {disc.name}")
try:
identity = verify_disc_path(
disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress
)
except DiscVerifyError as exc:
progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True)
return EXIT_VERIFY
progress.phase("done", pct=1.0, message="Disc OK")
progress.result(ok=True, **identity)
return EXIT_OK
def run_prepare_disc(
project_root: Path,
config: Path,
source: Path,
progress: ProgressReporter,
) -> Path:
script = ROOT / "tools" / "prepare_disc.py"
if not script.is_file():
raise RuntimeError(f"missing {script}")
progress.phase("prepare_disc", pct=0.15, message="Normalizing disc image…")
cmd = [
sys.executable,
str(script),
"--config",
str(config),
"--project-root",
str(project_root),
str(source),
]
proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True)
out = (proc.stdout or "") + (proc.stderr or "")
for line in out.splitlines():
if line.strip():
progress.log(line)
if proc.returncode != 0:
raise RuntimeError(f"prepare_disc failed (exit {proc.returncode})")
marker = "RESULT_CUE="
cue = None
for line in out.splitlines():
if line.startswith(marker):
cue = Path(line[len(marker) :].strip())
if not cue or not cue.is_file():
raise RuntimeError("prepare_disc did not print RESULT_CUE=")
return cue.resolve()
def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int:
config = Path(args.config).expanduser().resolve()
if not config.is_file():
progress.error(f"config not found: {config}", code=EXIT_USAGE)
return EXIT_USAGE
project_root = (
Path(args.project_root).expanduser().resolve()
if args.project_root
else config.parent
)
secs = load_sections(config)
game = secs.get("game") or {}
prep = secs.get("prepare_disc") or {}
recomp = secs.get("recompiler") or {}
disc_arg = args.disc
if not disc_arg:
disc_arg = game.get("disc") or ""
if not disc_arg:
progress.error("no --disc and game.disc empty", code=EXIT_USAGE)
return EXIT_USAGE
disc = Path(str(disc_arg)).expanduser()
if not disc.is_absolute():
disc = (project_root / disc).resolve()
else:
disc = disc.resolve()
progress.phase("verify", pct=0.05, message=f"Checking disc {disc.name}")
try:
if disc.is_file():
verify_disc_path(
disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress
)
except DiscVerifyError as exc:
progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True)
return EXIT_VERIFY
boot = str(prep.get("boot_exe") or Path(str(game.get("exe") or "")).name)
out_rel = str(prep.get("out_dir") or "prepared_disc")
boot_path = project_root / out_rel / boot
working_disc = disc
# Normalize library dumps when boot EXE missing or source looks like ISO/raw.
need_prep = (not boot_path.is_file()) or disc.suffix.lower() in (
".iso",
".ISO",
)
if need_prep or args.force_prepare:
try:
working_disc = run_prepare_disc(project_root, config, disc, progress)
except Exception as exc: # noqa: BLE001
progress.error(str(exc), code=EXIT_ERROR)
return EXIT_ERROR
if not boot_path.is_file():
progress.error(f"boot EXE missing after prepare: {boot_path}", code=EXIT_ERROR)
return EXIT_ERROR
try:
game_tool = find_psxrecomp_game(project_root)
except FileNotFoundError as exc:
progress.error(str(exc), code=EXIT_ERROR)
return EXIT_ERROR
out_dir = project_root / str(recomp.get("out_dir") or "generated")
progress.phase(
"emit",
pct=0.35,
message=f"Running psxrecomp-game → {out_dir}",
)
cmd = [str(game_tool), "--config", str(config)]
proc = subprocess.run(
cmd,
cwd=str(project_root),
capture_output=True,
text=True,
)
for stream in (proc.stdout, proc.stderr):
if not stream:
continue
for line in stream.splitlines():
if line.strip():
progress.log(line)
if proc.returncode != 0:
progress.error(
f"psxrecomp-game failed (exit {proc.returncode})", code=EXIT_ERROR
)
return EXIT_ERROR
marker_name = args.gen_marker or f"{boot}_dispatch.c"
marker = out_dir / marker_name
if not marker.is_file():
# Accept any *_dispatch.c
hits = list(out_dir.glob("*_dispatch.c"))
if not hits:
progress.error(f"generate produced no dispatch under {out_dir}", code=EXIT_ERROR)
return EXIT_ERROR
marker = hits[0]
progress.phase("done", pct=1.0, message="Generate complete")
progress.result(
ok=True,
out_dir=str(out_dir),
marker=str(marker),
disc=str(working_disc),
boot_exe=str(boot_path),
)
return EXIT_OK
def _cmake_configure(
project_root: Path,
build_dir: Path,
*,
pgo: str,
extra: list[str],
progress: ProgressReporter,
) -> None:
build_dir.mkdir(parents=True, exist_ok=True)
cmd = [
"cmake",
"-S",
str(project_root),
"-B",
str(build_dir),
]
# Only force generator on fresh trees (avoid clash with existing cache).
if not (build_dir / "CMakeCache.txt").is_file():
cmd.extend(["-G", "Ninja"])
cmd.append(f"-DPSX_PGO={pgo}")
cmd.extend(extra)
progress.log(" ".join(cmd))
proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True)
for stream in (proc.stdout, proc.stderr):
if stream:
for line in stream.splitlines():
if line.strip():
progress.log(line)
if proc.returncode != 0:
raise RuntimeError(f"cmake configure failed (exit {proc.returncode})")
def _cmake_build(
build_dir: Path,
target: str,
progress: ProgressReporter,
) -> None:
jobs = os.environ.get("CMAKE_BUILD_PARALLEL_LEVEL") or str(
os.cpu_count() or 4
)
cmd = [
"cmake",
"--build",
str(build_dir),
"--parallel",
jobs,
"--target",
target,
]
progress.log(" ".join(cmd))
proc = subprocess.run(cmd, capture_output=True, text=True)
for stream in (proc.stdout, proc.stderr):
if stream:
for line in stream.splitlines():
if line.strip():
progress.log(line)
if proc.returncode != 0:
raise RuntimeError(f"cmake --build failed (exit {proc.returncode})")
def _soft_stop(pid: int, timeout: int = 30) -> None:
try:
os.kill(pid, 15) # SIGTERM
except ProcessLookupError:
return
for _ in range(timeout):
try:
os.kill(pid, 0)
except ProcessLookupError:
return
time.sleep(1)
try:
os.kill(pid, 9)
except ProcessLookupError:
pass
def _pgo_train_warning(*, hide_video: bool) -> str:
if hide_video:
return (
"WARNING: PGO training is running (no video window). Do not cancel "
"or kill this process until training finishes — interrupting may "
"corrupt profiles and force a restart of this step."
)
return (
"WARNING: PGO training is running. Do not close the game window or "
"interact with the application until training finishes — closing or "
"clicking may corrupt profiles and force a restart of this step."
)
def run_pgo_train(
project_root: Path,
build_dir: Path,
*,
exe_basename: str,
disc: Path,
config: Path,
train_secs: int,
train_runs: int,
mute_host_audio: bool,
hide_video: bool,
progress: ProgressReporter,
) -> None:
exe = build_dir / exe_basename
if not exe.is_file():
exe = build_dir / f"{exe_basename}.exe"
if not exe.is_file():
raise RuntimeError(f"train binary missing: {exe_basename} under {build_dir}")
pgo_dir = build_dir / "pgo"
if pgo_dir.exists():
shutil.rmtree(pgo_dir)
pgo_dir.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env.setdefault("PSX_BIOS_HLE", "0")
env["LLVM_PROFILE_FILE"] = str(pgo_dir / "bpe-%p.profraw")
# Discard SDL output; SPU/CD still advance (profiles stay valid).
if mute_host_audio:
env["PSX_HOST_MUTE"] = "1"
else:
env.pop("PSX_HOST_MUTE", None)
# No on-screen FMV (photosensitive / abrasive content). Guest MDEC/GPU
# still run; host present is skipped. Matches prior CI train shape.
if hide_video:
env["PSX_HEADLESS"] = "1"
env.setdefault("SDL_VIDEODRIVER", "dummy")
else:
env.pop("PSX_HEADLESS", None)
warning = _pgo_train_warning(hide_video=hide_video)
progress.phase("pgo_train", pct=0.35, message=warning)
# Always surface on the console even when JSONL owns stdout.
print(f"\n*** {warning} ***\n", file=sys.stderr, flush=True)
if mute_host_audio:
progress.log("Host audio muted for train (PSX_HOST_MUTE=1; SPU still runs).")
else:
progress.log("Host audio unmuted for train (speakers will play).")
if hide_video:
progress.log(
"Video hidden for train (--headless / PSX_HEADLESS=1; "
"guest FMV decode still runs, nothing shown on screen)."
)
else:
progress.log("Video window visible for train (host present paths included).")
for run in range(1, train_runs + 1):
progress.phase(
"pgo_train",
pct=0.4 + 0.4 * (run - 1) / max(train_runs, 1),
message=(
f"WARNING: PGO train run {run}/{train_runs} ({train_secs}s) — "
+ (
"Do not cancel this process."
if hide_video
else "Do not close or interact with the application."
)
),
)
cmd = [
str(exe),
"--no-launcher",
"--game",
str(config),
"--disc",
str(disc),
]
if hide_video:
cmd.append("--headless")
proc = subprocess.Popen(
cmd,
cwd=str(project_root),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(train_secs)
if proc.poll() is None:
_soft_stop(proc.pid)
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
n_gcda = len(list(build_dir.rglob("*.gcda")))
n_raw = len(list(pgo_dir.glob("*.profraw")))
progress.log(f"PGO profiles: {n_gcda} .gcda, {n_raw} .profraw")
if n_raw >= 1:
merge = shutil.which("llvm-profdata")
if not merge:
# macOS xcrun
try:
r = subprocess.run(
["xcrun", "--find", "llvm-profdata"],
capture_output=True,
text=True,
check=False,
)
if r.returncode == 0 and r.stdout.strip():
merge = "xcrun"
except OSError:
pass
raws = list(pgo_dir.glob("*.profraw"))
out = pgo_dir / "default.profdata"
if merge == "xcrun":
cmd = ["xcrun", "llvm-profdata", "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)]
elif merge:
cmd = [merge, "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)]
else:
raise RuntimeError("llvm-profdata not found (Clang PGO merge)")
subprocess.run(cmd, check=True)
if n_gcda < 1 and not (pgo_dir / "default.profdata").is_file():
raise RuntimeError("no PGO profiles written — train did not flush")
def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int:
config = Path(args.config).expanduser().resolve()
if not config.is_file():
progress.error(f"config not found: {config}", code=EXIT_USAGE)
return EXIT_USAGE
project_root = (
Path(args.project_root).expanduser().resolve()
if args.project_root
else config.parent
)
build_dir = Path(args.build_dir).expanduser()
if not build_dir.is_absolute():
build_dir = (project_root / build_dir).resolve()
else:
build_dir = build_dir.resolve()
target = args.target or "psx-runtime"
exe_basename = args.exe_basename or "psx-runtime"
secs = load_sections(config)
pgo = secs.get("pgo") or {}
# Opt-in only: requires game.toml [pgo] enabled = true (or --force-pgo).
pgo_enabled = bool(pgo.get("enabled", False)) and not args.no_pgo
if args.force_pgo:
pgo_enabled = True
game = secs.get("game") or {}
prep = secs.get("prepare_disc") or {}
if args.disc:
disc = Path(args.disc).expanduser()
if not disc.is_absolute():
disc = (project_root / disc).resolve()
else:
disc = disc.resolve()
else:
cue_name = prep.get("cue_name")
if cue_name:
disc = (
project_root / str(prep.get("out_dir") or "bpe") / str(cue_name)
).resolve()
elif game.get("disc"):
disc = (project_root / str(game["disc"])).resolve()
if disc.suffix.lower() == ".bin":
cue = disc.with_suffix(".cue")
if cue.is_file():
disc = cue
else:
disc = None
cmake_extra = []
if args.cmake_extra:
cmake_extra.extend(args.cmake_extra)
# mute_host_audio / hide_video: default ON; game.toml / CLI can disable.
mute_host = True
if "mute_host_audio" in pgo:
mute_host = bool(pgo.get("mute_host_audio"))
if getattr(args, "pgo_audio", False):
mute_host = False
if getattr(args, "pgo_mute", False):
mute_host = True
hide_video = True
if "hide_video" in pgo:
hide_video = bool(pgo.get("hide_video"))
if getattr(args, "pgo_video", False):
hide_video = False
if getattr(args, "pgo_hide_video", False):
hide_video = True
try:
if not pgo_enabled:
progress.phase("build", pct=0.2, message="cmake Release build…")
_cmake_configure(
project_root, build_dir, pgo="", extra=cmake_extra, progress=progress
)
_cmake_build(build_dir, target, progress)
else:
# Framework defaults (60×2); game.toml / CLI may lengthen for hard titles.
train_secs = int(getattr(args, "train_secs", 0) or 0)
if train_secs <= 0:
train_secs = int(pgo.get("train_secs") or 60)
train_runs = int(getattr(args, "train_runs", 0) or 0)
if train_runs <= 0:
train_runs = int(pgo.get("train_runs") or 2)
progress.phase(
"pgo_instrument",
pct=0.1,
message="PGO instrumented build…",
)
_cmake_configure(
project_root,
build_dir,
pgo="generate",
extra=cmake_extra,
progress=progress,
)
_cmake_build(build_dir, target, progress)
if not disc or not Path(disc).is_file():
raise RuntimeError(
f"PGO train needs a playable disc (got {disc!s}). "
"Pass --disc or set game.disc / prepare_disc.cue_name."
)
run_pgo_train(
project_root,
build_dir,
exe_basename=exe_basename,
disc=Path(disc),
config=config,
train_secs=train_secs,
train_runs=train_runs,
mute_host_audio=mute_host,
hide_video=hide_video,
progress=progress,
)
progress.phase("pgo_use", pct=0.85, message="PGO optimized rebuild…")
_cmake_configure(
project_root,
build_dir,
pgo="use",
extra=cmake_extra,
progress=progress,
)
_cmake_build(build_dir, target, progress)
except Exception as exc: # noqa: BLE001
progress.error(str(exc), code=EXIT_ERROR)
return EXIT_ERROR
exe = build_dir / exe_basename
if not exe.is_file():
exe = build_dir / f"{exe_basename}.exe"
progress.phase("done", pct=1.0, message="Rebuild complete")
progress.result(ok=True, exe=str(exe), pgo=pgo_enabled)
return EXIT_OK
def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int:
args.force_pgo = True
args.no_pgo = False
return cmd_rebuild(args, progress)
def build_parser() -> argparse.ArgumentParser:
ap = argparse.ArgumentParser(prog="psxrecomp_cli", description=__doc__)
sub = ap.add_subparsers(dest="command", required=True)
def add_common(p: argparse.ArgumentParser) -> None:
p.add_argument("--config", default="game.toml", help="game.toml path")
p.add_argument("--project-root", default="", help="game project root")
p.add_argument("--json-progress", action="store_true")
v = sub.add_parser("verify-disc", help="verify disc digests")
add_common(v)
v.add_argument("--disc", required=True)
v.add_argument("--skip-hash-check", action="store_true")
v.set_defaults(handler=cmd_verify_disc)
g = sub.add_parser("generate", help="prepare disc + generate C")
add_common(g)
g.add_argument("--disc", default="", help="source dump or working cue/bin")
g.add_argument("--skip-hash-check", action="store_true")
g.add_argument("--force-prepare", action="store_true")
g.add_argument("--gen-marker", default="", help="expected marker under out_dir")
g.set_defaults(handler=cmd_generate)
r = sub.add_parser("rebuild", help="cmake build (+ optional local PGO)")
add_common(r)
r.add_argument("--build-dir", required=True)
r.add_argument("--target", default="psx-runtime")
r.add_argument("--exe-basename", default="")
r.add_argument("--disc", default="", help="disc for PGO train")
r.add_argument(
"--no-pgo",
action="store_true",
help="skip PGO even if game.toml [pgo] enabled=true",
)
r.add_argument(
"--force-pgo",
action="store_true",
help="run PGO even without [pgo] enabled=true",
)
r.add_argument(
"--train-secs",
type=int,
default=0,
help="override [pgo] train_secs (default framework 60)",
)
r.add_argument(
"--train-runs",
type=int,
default=0,
help="override [pgo] train_runs (default framework 2)",
)
r.add_argument(
"--pgo-mute",
action="store_true",
help="force mute host speakers during train (default)",
)
r.add_argument(
"--pgo-audio",
action="store_true",
help="allow host speakers during train (overrides mute default)",
)
r.add_argument(
"--pgo-hide-video",
action="store_true",
help="force headless train with no on-screen video (default)",
)
r.add_argument(
"--pgo-video",
action="store_true",
help="show a game window during train (includes host present paths)",
)
r.add_argument("--cmake-extra", action="append", default=[])
r.set_defaults(handler=cmd_rebuild)
p = sub.add_parser("pgo-train", help="force PGO rebuild+train+use")
add_common(p)
p.add_argument("--build-dir", required=True)
p.add_argument("--target", default="psx-runtime")
p.add_argument("--exe-basename", default="")
p.add_argument("--disc", default="")
p.add_argument("--train-secs", type=int, default=0)
p.add_argument("--train-runs", type=int, default=0)
p.add_argument("--pgo-mute", action="store_true")
p.add_argument("--pgo-audio", action="store_true")
p.add_argument("--pgo-hide-video", action="store_true")
p.add_argument("--pgo-video", action="store_true")
p.add_argument("--cmake-extra", action="append", default=[])
p.add_argument("--no-pgo", action="store_false", default=False)
p.add_argument("--force-pgo", action="store_true", default=True)
p.set_defaults(handler=cmd_pgo_train)
return ap
def main() -> int:
args = build_parser().parse_args()
progress = ProgressReporter(json_progress=bool(getattr(args, "json_progress", False)))
try:
return int(args.handler(args, progress))
except BrokenPipeError:
return EXIT_ERROR
except KeyboardInterrupt:
progress.error("interrupted", code=EXIT_ERROR)
return EXIT_ERROR
if __name__ == "__main__":
raise SystemExit(main())
+575
View File
@@ -0,0 +1,575 @@
#!/usr/bin/env python3
"""Normalize a PS1 disc dump to MODE2/2352 .bin/.cue for psxrecomp games.
Framework tool — game identity comes from ``game.toml`` (``[game]`` + optional
``[prepare_disc]``) or CLI flags.
Accepted inputs:
* Redump / raw MODE2/2352 ``.bin`` (+ ``.cue``)
* Cooked 2048-byte/sector ``.iso`` (RomM-style libraries)
* 2448-byte/sector raw dumps (MotK-style): trim to 2352
Writes working ``.bin`` + ``.cue``, extracts ``SYSTEM.CNF`` + boot EXE, and
prints ``RESULT_CUE=<abs path>`` for the first-run wizard / RetComM.
ISO→2352 sets Mode2 Form1 sync/header/subheader/EDC (ECC zeroed — fine for
software readers). Rebuilt images are not bit-identical to Redump.
"""
from __future__ import annotations
import argparse
import hashlib
import re
import struct
import sys
from dataclasses import dataclass, field
from pathlib import Path
DST_SEC = 2352
SRC_2448 = 2448
USER = 2048
USER_OFF = 24
SYNC = bytes([0x00] + [0xFF] * 10 + [0x00])
def _build_edc_table() -> list[int]:
table = []
for i in range(256):
x = i
for _ in range(8):
x = (x >> 1) ^ 0xD8018001 if (x & 1) else (x >> 1)
table.append(x & 0xFFFFFFFF)
return table
_EDC_TABLE = _build_edc_table()
def _adjust_edc(buf: bytearray, start: int, length: int) -> None:
x = 0
end = start + length
for i in range(start, end):
x = _EDC_TABLE[(x ^ buf[i]) & 0xFF] ^ (x >> 8)
struct.pack_into("<I", buf, end, x & 0xFFFFFFFF)
def encode_mode2_form1(user: bytes, lba: int, submode: int = 0x08) -> bytes:
if len(user) != USER:
raise ValueError("user data must be 2048 bytes")
sector = bytearray(DST_SEC)
sector[0:12] = SYNC
t = lba + 150
mm, ss, ff = t // (75 * 60), (t // 75) % 60, t % 75
sector[12] = ((mm // 10) << 4) | (mm % 10)
sector[13] = ((ss // 10) << 4) | (ss % 10)
sector[14] = ((ff // 10) << 4) | (ff % 10)
sector[15] = 0x02
sub = bytes([0x00, 0x00, submode & 0xFF, 0x00])
sector[16:20] = sub
sector[20:24] = sub
sector[24 : 24 + USER] = user
_adjust_edc(sector, 0x10, 0x800 + 8)
return bytes(sector)
@dataclass
class KnownImage:
size: int = 0
md5: str = ""
sha1: str = ""
@dataclass
class PrepareConfig:
project_root: Path
out_dir: Path
bin_name: str
cue_name: str
boot_exe: str
serial: str = ""
known: list[KnownImage] = field(default_factory=list)
skip_hash_check: bool = False
def file_hashes(path: Path) -> tuple[str, str, int]:
h_md5 = hashlib.md5()
h_sha1 = hashlib.sha1()
size = 0
with open(path, "rb") as f:
while True:
chunk = f.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
h_md5.update(chunk)
h_sha1.update(chunk)
return h_md5.hexdigest(), h_sha1.hexdigest(), size
def _parse_array_items(inner: str) -> list[object]:
items: list[object] = []
if not inner.strip():
return items
for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner):
part = part.strip().rstrip(",").strip()
if not part:
continue
if part.startswith('"') and part.endswith('"'):
items.append(part[1:-1])
else:
try:
items.append(int(part, 0))
except ValueError:
items.append(part)
return items
def _parse_toml_simple(text: str) -> dict[str, dict[str, str]]:
"""Minimal TOML reader for flat string/int/bool + string/int arrays.
Supports multi-line arrays (``key = [ … ]`` spanning lines).
"""
sections: dict[str, dict[str, object]] = {"": {}}
cur = ""
array_key: str | None = None
array_buf: list[str] = []
for raw in text.splitlines():
line = raw.split("#", 1)[0].strip()
if array_key is not None:
array_buf.append(line)
joined = " ".join(array_buf)
if "]" in joined:
inner = joined.split("]", 1)[0]
# Drop leading "[" already consumed when array opened
if inner.startswith("["):
inner = inner[1:]
sections[cur][array_key] = _parse_array_items(inner)
array_key = None
array_buf = []
continue
if not line:
continue
if line.startswith("[") and line.endswith("]"):
cur = line[1:-1].strip()
sections.setdefault(cur, {})
continue
if "=" not in line:
continue
key, val = line.split("=", 1)
key = key.strip()
val = val.strip()
if val.startswith("["):
if val.endswith("]") and val.count("[") == val.count("]"):
sections[cur][key] = _parse_array_items(val[1:-1])
else:
array_key = key
array_buf = [val[1:]] # after opening [
continue
if val.startswith('"') and val.endswith('"'):
sections[cur][key] = val[1:-1]
elif val.lower() in ("true", "false"):
sections[cur][key] = val.lower() == "true"
else:
try:
sections[cur][key] = int(val, 0)
except ValueError:
sections[cur][key] = val
return sections # type: ignore[return-value]
def load_config(project_root: Path, config_path: Path | None, out_dir_cli: str | None,
skip_hash: bool) -> PrepareConfig:
project_root = project_root.resolve()
game: dict = {}
prep: dict = {}
if config_path and config_path.is_file():
sections = _parse_toml_simple(config_path.read_text(encoding="utf-8"))
game = sections.get("game", {}) or {}
prep = sections.get("prepare_disc", {}) or {}
boot = str(prep.get("boot_exe") or "")
if not boot:
exe = str(game.get("exe") or "")
boot = Path(exe).name if exe else ""
if not boot:
boot = "SLUS_000.00"
out_rel = str(prep.get("out_dir") or "prepared_disc")
out_dir = Path(out_dir_cli).expanduser() if out_dir_cli else (project_root / out_rel)
disc = str(game.get("disc") or prep.get("bin_name") or "game.bin")
bin_name = str(prep.get("bin_name") or Path(disc).name)
cue_name = str(prep.get("cue_name") or (Path(bin_name).stem + ".cue"))
known: list[KnownImage] = []
sizes = prep.get("known_sizes") or []
md5s = prep.get("known_md5") or []
sha1s = prep.get("known_sha1") or []
if isinstance(sizes, list) or isinstance(md5s, list) or isinstance(sha1s, list):
n = max(
len(sizes) if isinstance(sizes, list) else 0,
len(md5s) if isinstance(md5s, list) else 0,
len(sha1s) if isinstance(sha1s, list) else 0,
)
for i in range(n):
known.append(
KnownImage(
size=int(sizes[i]) if isinstance(sizes, list) and i < len(sizes) else 0,
md5=str(md5s[i]).lower()
if isinstance(md5s, list) and i < len(md5s)
else "",
sha1=str(sha1s[i]).lower()
if isinstance(sha1s, list) and i < len(sha1s)
else "",
)
)
return PrepareConfig(
project_root=project_root,
out_dir=out_dir.resolve(),
bin_name=bin_name,
cue_name=cue_name,
boot_exe=boot,
serial=str(game.get("id") or ""),
known=known,
skip_hash_check=skip_hash,
)
def resolve_cue_bin(cue_path: Path) -> Path:
text = cue_path.read_text(encoding="utf-8", errors="replace")
m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I)
if not m:
m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I)
if not m:
raise SystemExit(f"no BINARY FILE in cue: {cue_path}")
cand = Path(m.group(1))
if not cand.is_absolute():
cand = cue_path.parent / cand
if not cand.is_file():
raise SystemExit(f"cue references missing bin: {cand}")
return cand
def detect_kind(path: Path, size: int) -> str:
suf = path.suffix.lower()
if suf == ".cue":
return "cue"
if suf == ".iso":
return "iso"
# MotK-style 2448 before generic 2352 (some sizes divisible by both).
if size % SRC_2448 == 0 and size > SRC_2448 and size % DST_SEC != 0:
return "raw2448"
if size % SRC_2448 == 0 and size > SRC_2448:
# Peek: if sync looks like CD at +0, treat as 2448 when user data path works.
with open(path, "rb") as f:
head = f.read(16)
if head[:12] == SYNC:
# Could be 2352 or 2448; check sector stride via second sync.
fpos_2352_ok = False
fpos_2448_ok = False
with open(path, "rb") as f:
f.seek(DST_SEC)
fpos_2352_ok = f.read(12) == SYNC
f.seek(SRC_2448)
fpos_2448_ok = f.read(12) == SYNC
if fpos_2448_ok and not fpos_2352_ok:
return "raw2448"
if size % DST_SEC == 0 and size > DST_SEC:
with open(path, "rb") as f:
sec0 = f.read(12)
f.seek(16 * DST_SEC)
sync16 = f.read(12)
if sec0 == SYNC or sync16 == SYNC:
return "bin2352"
if size % USER == 0 and size > USER:
return "iso"
if size % SRC_2448 == 0 and size > SRC_2448:
return "raw2448"
raise SystemExit(
f"unrecognized disc image {path} (size={size}). "
"Need MODE2/2352 .bin/.cue, cooked 2048 .iso, or 2448-byte/sector dump."
)
def read_user_bin(data: bytes, lba: int) -> bytes:
off = lba * DST_SEC
if off + DST_SEC > len(data):
raise KeyError(lba)
if data[off : off + 12] != SYNC:
raise SystemExit(f"bad sync at LBA {lba}")
return data[off + USER_OFF : off + USER_OFF + USER]
def read_user_iso(data: bytes, lba: int) -> bytes:
off = lba * USER
if off + USER > len(data):
raise KeyError(lba)
return data[off : off + USER]
def parse_root_entries(root: bytes) -> dict[str, tuple[int, int]]:
entries: dict[str, tuple[int, int]] = {}
i = 0
while i < len(root):
reclen = root[i]
if reclen == 0:
i = ((i // USER) + 1) * USER
if i >= len(root):
break
continue
if i + reclen > len(root):
break
extent = struct.unpack_from("<I", root, i + 2)[0]
size = struct.unpack_from("<I", root, i + 10)[0]
namelen = root[i + 32]
name = root[i + 33 : i + 33 + namelen]
if b";" in name:
name = name.split(b";")[0]
if name not in (b"\x00", b"\x01"):
entries[name.decode("ascii", "replace")] = (extent, size)
i += reclen
return entries
def extract_via(
read_user, data: bytes, boot_exe: str
) -> tuple[dict[str, tuple[int, int]], dict[str, bytes]]:
pvd = read_user(data, 16)
if pvd[1:6] != b"CD001":
raise SystemExit("PVD not found at LBA 16 (expected CD001)")
root_extent = struct.unpack_from("<I", pvd, 158)[0]
root_size = struct.unpack_from("<I", pvd, 166)[0]
root = bytearray()
for i in range((root_size + USER - 1) // USER):
root += read_user(data, root_extent + i)
root = bytes(root[:root_size])
entries = parse_root_entries(root)
files: dict[str, bytes] = {}
for need in ("SYSTEM.CNF", boot_exe):
if need not in entries:
raise SystemExit(f"missing {need} on disc (found {sorted(entries)[:20]})")
extent, size = entries[need]
out = bytearray()
rem, lba = size, extent
while rem > 0:
sector = read_user(data, lba)
take = min(USER, rem)
out += sector[:take]
rem -= take
lba += 1
files[need] = bytes(out)
if files[boot_exe][:8] != b"PS-X EXE":
raise SystemExit(f"{boot_exe} is not a PS-X EXE")
return entries, files
def iso_to_bin(iso_data: bytes) -> bytes:
if len(iso_data) % USER != 0:
raise SystemExit(f"ISO size {len(iso_data)} is not a multiple of 2048")
n = len(iso_data) // USER
out = bytearray()
for lba in range(n):
user = iso_data[lba * USER : (lba + 1) * USER]
out += encode_mode2_form1(user, lba, submode=0x08)
return bytes(out)
def raw2448_to_bin(src: Path) -> bytes:
out = bytearray()
n = 0
with open(src, "rb") as inf:
while True:
sec = inf.read(SRC_2448)
if not sec:
break
if len(sec) != SRC_2448:
raise SystemExit(f"truncated 2448 sector {n}: got {len(sec)} bytes")
out += sec[:DST_SEC]
n += 1
print(f" trimmed {n} sectors 2448 → 2352")
return bytes(out)
def matches_known(cfg: PrepareConfig, size: int, md5: str, sha1: str) -> bool:
if not cfg.known:
return False
md5 = md5.lower()
sha1 = sha1.lower()
for k in cfg.known:
if k.md5 and k.md5.lower() == md5:
if not k.size or k.size == size:
return True
if k.sha1 and k.sha1.lower() == sha1:
if not k.size or k.size == size:
return True
return False
def main() -> int:
here = Path(__file__).resolve().parent
# tools/ → psxrecomp/ → often game root is parent of psxrecomp
fw_root = here.parent
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("source", nargs="?", default="", help="path to .iso / .cue / .bin")
ap.add_argument(
"--config",
default="",
help="game.toml (reads [game] + [prepare_disc])",
)
ap.add_argument(
"--project-root",
default="",
help="game project root (default: parent of config, else cwd)",
)
ap.add_argument("--out-dir", default="", help="override prepare_disc.out_dir")
ap.add_argument("--bin-name", default="", help="override output .bin basename")
ap.add_argument("--cue-name", default="", help="override output .cue basename")
ap.add_argument("--boot-exe", default="", help="override boot EXE name on disc")
ap.add_argument(
"--skip-hash-check",
action="store_true",
help="do not require prepare_disc.known_* digests",
)
args = ap.parse_args()
config_path = Path(args.config).expanduser() if args.config else None
if config_path and not config_path.is_file():
print(f"config not found: {config_path}", file=sys.stderr)
return 1
if args.project_root:
project_root = Path(args.project_root).expanduser().resolve()
elif config_path:
project_root = config_path.parent.resolve()
else:
# Invoked from a game tree: prefer cwd if it has game.toml
cwd = Path.cwd()
project_root = cwd if (cwd / "game.toml").is_file() else fw_root.parent
if not config_path:
cand = project_root / "game.toml"
if cand.is_file():
config_path = cand
cfg = load_config(project_root, config_path, args.out_dir or None, args.skip_hash_check)
if args.bin_name:
cfg.bin_name = args.bin_name
if args.cue_name:
cfg.cue_name = args.cue_name
if args.boot_exe:
cfg.boot_exe = args.boot_exe
if not args.source:
# Convenience: existing working bin under out_dir
cand = cfg.out_dir / cfg.bin_name
if cand.is_file():
args.source = str(cand)
else:
print("source required (.iso / .cue / .bin)", file=sys.stderr)
return 1
src = Path(args.source).expanduser().resolve()
if not src.is_file():
print(f"source not found: {src}", file=sys.stderr)
return 1
cfg.out_dir.mkdir(parents=True, exist_ok=True)
src_md5, src_sha1, src_size = file_hashes(src)
print(f"source: {src}")
print(f" size {src_size}")
print(f" md5 {src_md5}")
print(f" sha1 {src_sha1}")
if cfg.serial:
print(f" game {cfg.serial} boot={cfg.boot_exe}")
kind = detect_kind(src, src_size)
if kind == "cue":
src = resolve_cue_bin(src)
src_md5, src_sha1, src_size = file_hashes(src)
print(f"cue → bin: {src}")
print(f" size {src_size}")
print(f" md5 {src_md5}")
print(f" sha1 {src_sha1}")
kind = detect_kind(src, src_size)
known_hit = matches_known(cfg, src_size, src_md5, src_sha1)
if cfg.known and not cfg.skip_hash_check:
if known_hit:
print(" known digest OK")
elif kind in ("bin2352", "raw2448"):
print(
"source digests are not in prepare_disc.known_* "
"(pass --skip-hash-check to force)",
file=sys.stderr,
)
return 1
else:
print(
"ISO digests are not in prepare_disc.known_*; "
"verifying boot EXE only."
)
elif not cfg.known and not cfg.skip_hash_check and kind == "bin2352":
print(" no prepare_disc.known_* configured — verifying boot EXE only")
if kind == "bin2352":
bin_data = src.read_bytes()
entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe)
elif kind == "iso":
iso_data = src.read_bytes()
entries, files = extract_via(read_user_iso, iso_data, cfg.boot_exe)
print(f" converting {len(iso_data) // USER} sectors ISO → MODE2/2352…")
bin_data = iso_to_bin(iso_data)
extract_via(read_user_bin, bin_data, cfg.boot_exe)
elif kind == "raw2448":
print(" converting 2448 → 2352…")
bin_data = raw2448_to_bin(src)
entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe)
else:
print(f"unsupported kind {kind}", file=sys.stderr)
return 1
print(f" root entries: {sorted(entries)[:24]}")
pc0 = struct.unpack_from("<I", files[cfg.boot_exe], 0x10)[0]
print(f" {cfg.boot_exe} PC0={pc0:#010x} ({len(files[cfg.boot_exe])} bytes)")
for name, blob in files.items():
out_path = cfg.out_dir / name
out_path.write_bytes(blob)
print(f"wrote {out_path} ({len(blob)} bytes)")
bin_path = cfg.out_dir / cfg.bin_name
if src.resolve() == bin_path.resolve() and kind == "bin2352":
print(f"source already at {bin_path}")
else:
print(f"writing {bin_path}")
bin_path.write_bytes(bin_data)
cue_path = cfg.out_dir / cfg.cue_name
cue_path.write_text(
f'FILE "{cfg.bin_name}" BINARY\n'
f" TRACK 01 MODE2/2352\n"
f" INDEX 01 00:00:00\n",
encoding="ascii",
newline="\n",
)
print(f"wrote {cue_path}")
out_md5, out_sha1, out_size = file_hashes(bin_path)
print("working bin:")
print(f" size {out_size}")
print(f" md5 {out_md5}")
print(f" sha1 {out_sha1}")
if kind == "iso":
print(" rebuilt from ISO (EDC set; ECC zeroed — fine for software readers)")
elif kind == "raw2448":
print(" trimmed from 2448-byte/sector dump")
print(f"RESULT_CUE={cue_path.resolve()}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+76
View File
@@ -0,0 +1,76 @@
"""JSONL progress events for headless psxrecomp SDK consumers."""
from __future__ import annotations
import json
import sys
import time
from typing import Any, Mapping, Optional, TextIO
class ProgressReporter:
"""Emit coarse phase events for UIs and launchers.
When ``json_progress`` is true, stdout is reserved for one JSON object
per line. Human-readable text goes to stderr (or is mirrored as
``log`` events when ``mirror_logs`` is true).
"""
def __init__(
self,
*,
json_progress: bool = False,
stream: Optional[TextIO] = None,
log_stream: Optional[TextIO] = None,
mirror_logs: bool = True,
) -> None:
self.json_progress = bool(json_progress)
self.stream = stream if stream is not None else sys.stdout
self.log_stream = log_stream if log_stream is not None else sys.stderr
self.mirror_logs = bool(mirror_logs)
self._started = time.perf_counter()
def _emit(self, payload: Mapping[str, Any]) -> None:
if not self.json_progress:
return
line = json.dumps(payload, sort_keys=True, separators=(",", ":"))
self.stream.write(line + "\n")
self.stream.flush()
def event(self, event: str, **fields: Any) -> None:
payload: dict[str, Any] = {
"event": event,
"t": round(time.perf_counter() - self._started, 3),
}
for key, value in fields.items():
if value is not None:
payload[key] = value
self._emit(payload)
def phase(
self,
name: str,
*,
pct: Optional[float] = None,
message: Optional[str] = None,
**fields: Any,
) -> None:
self.event("phase", phase=name, pct=pct, message=message, **fields)
if message and not self.json_progress:
print(message, file=self.log_stream, flush=True)
def log(self, message: str, *, level: str = "info") -> None:
if self.json_progress:
self.event("log", level=level, message=message)
if self.mirror_logs:
print(message, file=self.log_stream, flush=True)
else:
print(message, file=self.log_stream, flush=True)
def result(self, **fields: Any) -> None:
self.event("result", **fields)
def error(self, message: str, *, code: int = 1, **fields: Any) -> None:
self.event("error", message=message, code=code, **fields)
if not self.json_progress:
print(message, file=self.log_stream, flush=True)