mirror of
https://github.com/RetroPortingToolKit/psxrecomp
synced 2026-10-25 18:05:12 -04:00
disc conversion and optimizations to generate+build flow
This commit is contained in:
Executable
+844
@@ -0,0 +1,844 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Headless disc → generate / rebuild / optional PGO for psxrecomp games.
|
||||
|
||||
Commands:
|
||||
verify-disc Hash-check a dump against game.toml [prepare_disc]
|
||||
generate Prepare disc (if needed) + run psxrecomp-game → generated/
|
||||
rebuild cmake --build; if [pgo] enabled, instrument → train → use
|
||||
pgo-train Standalone PGO train (same as rebuild's PGO phase)
|
||||
|
||||
Exit codes: 0 ok · 1 runtime · 2 usage · 3 disc verify fail
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(ROOT / "tools"))
|
||||
from sdk_progress import ProgressReporter # noqa: E402
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_ERROR = 1
|
||||
EXIT_USAGE = 2
|
||||
EXIT_VERIFY = 3
|
||||
|
||||
|
||||
def _parse_array_items(inner: str) -> list[Any]:
|
||||
items: list[Any] = []
|
||||
if not inner.strip():
|
||||
return items
|
||||
for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner):
|
||||
part = part.strip().rstrip(",").strip()
|
||||
if not part:
|
||||
continue
|
||||
if part.startswith('"') and part.endswith('"'):
|
||||
items.append(part[1:-1])
|
||||
else:
|
||||
try:
|
||||
items.append(int(part, 0))
|
||||
except ValueError:
|
||||
items.append(part)
|
||||
return items
|
||||
|
||||
|
||||
def parse_toml_simple(text: str) -> dict[str, dict[str, Any]]:
|
||||
sections: dict[str, dict[str, Any]] = {"": {}}
|
||||
cur = ""
|
||||
array_key: Optional[str] = None
|
||||
array_buf: list[str] = []
|
||||
for raw in text.splitlines():
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if array_key is not None:
|
||||
array_buf.append(line)
|
||||
joined = " ".join(array_buf)
|
||||
if "]" in joined:
|
||||
inner = joined.split("]", 1)[0]
|
||||
if inner.startswith("["):
|
||||
inner = inner[1:]
|
||||
sections[cur][array_key] = _parse_array_items(inner)
|
||||
array_key = None
|
||||
array_buf = []
|
||||
continue
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
cur = line[1:-1].strip()
|
||||
sections.setdefault(cur, {})
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, val = line.split("=", 1)
|
||||
key, val = key.strip(), val.strip()
|
||||
if val.startswith("["):
|
||||
if val.endswith("]") and val.count("[") == val.count("]"):
|
||||
sections[cur][key] = _parse_array_items(val[1:-1])
|
||||
else:
|
||||
array_key = key
|
||||
array_buf = [val[1:]]
|
||||
continue
|
||||
if val.startswith('"') and val.endswith('"'):
|
||||
sections[cur][key] = val[1:-1]
|
||||
elif val.lower() in ("true", "false"):
|
||||
sections[cur][key] = val.lower() == "true"
|
||||
else:
|
||||
try:
|
||||
sections[cur][key] = int(val, 0)
|
||||
except ValueError:
|
||||
sections[cur][key] = val
|
||||
return sections
|
||||
|
||||
|
||||
def file_hashes(path: Path) -> tuple[str, str, int]:
|
||||
h_md5, h_sha1 = hashlib.md5(), hashlib.sha1()
|
||||
size = 0
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
h_md5.update(chunk)
|
||||
h_sha1.update(chunk)
|
||||
return h_md5.hexdigest(), h_sha1.hexdigest(), size
|
||||
|
||||
|
||||
def resolve_cue_bin(cue_path: Path) -> Path:
|
||||
text = cue_path.read_text(encoding="utf-8", errors="replace")
|
||||
m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I)
|
||||
if not m:
|
||||
m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I)
|
||||
if not m:
|
||||
raise ValueError(f"no BINARY FILE in cue: {cue_path}")
|
||||
cand = Path(m.group(1))
|
||||
if not cand.is_absolute():
|
||||
cand = cue_path.parent / cand
|
||||
if not cand.is_file():
|
||||
raise ValueError(f"cue references missing bin: {cand}")
|
||||
return cand
|
||||
|
||||
|
||||
def find_psxrecomp_game(project_root: Path) -> Path:
|
||||
env = os.environ.get("PSXRECOMP_GAME", "").strip()
|
||||
if env:
|
||||
p = Path(env).expanduser()
|
||||
if p.is_file():
|
||||
return p.resolve()
|
||||
candidates = [
|
||||
project_root / "psxrecomp" / "recompiler" / "build" / "psxrecomp-game",
|
||||
project_root / "psxrecomp" / "recompiler" / "build" / "psxrecomp-game.exe",
|
||||
ROOT / "recompiler" / "build" / "psxrecomp-game",
|
||||
ROOT / "recompiler" / "build" / "psxrecomp-game.exe",
|
||||
]
|
||||
for c in candidates:
|
||||
if c.is_file():
|
||||
return c.resolve()
|
||||
which = shutil.which("psxrecomp-game")
|
||||
if which:
|
||||
return Path(which).resolve()
|
||||
raise FileNotFoundError(
|
||||
"psxrecomp-game not found. Build it:\n"
|
||||
" cmake -S psxrecomp/recompiler -B psxrecomp/recompiler/build -G Ninja\n"
|
||||
" cmake --build psxrecomp/recompiler/build --target psxrecomp-game\n"
|
||||
"Or set PSXRECOMP_GAME=/path/to/psxrecomp-game"
|
||||
)
|
||||
|
||||
|
||||
def load_sections(config: Path) -> dict[str, dict[str, Any]]:
|
||||
return parse_toml_simple(config.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def verify_disc_path(
|
||||
disc: Path,
|
||||
prep: dict[str, Any],
|
||||
*,
|
||||
skip_hash: bool,
|
||||
progress: ProgressReporter,
|
||||
) -> dict[str, Any]:
|
||||
path = disc.resolve()
|
||||
if path.suffix.lower() == ".cue":
|
||||
path = resolve_cue_bin(path)
|
||||
md5, sha1, size = file_hashes(path)
|
||||
identity = {
|
||||
"path": str(path),
|
||||
"md5": md5,
|
||||
"sha1": sha1,
|
||||
"size": size,
|
||||
"verified": False,
|
||||
}
|
||||
progress.event("disc", **identity)
|
||||
sizes = [int(s) for s in (prep.get("known_sizes") or [])]
|
||||
md5s = [str(x).lower() for x in (prep.get("known_md5") or [])]
|
||||
sha1s = [str(x).lower() for x in (prep.get("known_sha1") or [])]
|
||||
if not md5s and not sha1s and not sizes:
|
||||
identity["verified"] = True
|
||||
return identity
|
||||
if skip_hash:
|
||||
return identity
|
||||
ok = (md5 in md5s) or (sha1 in sha1s)
|
||||
if not ok and sizes and size in sizes and not md5s and not sha1s:
|
||||
ok = True
|
||||
if not ok:
|
||||
raise DiscVerifyError(
|
||||
f"disc digests not in prepare_disc.known_* "
|
||||
f"(size={size} md5={md5} sha1={sha1})"
|
||||
)
|
||||
identity["verified"] = True
|
||||
return identity
|
||||
|
||||
|
||||
class DiscVerifyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def cmd_verify_disc(args: argparse.Namespace, progress: ProgressReporter) -> int:
|
||||
config = Path(args.config).expanduser().resolve()
|
||||
if not config.is_file():
|
||||
progress.error(f"config not found: {config}", code=EXIT_USAGE)
|
||||
return EXIT_USAGE
|
||||
project_root = (
|
||||
Path(args.project_root).expanduser().resolve()
|
||||
if args.project_root
|
||||
else config.parent
|
||||
)
|
||||
disc = Path(args.disc).expanduser()
|
||||
if not disc.is_absolute():
|
||||
disc = (project_root / disc).resolve()
|
||||
else:
|
||||
disc = disc.resolve()
|
||||
if not disc.is_file():
|
||||
progress.error(f"disc not found: {disc}", code=EXIT_USAGE)
|
||||
return EXIT_USAGE
|
||||
secs = load_sections(config)
|
||||
prep = secs.get("prepare_disc") or {}
|
||||
progress.phase("verify", pct=0.1, message=f"Verifying {disc.name}")
|
||||
try:
|
||||
identity = verify_disc_path(
|
||||
disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress
|
||||
)
|
||||
except DiscVerifyError as exc:
|
||||
progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True)
|
||||
return EXIT_VERIFY
|
||||
progress.phase("done", pct=1.0, message="Disc OK")
|
||||
progress.result(ok=True, **identity)
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
def run_prepare_disc(
|
||||
project_root: Path,
|
||||
config: Path,
|
||||
source: Path,
|
||||
progress: ProgressReporter,
|
||||
) -> Path:
|
||||
script = ROOT / "tools" / "prepare_disc.py"
|
||||
if not script.is_file():
|
||||
raise RuntimeError(f"missing {script}")
|
||||
progress.phase("prepare_disc", pct=0.15, message="Normalizing disc image…")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(script),
|
||||
"--config",
|
||||
str(config),
|
||||
"--project-root",
|
||||
str(project_root),
|
||||
str(source),
|
||||
]
|
||||
proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True)
|
||||
out = (proc.stdout or "") + (proc.stderr or "")
|
||||
for line in out.splitlines():
|
||||
if line.strip():
|
||||
progress.log(line)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"prepare_disc failed (exit {proc.returncode})")
|
||||
marker = "RESULT_CUE="
|
||||
cue = None
|
||||
for line in out.splitlines():
|
||||
if line.startswith(marker):
|
||||
cue = Path(line[len(marker) :].strip())
|
||||
if not cue or not cue.is_file():
|
||||
raise RuntimeError("prepare_disc did not print RESULT_CUE=")
|
||||
return cue.resolve()
|
||||
|
||||
|
||||
def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int:
|
||||
config = Path(args.config).expanduser().resolve()
|
||||
if not config.is_file():
|
||||
progress.error(f"config not found: {config}", code=EXIT_USAGE)
|
||||
return EXIT_USAGE
|
||||
project_root = (
|
||||
Path(args.project_root).expanduser().resolve()
|
||||
if args.project_root
|
||||
else config.parent
|
||||
)
|
||||
secs = load_sections(config)
|
||||
game = secs.get("game") or {}
|
||||
prep = secs.get("prepare_disc") or {}
|
||||
recomp = secs.get("recompiler") or {}
|
||||
|
||||
disc_arg = args.disc
|
||||
if not disc_arg:
|
||||
disc_arg = game.get("disc") or ""
|
||||
if not disc_arg:
|
||||
progress.error("no --disc and game.disc empty", code=EXIT_USAGE)
|
||||
return EXIT_USAGE
|
||||
disc = Path(str(disc_arg)).expanduser()
|
||||
if not disc.is_absolute():
|
||||
disc = (project_root / disc).resolve()
|
||||
else:
|
||||
disc = disc.resolve()
|
||||
|
||||
progress.phase("verify", pct=0.05, message=f"Checking disc {disc.name}")
|
||||
try:
|
||||
if disc.is_file():
|
||||
verify_disc_path(
|
||||
disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress
|
||||
)
|
||||
except DiscVerifyError as exc:
|
||||
progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True)
|
||||
return EXIT_VERIFY
|
||||
|
||||
boot = str(prep.get("boot_exe") or Path(str(game.get("exe") or "")).name)
|
||||
out_rel = str(prep.get("out_dir") or "prepared_disc")
|
||||
boot_path = project_root / out_rel / boot
|
||||
working_disc = disc
|
||||
# Normalize library dumps when boot EXE missing or source looks like ISO/raw.
|
||||
need_prep = (not boot_path.is_file()) or disc.suffix.lower() in (
|
||||
".iso",
|
||||
".ISO",
|
||||
)
|
||||
if need_prep or args.force_prepare:
|
||||
try:
|
||||
working_disc = run_prepare_disc(project_root, config, disc, progress)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
progress.error(str(exc), code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
if not boot_path.is_file():
|
||||
progress.error(f"boot EXE missing after prepare: {boot_path}", code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
|
||||
try:
|
||||
game_tool = find_psxrecomp_game(project_root)
|
||||
except FileNotFoundError as exc:
|
||||
progress.error(str(exc), code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
|
||||
out_dir = project_root / str(recomp.get("out_dir") or "generated")
|
||||
progress.phase(
|
||||
"emit",
|
||||
pct=0.35,
|
||||
message=f"Running psxrecomp-game → {out_dir}",
|
||||
)
|
||||
cmd = [str(game_tool), "--config", str(config)]
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(project_root),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for stream in (proc.stdout, proc.stderr):
|
||||
if not stream:
|
||||
continue
|
||||
for line in stream.splitlines():
|
||||
if line.strip():
|
||||
progress.log(line)
|
||||
if proc.returncode != 0:
|
||||
progress.error(
|
||||
f"psxrecomp-game failed (exit {proc.returncode})", code=EXIT_ERROR
|
||||
)
|
||||
return EXIT_ERROR
|
||||
|
||||
marker_name = args.gen_marker or f"{boot}_dispatch.c"
|
||||
marker = out_dir / marker_name
|
||||
if not marker.is_file():
|
||||
# Accept any *_dispatch.c
|
||||
hits = list(out_dir.glob("*_dispatch.c"))
|
||||
if not hits:
|
||||
progress.error(f"generate produced no dispatch under {out_dir}", code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
marker = hits[0]
|
||||
|
||||
progress.phase("done", pct=1.0, message="Generate complete")
|
||||
progress.result(
|
||||
ok=True,
|
||||
out_dir=str(out_dir),
|
||||
marker=str(marker),
|
||||
disc=str(working_disc),
|
||||
boot_exe=str(boot_path),
|
||||
)
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
def _cmake_configure(
|
||||
project_root: Path,
|
||||
build_dir: Path,
|
||||
*,
|
||||
pgo: str,
|
||||
extra: list[str],
|
||||
progress: ProgressReporter,
|
||||
) -> None:
|
||||
build_dir.mkdir(parents=True, exist_ok=True)
|
||||
cmd = [
|
||||
"cmake",
|
||||
"-S",
|
||||
str(project_root),
|
||||
"-B",
|
||||
str(build_dir),
|
||||
]
|
||||
# Only force generator on fresh trees (avoid clash with existing cache).
|
||||
if not (build_dir / "CMakeCache.txt").is_file():
|
||||
cmd.extend(["-G", "Ninja"])
|
||||
cmd.append(f"-DPSX_PGO={pgo}")
|
||||
cmd.extend(extra)
|
||||
progress.log(" ".join(cmd))
|
||||
proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True)
|
||||
for stream in (proc.stdout, proc.stderr):
|
||||
if stream:
|
||||
for line in stream.splitlines():
|
||||
if line.strip():
|
||||
progress.log(line)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"cmake configure failed (exit {proc.returncode})")
|
||||
|
||||
|
||||
def _cmake_build(
|
||||
build_dir: Path,
|
||||
target: str,
|
||||
progress: ProgressReporter,
|
||||
) -> None:
|
||||
jobs = os.environ.get("CMAKE_BUILD_PARALLEL_LEVEL") or str(
|
||||
os.cpu_count() or 4
|
||||
)
|
||||
cmd = [
|
||||
"cmake",
|
||||
"--build",
|
||||
str(build_dir),
|
||||
"--parallel",
|
||||
jobs,
|
||||
"--target",
|
||||
target,
|
||||
]
|
||||
progress.log(" ".join(cmd))
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
for stream in (proc.stdout, proc.stderr):
|
||||
if stream:
|
||||
for line in stream.splitlines():
|
||||
if line.strip():
|
||||
progress.log(line)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"cmake --build failed (exit {proc.returncode})")
|
||||
|
||||
|
||||
def _soft_stop(pid: int, timeout: int = 30) -> None:
|
||||
try:
|
||||
os.kill(pid, 15) # SIGTERM
|
||||
except ProcessLookupError:
|
||||
return
|
||||
for _ in range(timeout):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
time.sleep(1)
|
||||
try:
|
||||
os.kill(pid, 9)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def _pgo_train_warning(*, hide_video: bool) -> str:
|
||||
if hide_video:
|
||||
return (
|
||||
"WARNING: PGO training is running (no video window). Do not cancel "
|
||||
"or kill this process until training finishes — interrupting may "
|
||||
"corrupt profiles and force a restart of this step."
|
||||
)
|
||||
return (
|
||||
"WARNING: PGO training is running. Do not close the game window or "
|
||||
"interact with the application until training finishes — closing or "
|
||||
"clicking may corrupt profiles and force a restart of this step."
|
||||
)
|
||||
|
||||
|
||||
def run_pgo_train(
|
||||
project_root: Path,
|
||||
build_dir: Path,
|
||||
*,
|
||||
exe_basename: str,
|
||||
disc: Path,
|
||||
config: Path,
|
||||
train_secs: int,
|
||||
train_runs: int,
|
||||
mute_host_audio: bool,
|
||||
hide_video: bool,
|
||||
progress: ProgressReporter,
|
||||
) -> None:
|
||||
exe = build_dir / exe_basename
|
||||
if not exe.is_file():
|
||||
exe = build_dir / f"{exe_basename}.exe"
|
||||
if not exe.is_file():
|
||||
raise RuntimeError(f"train binary missing: {exe_basename} under {build_dir}")
|
||||
|
||||
pgo_dir = build_dir / "pgo"
|
||||
if pgo_dir.exists():
|
||||
shutil.rmtree(pgo_dir)
|
||||
pgo_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.setdefault("PSX_BIOS_HLE", "0")
|
||||
env["LLVM_PROFILE_FILE"] = str(pgo_dir / "bpe-%p.profraw")
|
||||
# Discard SDL output; SPU/CD still advance (profiles stay valid).
|
||||
if mute_host_audio:
|
||||
env["PSX_HOST_MUTE"] = "1"
|
||||
else:
|
||||
env.pop("PSX_HOST_MUTE", None)
|
||||
# No on-screen FMV (photosensitive / abrasive content). Guest MDEC/GPU
|
||||
# still run; host present is skipped. Matches prior CI train shape.
|
||||
if hide_video:
|
||||
env["PSX_HEADLESS"] = "1"
|
||||
env.setdefault("SDL_VIDEODRIVER", "dummy")
|
||||
else:
|
||||
env.pop("PSX_HEADLESS", None)
|
||||
|
||||
warning = _pgo_train_warning(hide_video=hide_video)
|
||||
progress.phase("pgo_train", pct=0.35, message=warning)
|
||||
# Always surface on the console even when JSONL owns stdout.
|
||||
print(f"\n*** {warning} ***\n", file=sys.stderr, flush=True)
|
||||
if mute_host_audio:
|
||||
progress.log("Host audio muted for train (PSX_HOST_MUTE=1; SPU still runs).")
|
||||
else:
|
||||
progress.log("Host audio unmuted for train (speakers will play).")
|
||||
if hide_video:
|
||||
progress.log(
|
||||
"Video hidden for train (--headless / PSX_HEADLESS=1; "
|
||||
"guest FMV decode still runs, nothing shown on screen)."
|
||||
)
|
||||
else:
|
||||
progress.log("Video window visible for train (host present paths included).")
|
||||
|
||||
for run in range(1, train_runs + 1):
|
||||
progress.phase(
|
||||
"pgo_train",
|
||||
pct=0.4 + 0.4 * (run - 1) / max(train_runs, 1),
|
||||
message=(
|
||||
f"WARNING: PGO train run {run}/{train_runs} ({train_secs}s) — "
|
||||
+ (
|
||||
"Do not cancel this process."
|
||||
if hide_video
|
||||
else "Do not close or interact with the application."
|
||||
)
|
||||
),
|
||||
)
|
||||
cmd = [
|
||||
str(exe),
|
||||
"--no-launcher",
|
||||
"--game",
|
||||
str(config),
|
||||
"--disc",
|
||||
str(disc),
|
||||
]
|
||||
if hide_video:
|
||||
cmd.append("--headless")
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(project_root),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(train_secs)
|
||||
if proc.poll() is None:
|
||||
_soft_stop(proc.pid)
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
|
||||
n_gcda = len(list(build_dir.rglob("*.gcda")))
|
||||
n_raw = len(list(pgo_dir.glob("*.profraw")))
|
||||
progress.log(f"PGO profiles: {n_gcda} .gcda, {n_raw} .profraw")
|
||||
if n_raw >= 1:
|
||||
merge = shutil.which("llvm-profdata")
|
||||
if not merge:
|
||||
# macOS xcrun
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["xcrun", "--find", "llvm-profdata"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
merge = "xcrun"
|
||||
except OSError:
|
||||
pass
|
||||
raws = list(pgo_dir.glob("*.profraw"))
|
||||
out = pgo_dir / "default.profdata"
|
||||
if merge == "xcrun":
|
||||
cmd = ["xcrun", "llvm-profdata", "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)]
|
||||
elif merge:
|
||||
cmd = [merge, "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)]
|
||||
else:
|
||||
raise RuntimeError("llvm-profdata not found (Clang PGO merge)")
|
||||
subprocess.run(cmd, check=True)
|
||||
if n_gcda < 1 and not (pgo_dir / "default.profdata").is_file():
|
||||
raise RuntimeError("no PGO profiles written — train did not flush")
|
||||
|
||||
|
||||
def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int:
|
||||
config = Path(args.config).expanduser().resolve()
|
||||
if not config.is_file():
|
||||
progress.error(f"config not found: {config}", code=EXIT_USAGE)
|
||||
return EXIT_USAGE
|
||||
project_root = (
|
||||
Path(args.project_root).expanduser().resolve()
|
||||
if args.project_root
|
||||
else config.parent
|
||||
)
|
||||
build_dir = Path(args.build_dir).expanduser()
|
||||
if not build_dir.is_absolute():
|
||||
build_dir = (project_root / build_dir).resolve()
|
||||
else:
|
||||
build_dir = build_dir.resolve()
|
||||
target = args.target or "psx-runtime"
|
||||
exe_basename = args.exe_basename or "psx-runtime"
|
||||
|
||||
secs = load_sections(config)
|
||||
pgo = secs.get("pgo") or {}
|
||||
# Opt-in only: requires game.toml [pgo] enabled = true (or --force-pgo).
|
||||
pgo_enabled = bool(pgo.get("enabled", False)) and not args.no_pgo
|
||||
if args.force_pgo:
|
||||
pgo_enabled = True
|
||||
|
||||
game = secs.get("game") or {}
|
||||
prep = secs.get("prepare_disc") or {}
|
||||
if args.disc:
|
||||
disc = Path(args.disc).expanduser()
|
||||
if not disc.is_absolute():
|
||||
disc = (project_root / disc).resolve()
|
||||
else:
|
||||
disc = disc.resolve()
|
||||
else:
|
||||
cue_name = prep.get("cue_name")
|
||||
if cue_name:
|
||||
disc = (
|
||||
project_root / str(prep.get("out_dir") or "bpe") / str(cue_name)
|
||||
).resolve()
|
||||
elif game.get("disc"):
|
||||
disc = (project_root / str(game["disc"])).resolve()
|
||||
if disc.suffix.lower() == ".bin":
|
||||
cue = disc.with_suffix(".cue")
|
||||
if cue.is_file():
|
||||
disc = cue
|
||||
else:
|
||||
disc = None
|
||||
|
||||
cmake_extra = []
|
||||
if args.cmake_extra:
|
||||
cmake_extra.extend(args.cmake_extra)
|
||||
|
||||
# mute_host_audio / hide_video: default ON; game.toml / CLI can disable.
|
||||
mute_host = True
|
||||
if "mute_host_audio" in pgo:
|
||||
mute_host = bool(pgo.get("mute_host_audio"))
|
||||
if getattr(args, "pgo_audio", False):
|
||||
mute_host = False
|
||||
if getattr(args, "pgo_mute", False):
|
||||
mute_host = True
|
||||
|
||||
hide_video = True
|
||||
if "hide_video" in pgo:
|
||||
hide_video = bool(pgo.get("hide_video"))
|
||||
if getattr(args, "pgo_video", False):
|
||||
hide_video = False
|
||||
if getattr(args, "pgo_hide_video", False):
|
||||
hide_video = True
|
||||
|
||||
try:
|
||||
if not pgo_enabled:
|
||||
progress.phase("build", pct=0.2, message="cmake Release build…")
|
||||
_cmake_configure(
|
||||
project_root, build_dir, pgo="", extra=cmake_extra, progress=progress
|
||||
)
|
||||
_cmake_build(build_dir, target, progress)
|
||||
else:
|
||||
# Framework defaults (60×2); game.toml / CLI may lengthen for hard titles.
|
||||
train_secs = int(getattr(args, "train_secs", 0) or 0)
|
||||
if train_secs <= 0:
|
||||
train_secs = int(pgo.get("train_secs") or 60)
|
||||
train_runs = int(getattr(args, "train_runs", 0) or 0)
|
||||
if train_runs <= 0:
|
||||
train_runs = int(pgo.get("train_runs") or 2)
|
||||
progress.phase(
|
||||
"pgo_instrument",
|
||||
pct=0.1,
|
||||
message="PGO instrumented build…",
|
||||
)
|
||||
_cmake_configure(
|
||||
project_root,
|
||||
build_dir,
|
||||
pgo="generate",
|
||||
extra=cmake_extra,
|
||||
progress=progress,
|
||||
)
|
||||
_cmake_build(build_dir, target, progress)
|
||||
if not disc or not Path(disc).is_file():
|
||||
raise RuntimeError(
|
||||
f"PGO train needs a playable disc (got {disc!s}). "
|
||||
"Pass --disc or set game.disc / prepare_disc.cue_name."
|
||||
)
|
||||
run_pgo_train(
|
||||
project_root,
|
||||
build_dir,
|
||||
exe_basename=exe_basename,
|
||||
disc=Path(disc),
|
||||
config=config,
|
||||
train_secs=train_secs,
|
||||
train_runs=train_runs,
|
||||
mute_host_audio=mute_host,
|
||||
hide_video=hide_video,
|
||||
progress=progress,
|
||||
)
|
||||
progress.phase("pgo_use", pct=0.85, message="PGO optimized rebuild…")
|
||||
_cmake_configure(
|
||||
project_root,
|
||||
build_dir,
|
||||
pgo="use",
|
||||
extra=cmake_extra,
|
||||
progress=progress,
|
||||
)
|
||||
_cmake_build(build_dir, target, progress)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
progress.error(str(exc), code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
|
||||
exe = build_dir / exe_basename
|
||||
if not exe.is_file():
|
||||
exe = build_dir / f"{exe_basename}.exe"
|
||||
progress.phase("done", pct=1.0, message="Rebuild complete")
|
||||
progress.result(ok=True, exe=str(exe), pgo=pgo_enabled)
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int:
|
||||
args.force_pgo = True
|
||||
args.no_pgo = False
|
||||
return cmd_rebuild(args, progress)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
ap = argparse.ArgumentParser(prog="psxrecomp_cli", description=__doc__)
|
||||
sub = ap.add_subparsers(dest="command", required=True)
|
||||
|
||||
def add_common(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument("--config", default="game.toml", help="game.toml path")
|
||||
p.add_argument("--project-root", default="", help="game project root")
|
||||
p.add_argument("--json-progress", action="store_true")
|
||||
|
||||
v = sub.add_parser("verify-disc", help="verify disc digests")
|
||||
add_common(v)
|
||||
v.add_argument("--disc", required=True)
|
||||
v.add_argument("--skip-hash-check", action="store_true")
|
||||
v.set_defaults(handler=cmd_verify_disc)
|
||||
|
||||
g = sub.add_parser("generate", help="prepare disc + generate C")
|
||||
add_common(g)
|
||||
g.add_argument("--disc", default="", help="source dump or working cue/bin")
|
||||
g.add_argument("--skip-hash-check", action="store_true")
|
||||
g.add_argument("--force-prepare", action="store_true")
|
||||
g.add_argument("--gen-marker", default="", help="expected marker under out_dir")
|
||||
g.set_defaults(handler=cmd_generate)
|
||||
|
||||
r = sub.add_parser("rebuild", help="cmake build (+ optional local PGO)")
|
||||
add_common(r)
|
||||
r.add_argument("--build-dir", required=True)
|
||||
r.add_argument("--target", default="psx-runtime")
|
||||
r.add_argument("--exe-basename", default="")
|
||||
r.add_argument("--disc", default="", help="disc for PGO train")
|
||||
r.add_argument(
|
||||
"--no-pgo",
|
||||
action="store_true",
|
||||
help="skip PGO even if game.toml [pgo] enabled=true",
|
||||
)
|
||||
r.add_argument(
|
||||
"--force-pgo",
|
||||
action="store_true",
|
||||
help="run PGO even without [pgo] enabled=true",
|
||||
)
|
||||
r.add_argument(
|
||||
"--train-secs",
|
||||
type=int,
|
||||
default=0,
|
||||
help="override [pgo] train_secs (default framework 60)",
|
||||
)
|
||||
r.add_argument(
|
||||
"--train-runs",
|
||||
type=int,
|
||||
default=0,
|
||||
help="override [pgo] train_runs (default framework 2)",
|
||||
)
|
||||
r.add_argument(
|
||||
"--pgo-mute",
|
||||
action="store_true",
|
||||
help="force mute host speakers during train (default)",
|
||||
)
|
||||
r.add_argument(
|
||||
"--pgo-audio",
|
||||
action="store_true",
|
||||
help="allow host speakers during train (overrides mute default)",
|
||||
)
|
||||
r.add_argument(
|
||||
"--pgo-hide-video",
|
||||
action="store_true",
|
||||
help="force headless train with no on-screen video (default)",
|
||||
)
|
||||
r.add_argument(
|
||||
"--pgo-video",
|
||||
action="store_true",
|
||||
help="show a game window during train (includes host present paths)",
|
||||
)
|
||||
r.add_argument("--cmake-extra", action="append", default=[])
|
||||
r.set_defaults(handler=cmd_rebuild)
|
||||
|
||||
p = sub.add_parser("pgo-train", help="force PGO rebuild+train+use")
|
||||
add_common(p)
|
||||
p.add_argument("--build-dir", required=True)
|
||||
p.add_argument("--target", default="psx-runtime")
|
||||
p.add_argument("--exe-basename", default="")
|
||||
p.add_argument("--disc", default="")
|
||||
p.add_argument("--train-secs", type=int, default=0)
|
||||
p.add_argument("--train-runs", type=int, default=0)
|
||||
p.add_argument("--pgo-mute", action="store_true")
|
||||
p.add_argument("--pgo-audio", action="store_true")
|
||||
p.add_argument("--pgo-hide-video", action="store_true")
|
||||
p.add_argument("--pgo-video", action="store_true")
|
||||
p.add_argument("--cmake-extra", action="append", default=[])
|
||||
p.add_argument("--no-pgo", action="store_false", default=False)
|
||||
p.add_argument("--force-pgo", action="store_true", default=True)
|
||||
p.set_defaults(handler=cmd_pgo_train)
|
||||
|
||||
return ap
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
progress = ProgressReporter(json_progress=bool(getattr(args, "json_progress", False)))
|
||||
try:
|
||||
return int(args.handler(args, progress))
|
||||
except BrokenPipeError:
|
||||
return EXIT_ERROR
|
||||
except KeyboardInterrupt:
|
||||
progress.error("interrupted", code=EXIT_ERROR)
|
||||
return EXIT_ERROR
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,575 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Normalize a PS1 disc dump to MODE2/2352 .bin/.cue for psxrecomp games.
|
||||
|
||||
Framework tool — game identity comes from ``game.toml`` (``[game]`` + optional
|
||||
``[prepare_disc]``) or CLI flags.
|
||||
|
||||
Accepted inputs:
|
||||
* Redump / raw MODE2/2352 ``.bin`` (+ ``.cue``)
|
||||
* Cooked 2048-byte/sector ``.iso`` (RomM-style libraries)
|
||||
* 2448-byte/sector raw dumps (MotK-style): trim to 2352
|
||||
|
||||
Writes working ``.bin`` + ``.cue``, extracts ``SYSTEM.CNF`` + boot EXE, and
|
||||
prints ``RESULT_CUE=<abs path>`` for the first-run wizard / RetComM.
|
||||
|
||||
ISO→2352 sets Mode2 Form1 sync/header/subheader/EDC (ECC zeroed — fine for
|
||||
software readers). Rebuilt images are not bit-identical to Redump.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
DST_SEC = 2352
|
||||
SRC_2448 = 2448
|
||||
USER = 2048
|
||||
USER_OFF = 24
|
||||
SYNC = bytes([0x00] + [0xFF] * 10 + [0x00])
|
||||
|
||||
|
||||
def _build_edc_table() -> list[int]:
|
||||
table = []
|
||||
for i in range(256):
|
||||
x = i
|
||||
for _ in range(8):
|
||||
x = (x >> 1) ^ 0xD8018001 if (x & 1) else (x >> 1)
|
||||
table.append(x & 0xFFFFFFFF)
|
||||
return table
|
||||
|
||||
|
||||
_EDC_TABLE = _build_edc_table()
|
||||
|
||||
|
||||
def _adjust_edc(buf: bytearray, start: int, length: int) -> None:
|
||||
x = 0
|
||||
end = start + length
|
||||
for i in range(start, end):
|
||||
x = _EDC_TABLE[(x ^ buf[i]) & 0xFF] ^ (x >> 8)
|
||||
struct.pack_into("<I", buf, end, x & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def encode_mode2_form1(user: bytes, lba: int, submode: int = 0x08) -> bytes:
|
||||
if len(user) != USER:
|
||||
raise ValueError("user data must be 2048 bytes")
|
||||
sector = bytearray(DST_SEC)
|
||||
sector[0:12] = SYNC
|
||||
t = lba + 150
|
||||
mm, ss, ff = t // (75 * 60), (t // 75) % 60, t % 75
|
||||
sector[12] = ((mm // 10) << 4) | (mm % 10)
|
||||
sector[13] = ((ss // 10) << 4) | (ss % 10)
|
||||
sector[14] = ((ff // 10) << 4) | (ff % 10)
|
||||
sector[15] = 0x02
|
||||
sub = bytes([0x00, 0x00, submode & 0xFF, 0x00])
|
||||
sector[16:20] = sub
|
||||
sector[20:24] = sub
|
||||
sector[24 : 24 + USER] = user
|
||||
_adjust_edc(sector, 0x10, 0x800 + 8)
|
||||
return bytes(sector)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnownImage:
|
||||
size: int = 0
|
||||
md5: str = ""
|
||||
sha1: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrepareConfig:
|
||||
project_root: Path
|
||||
out_dir: Path
|
||||
bin_name: str
|
||||
cue_name: str
|
||||
boot_exe: str
|
||||
serial: str = ""
|
||||
known: list[KnownImage] = field(default_factory=list)
|
||||
skip_hash_check: bool = False
|
||||
|
||||
|
||||
def file_hashes(path: Path) -> tuple[str, str, int]:
|
||||
h_md5 = hashlib.md5()
|
||||
h_sha1 = hashlib.sha1()
|
||||
size = 0
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
h_md5.update(chunk)
|
||||
h_sha1.update(chunk)
|
||||
return h_md5.hexdigest(), h_sha1.hexdigest(), size
|
||||
|
||||
|
||||
def _parse_array_items(inner: str) -> list[object]:
|
||||
items: list[object] = []
|
||||
if not inner.strip():
|
||||
return items
|
||||
for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner):
|
||||
part = part.strip().rstrip(",").strip()
|
||||
if not part:
|
||||
continue
|
||||
if part.startswith('"') and part.endswith('"'):
|
||||
items.append(part[1:-1])
|
||||
else:
|
||||
try:
|
||||
items.append(int(part, 0))
|
||||
except ValueError:
|
||||
items.append(part)
|
||||
return items
|
||||
|
||||
|
||||
def _parse_toml_simple(text: str) -> dict[str, dict[str, str]]:
|
||||
"""Minimal TOML reader for flat string/int/bool + string/int arrays.
|
||||
|
||||
Supports multi-line arrays (``key = [ … ]`` spanning lines).
|
||||
"""
|
||||
sections: dict[str, dict[str, object]] = {"": {}}
|
||||
cur = ""
|
||||
array_key: str | None = None
|
||||
array_buf: list[str] = []
|
||||
for raw in text.splitlines():
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if array_key is not None:
|
||||
array_buf.append(line)
|
||||
joined = " ".join(array_buf)
|
||||
if "]" in joined:
|
||||
inner = joined.split("]", 1)[0]
|
||||
# Drop leading "[" already consumed when array opened
|
||||
if inner.startswith("["):
|
||||
inner = inner[1:]
|
||||
sections[cur][array_key] = _parse_array_items(inner)
|
||||
array_key = None
|
||||
array_buf = []
|
||||
continue
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
cur = line[1:-1].strip()
|
||||
sections.setdefault(cur, {})
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, val = line.split("=", 1)
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if val.startswith("["):
|
||||
if val.endswith("]") and val.count("[") == val.count("]"):
|
||||
sections[cur][key] = _parse_array_items(val[1:-1])
|
||||
else:
|
||||
array_key = key
|
||||
array_buf = [val[1:]] # after opening [
|
||||
continue
|
||||
if val.startswith('"') and val.endswith('"'):
|
||||
sections[cur][key] = val[1:-1]
|
||||
elif val.lower() in ("true", "false"):
|
||||
sections[cur][key] = val.lower() == "true"
|
||||
else:
|
||||
try:
|
||||
sections[cur][key] = int(val, 0)
|
||||
except ValueError:
|
||||
sections[cur][key] = val
|
||||
return sections # type: ignore[return-value]
|
||||
|
||||
|
||||
def load_config(project_root: Path, config_path: Path | None, out_dir_cli: str | None,
|
||||
skip_hash: bool) -> PrepareConfig:
|
||||
project_root = project_root.resolve()
|
||||
game: dict = {}
|
||||
prep: dict = {}
|
||||
if config_path and config_path.is_file():
|
||||
sections = _parse_toml_simple(config_path.read_text(encoding="utf-8"))
|
||||
game = sections.get("game", {}) or {}
|
||||
prep = sections.get("prepare_disc", {}) or {}
|
||||
|
||||
boot = str(prep.get("boot_exe") or "")
|
||||
if not boot:
|
||||
exe = str(game.get("exe") or "")
|
||||
boot = Path(exe).name if exe else ""
|
||||
if not boot:
|
||||
boot = "SLUS_000.00"
|
||||
|
||||
out_rel = str(prep.get("out_dir") or "prepared_disc")
|
||||
out_dir = Path(out_dir_cli).expanduser() if out_dir_cli else (project_root / out_rel)
|
||||
|
||||
disc = str(game.get("disc") or prep.get("bin_name") or "game.bin")
|
||||
bin_name = str(prep.get("bin_name") or Path(disc).name)
|
||||
cue_name = str(prep.get("cue_name") or (Path(bin_name).stem + ".cue"))
|
||||
|
||||
known: list[KnownImage] = []
|
||||
sizes = prep.get("known_sizes") or []
|
||||
md5s = prep.get("known_md5") or []
|
||||
sha1s = prep.get("known_sha1") or []
|
||||
if isinstance(sizes, list) or isinstance(md5s, list) or isinstance(sha1s, list):
|
||||
n = max(
|
||||
len(sizes) if isinstance(sizes, list) else 0,
|
||||
len(md5s) if isinstance(md5s, list) else 0,
|
||||
len(sha1s) if isinstance(sha1s, list) else 0,
|
||||
)
|
||||
for i in range(n):
|
||||
known.append(
|
||||
KnownImage(
|
||||
size=int(sizes[i]) if isinstance(sizes, list) and i < len(sizes) else 0,
|
||||
md5=str(md5s[i]).lower()
|
||||
if isinstance(md5s, list) and i < len(md5s)
|
||||
else "",
|
||||
sha1=str(sha1s[i]).lower()
|
||||
if isinstance(sha1s, list) and i < len(sha1s)
|
||||
else "",
|
||||
)
|
||||
)
|
||||
|
||||
return PrepareConfig(
|
||||
project_root=project_root,
|
||||
out_dir=out_dir.resolve(),
|
||||
bin_name=bin_name,
|
||||
cue_name=cue_name,
|
||||
boot_exe=boot,
|
||||
serial=str(game.get("id") or ""),
|
||||
known=known,
|
||||
skip_hash_check=skip_hash,
|
||||
)
|
||||
|
||||
|
||||
def resolve_cue_bin(cue_path: Path) -> Path:
|
||||
text = cue_path.read_text(encoding="utf-8", errors="replace")
|
||||
m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I)
|
||||
if not m:
|
||||
m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I)
|
||||
if not m:
|
||||
raise SystemExit(f"no BINARY FILE in cue: {cue_path}")
|
||||
cand = Path(m.group(1))
|
||||
if not cand.is_absolute():
|
||||
cand = cue_path.parent / cand
|
||||
if not cand.is_file():
|
||||
raise SystemExit(f"cue references missing bin: {cand}")
|
||||
return cand
|
||||
|
||||
|
||||
def detect_kind(path: Path, size: int) -> str:
|
||||
suf = path.suffix.lower()
|
||||
if suf == ".cue":
|
||||
return "cue"
|
||||
if suf == ".iso":
|
||||
return "iso"
|
||||
# MotK-style 2448 before generic 2352 (some sizes divisible by both).
|
||||
if size % SRC_2448 == 0 and size > SRC_2448 and size % DST_SEC != 0:
|
||||
return "raw2448"
|
||||
if size % SRC_2448 == 0 and size > SRC_2448:
|
||||
# Peek: if sync looks like CD at +0, treat as 2448 when user data path works.
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(16)
|
||||
if head[:12] == SYNC:
|
||||
# Could be 2352 or 2448; check sector stride via second sync.
|
||||
fpos_2352_ok = False
|
||||
fpos_2448_ok = False
|
||||
with open(path, "rb") as f:
|
||||
f.seek(DST_SEC)
|
||||
fpos_2352_ok = f.read(12) == SYNC
|
||||
f.seek(SRC_2448)
|
||||
fpos_2448_ok = f.read(12) == SYNC
|
||||
if fpos_2448_ok and not fpos_2352_ok:
|
||||
return "raw2448"
|
||||
if size % DST_SEC == 0 and size > DST_SEC:
|
||||
with open(path, "rb") as f:
|
||||
sec0 = f.read(12)
|
||||
f.seek(16 * DST_SEC)
|
||||
sync16 = f.read(12)
|
||||
if sec0 == SYNC or sync16 == SYNC:
|
||||
return "bin2352"
|
||||
if size % USER == 0 and size > USER:
|
||||
return "iso"
|
||||
if size % SRC_2448 == 0 and size > SRC_2448:
|
||||
return "raw2448"
|
||||
raise SystemExit(
|
||||
f"unrecognized disc image {path} (size={size}). "
|
||||
"Need MODE2/2352 .bin/.cue, cooked 2048 .iso, or 2448-byte/sector dump."
|
||||
)
|
||||
|
||||
|
||||
def read_user_bin(data: bytes, lba: int) -> bytes:
|
||||
off = lba * DST_SEC
|
||||
if off + DST_SEC > len(data):
|
||||
raise KeyError(lba)
|
||||
if data[off : off + 12] != SYNC:
|
||||
raise SystemExit(f"bad sync at LBA {lba}")
|
||||
return data[off + USER_OFF : off + USER_OFF + USER]
|
||||
|
||||
|
||||
def read_user_iso(data: bytes, lba: int) -> bytes:
|
||||
off = lba * USER
|
||||
if off + USER > len(data):
|
||||
raise KeyError(lba)
|
||||
return data[off : off + USER]
|
||||
|
||||
|
||||
def parse_root_entries(root: bytes) -> dict[str, tuple[int, int]]:
|
||||
entries: dict[str, tuple[int, int]] = {}
|
||||
i = 0
|
||||
while i < len(root):
|
||||
reclen = root[i]
|
||||
if reclen == 0:
|
||||
i = ((i // USER) + 1) * USER
|
||||
if i >= len(root):
|
||||
break
|
||||
continue
|
||||
if i + reclen > len(root):
|
||||
break
|
||||
extent = struct.unpack_from("<I", root, i + 2)[0]
|
||||
size = struct.unpack_from("<I", root, i + 10)[0]
|
||||
namelen = root[i + 32]
|
||||
name = root[i + 33 : i + 33 + namelen]
|
||||
if b";" in name:
|
||||
name = name.split(b";")[0]
|
||||
if name not in (b"\x00", b"\x01"):
|
||||
entries[name.decode("ascii", "replace")] = (extent, size)
|
||||
i += reclen
|
||||
return entries
|
||||
|
||||
|
||||
def extract_via(
|
||||
read_user, data: bytes, boot_exe: str
|
||||
) -> tuple[dict[str, tuple[int, int]], dict[str, bytes]]:
|
||||
pvd = read_user(data, 16)
|
||||
if pvd[1:6] != b"CD001":
|
||||
raise SystemExit("PVD not found at LBA 16 (expected CD001)")
|
||||
root_extent = struct.unpack_from("<I", pvd, 158)[0]
|
||||
root_size = struct.unpack_from("<I", pvd, 166)[0]
|
||||
root = bytearray()
|
||||
for i in range((root_size + USER - 1) // USER):
|
||||
root += read_user(data, root_extent + i)
|
||||
root = bytes(root[:root_size])
|
||||
entries = parse_root_entries(root)
|
||||
files: dict[str, bytes] = {}
|
||||
for need in ("SYSTEM.CNF", boot_exe):
|
||||
if need not in entries:
|
||||
raise SystemExit(f"missing {need} on disc (found {sorted(entries)[:20]})")
|
||||
extent, size = entries[need]
|
||||
out = bytearray()
|
||||
rem, lba = size, extent
|
||||
while rem > 0:
|
||||
sector = read_user(data, lba)
|
||||
take = min(USER, rem)
|
||||
out += sector[:take]
|
||||
rem -= take
|
||||
lba += 1
|
||||
files[need] = bytes(out)
|
||||
if files[boot_exe][:8] != b"PS-X EXE":
|
||||
raise SystemExit(f"{boot_exe} is not a PS-X EXE")
|
||||
return entries, files
|
||||
|
||||
|
||||
def iso_to_bin(iso_data: bytes) -> bytes:
|
||||
if len(iso_data) % USER != 0:
|
||||
raise SystemExit(f"ISO size {len(iso_data)} is not a multiple of 2048")
|
||||
n = len(iso_data) // USER
|
||||
out = bytearray()
|
||||
for lba in range(n):
|
||||
user = iso_data[lba * USER : (lba + 1) * USER]
|
||||
out += encode_mode2_form1(user, lba, submode=0x08)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def raw2448_to_bin(src: Path) -> bytes:
|
||||
out = bytearray()
|
||||
n = 0
|
||||
with open(src, "rb") as inf:
|
||||
while True:
|
||||
sec = inf.read(SRC_2448)
|
||||
if not sec:
|
||||
break
|
||||
if len(sec) != SRC_2448:
|
||||
raise SystemExit(f"truncated 2448 sector {n}: got {len(sec)} bytes")
|
||||
out += sec[:DST_SEC]
|
||||
n += 1
|
||||
print(f" trimmed {n} sectors 2448 → 2352")
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def matches_known(cfg: PrepareConfig, size: int, md5: str, sha1: str) -> bool:
|
||||
if not cfg.known:
|
||||
return False
|
||||
md5 = md5.lower()
|
||||
sha1 = sha1.lower()
|
||||
for k in cfg.known:
|
||||
if k.md5 and k.md5.lower() == md5:
|
||||
if not k.size or k.size == size:
|
||||
return True
|
||||
if k.sha1 and k.sha1.lower() == sha1:
|
||||
if not k.size or k.size == size:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
here = Path(__file__).resolve().parent
|
||||
# tools/ → psxrecomp/ → often game root is parent of psxrecomp
|
||||
fw_root = here.parent
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("source", nargs="?", default="", help="path to .iso / .cue / .bin")
|
||||
ap.add_argument(
|
||||
"--config",
|
||||
default="",
|
||||
help="game.toml (reads [game] + [prepare_disc])",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--project-root",
|
||||
default="",
|
||||
help="game project root (default: parent of config, else cwd)",
|
||||
)
|
||||
ap.add_argument("--out-dir", default="", help="override prepare_disc.out_dir")
|
||||
ap.add_argument("--bin-name", default="", help="override output .bin basename")
|
||||
ap.add_argument("--cue-name", default="", help="override output .cue basename")
|
||||
ap.add_argument("--boot-exe", default="", help="override boot EXE name on disc")
|
||||
ap.add_argument(
|
||||
"--skip-hash-check",
|
||||
action="store_true",
|
||||
help="do not require prepare_disc.known_* digests",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
config_path = Path(args.config).expanduser() if args.config else None
|
||||
if config_path and not config_path.is_file():
|
||||
print(f"config not found: {config_path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.project_root:
|
||||
project_root = Path(args.project_root).expanduser().resolve()
|
||||
elif config_path:
|
||||
project_root = config_path.parent.resolve()
|
||||
else:
|
||||
# Invoked from a game tree: prefer cwd if it has game.toml
|
||||
cwd = Path.cwd()
|
||||
project_root = cwd if (cwd / "game.toml").is_file() else fw_root.parent
|
||||
|
||||
if not config_path:
|
||||
cand = project_root / "game.toml"
|
||||
if cand.is_file():
|
||||
config_path = cand
|
||||
|
||||
cfg = load_config(project_root, config_path, args.out_dir or None, args.skip_hash_check)
|
||||
if args.bin_name:
|
||||
cfg.bin_name = args.bin_name
|
||||
if args.cue_name:
|
||||
cfg.cue_name = args.cue_name
|
||||
if args.boot_exe:
|
||||
cfg.boot_exe = args.boot_exe
|
||||
|
||||
if not args.source:
|
||||
# Convenience: existing working bin under out_dir
|
||||
cand = cfg.out_dir / cfg.bin_name
|
||||
if cand.is_file():
|
||||
args.source = str(cand)
|
||||
else:
|
||||
print("source required (.iso / .cue / .bin)", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
src = Path(args.source).expanduser().resolve()
|
||||
if not src.is_file():
|
||||
print(f"source not found: {src}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
cfg.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
src_md5, src_sha1, src_size = file_hashes(src)
|
||||
print(f"source: {src}")
|
||||
print(f" size {src_size}")
|
||||
print(f" md5 {src_md5}")
|
||||
print(f" sha1 {src_sha1}")
|
||||
if cfg.serial:
|
||||
print(f" game {cfg.serial} boot={cfg.boot_exe}")
|
||||
|
||||
kind = detect_kind(src, src_size)
|
||||
if kind == "cue":
|
||||
src = resolve_cue_bin(src)
|
||||
src_md5, src_sha1, src_size = file_hashes(src)
|
||||
print(f"cue → bin: {src}")
|
||||
print(f" size {src_size}")
|
||||
print(f" md5 {src_md5}")
|
||||
print(f" sha1 {src_sha1}")
|
||||
kind = detect_kind(src, src_size)
|
||||
|
||||
known_hit = matches_known(cfg, src_size, src_md5, src_sha1)
|
||||
if cfg.known and not cfg.skip_hash_check:
|
||||
if known_hit:
|
||||
print(" known digest OK")
|
||||
elif kind in ("bin2352", "raw2448"):
|
||||
print(
|
||||
"source digests are not in prepare_disc.known_* "
|
||||
"(pass --skip-hash-check to force)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
else:
|
||||
print(
|
||||
"ISO digests are not in prepare_disc.known_*; "
|
||||
"verifying boot EXE only."
|
||||
)
|
||||
elif not cfg.known and not cfg.skip_hash_check and kind == "bin2352":
|
||||
print(" no prepare_disc.known_* configured — verifying boot EXE only")
|
||||
|
||||
if kind == "bin2352":
|
||||
bin_data = src.read_bytes()
|
||||
entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe)
|
||||
elif kind == "iso":
|
||||
iso_data = src.read_bytes()
|
||||
entries, files = extract_via(read_user_iso, iso_data, cfg.boot_exe)
|
||||
print(f" converting {len(iso_data) // USER} sectors ISO → MODE2/2352…")
|
||||
bin_data = iso_to_bin(iso_data)
|
||||
extract_via(read_user_bin, bin_data, cfg.boot_exe)
|
||||
elif kind == "raw2448":
|
||||
print(" converting 2448 → 2352…")
|
||||
bin_data = raw2448_to_bin(src)
|
||||
entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe)
|
||||
else:
|
||||
print(f"unsupported kind {kind}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f" root entries: {sorted(entries)[:24]}")
|
||||
pc0 = struct.unpack_from("<I", files[cfg.boot_exe], 0x10)[0]
|
||||
print(f" {cfg.boot_exe} PC0={pc0:#010x} ({len(files[cfg.boot_exe])} bytes)")
|
||||
|
||||
for name, blob in files.items():
|
||||
out_path = cfg.out_dir / name
|
||||
out_path.write_bytes(blob)
|
||||
print(f"wrote {out_path} ({len(blob)} bytes)")
|
||||
|
||||
bin_path = cfg.out_dir / cfg.bin_name
|
||||
if src.resolve() == bin_path.resolve() and kind == "bin2352":
|
||||
print(f"source already at {bin_path}")
|
||||
else:
|
||||
print(f"writing {bin_path}")
|
||||
bin_path.write_bytes(bin_data)
|
||||
|
||||
cue_path = cfg.out_dir / cfg.cue_name
|
||||
cue_path.write_text(
|
||||
f'FILE "{cfg.bin_name}" BINARY\n'
|
||||
f" TRACK 01 MODE2/2352\n"
|
||||
f" INDEX 01 00:00:00\n",
|
||||
encoding="ascii",
|
||||
newline="\n",
|
||||
)
|
||||
print(f"wrote {cue_path}")
|
||||
|
||||
out_md5, out_sha1, out_size = file_hashes(bin_path)
|
||||
print("working bin:")
|
||||
print(f" size {out_size}")
|
||||
print(f" md5 {out_md5}")
|
||||
print(f" sha1 {out_sha1}")
|
||||
if kind == "iso":
|
||||
print(" rebuilt from ISO (EDC set; ECC zeroed — fine for software readers)")
|
||||
elif kind == "raw2448":
|
||||
print(" trimmed from 2448-byte/sector dump")
|
||||
|
||||
print(f"RESULT_CUE={cue_path.resolve()}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
"""JSONL progress events for headless psxrecomp SDK consumers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Mapping, Optional, TextIO
|
||||
|
||||
|
||||
class ProgressReporter:
|
||||
"""Emit coarse phase events for UIs and launchers.
|
||||
|
||||
When ``json_progress`` is true, stdout is reserved for one JSON object
|
||||
per line. Human-readable text goes to stderr (or is mirrored as
|
||||
``log`` events when ``mirror_logs`` is true).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
json_progress: bool = False,
|
||||
stream: Optional[TextIO] = None,
|
||||
log_stream: Optional[TextIO] = None,
|
||||
mirror_logs: bool = True,
|
||||
) -> None:
|
||||
self.json_progress = bool(json_progress)
|
||||
self.stream = stream if stream is not None else sys.stdout
|
||||
self.log_stream = log_stream if log_stream is not None else sys.stderr
|
||||
self.mirror_logs = bool(mirror_logs)
|
||||
self._started = time.perf_counter()
|
||||
|
||||
def _emit(self, payload: Mapping[str, Any]) -> None:
|
||||
if not self.json_progress:
|
||||
return
|
||||
line = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||
self.stream.write(line + "\n")
|
||||
self.stream.flush()
|
||||
|
||||
def event(self, event: str, **fields: Any) -> None:
|
||||
payload: dict[str, Any] = {
|
||||
"event": event,
|
||||
"t": round(time.perf_counter() - self._started, 3),
|
||||
}
|
||||
for key, value in fields.items():
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
self._emit(payload)
|
||||
|
||||
def phase(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
pct: Optional[float] = None,
|
||||
message: Optional[str] = None,
|
||||
**fields: Any,
|
||||
) -> None:
|
||||
self.event("phase", phase=name, pct=pct, message=message, **fields)
|
||||
if message and not self.json_progress:
|
||||
print(message, file=self.log_stream, flush=True)
|
||||
|
||||
def log(self, message: str, *, level: str = "info") -> None:
|
||||
if self.json_progress:
|
||||
self.event("log", level=level, message=message)
|
||||
if self.mirror_logs:
|
||||
print(message, file=self.log_stream, flush=True)
|
||||
else:
|
||||
print(message, file=self.log_stream, flush=True)
|
||||
|
||||
def result(self, **fields: Any) -> None:
|
||||
self.event("result", **fields)
|
||||
|
||||
def error(self, message: str, *, code: int = 1, **fields: Any) -> None:
|
||||
self.event("error", message=message, code=code, **fields)
|
||||
if not self.json_progress:
|
||||
print(message, file=self.log_stream, flush=True)
|
||||
Reference in New Issue
Block a user