diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 019961a2..b6af4f95 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -129,7 +129,11 @@ The backend tier is resolved in `overlay_backend.c` (see `main.cpp` around the Compiled overlays are stored in a content-addressed cache namespaced by compiler and target ABI (`/gcc//…` vs `/tcc/…`); a `gcc` -shard wins over a `tcc` shard for the same region. See +shard wins over a `tcc` shard for the same region. The `` is the +runtime's own (`PSX_OVERLAY_ARCH_ABI` in `overlay_loader.h`, the running slice +of a universal macOS binary): the runtime exports it to every compile it spawns +and macOS compiles pass the matching `-arch`, so an x86_64 runtime under Rosetta +gets x86_64 shards even though Xcode's clang defaults to arm64. See [`docs/FEATURES.md`](FEATURES.md), [`docs/OVERLAY_CACHE_V2.md`](OVERLAY_CACHE_V2.md), and [`docs/ASYNC_OVERLAY_COMPILE.md`](ASYNC_OVERLAY_COMPILE.md). diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index b1c969fa..cbcc1c49 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -1064,6 +1064,17 @@ if(BUILD_TESTING) add_test(NAME release_stage_cache_test COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_release_stage_cache.py) + # Overlay shards and BIOS modules are compiled for the RUNTIME's + # architecture: an x86_64 macOS runtime under Rosetta used to get arm64 + # shards (arm64-only Xcode clang, no -arch) in its macos-x64 cache and + # ran every overlay interpreted. Pins the PSX_OVERLAY_ARCH_ABI handoff + # (autocompile.c), -arch on macOS, the refusal when the Python cannot + # load the runtime's architecture, and unchanged Windows/Linux commands; + # with a real cc it builds and loads each slice this Mac can run. + add_test(NAME overlay_compile_arch_test + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_compile_arch.py + --cc ${CMAKE_C_COMPILER}) endif() # Full card protocol regression: read, write, address echo, per-slot state, diff --git a/runtime/include/overlay_loader.h b/runtime/include/overlay_loader.h index ed954d4c..e9300554 100644 --- a/runtime/include/overlay_loader.h +++ b/runtime/include/overlay_loader.h @@ -45,6 +45,34 @@ static inline int psx_overlay_code_segment_pc(uint32_t pc) { struct OverlayCallbacks; const struct OverlayCallbacks *overlay_loader_callbacks(void); +/* Canonical cache arch-abi tag (caches are namespaced per backend AND per + * target so a Windows-x64 gcc DLL and, later, a same-OS arm64 build for the + * same fragment never comingle). compile_overlays.py + * computes the IDENTICAL string; keep the two mappings in lockstep + * ("-": win|linux|macos + x64|arm64|x86). It is this build's own + * architecture -- the running slice of a universal macOS binary -- and the + * runtime exports it as PSX_OVERLAY_ARCH_ABI to every compile it spawns + * (autocompile.c) and passes it to bios_module_build.py, so shards and BIOS + * modules are compiled for the architecture that loads them, not for the + * architecture of the Python or the compiler binary. */ +#if defined(_WIN32) +# define PSX_OL_OS "win" +#elif defined(__APPLE__) +# define PSX_OL_OS "macos" +#else +# define PSX_OL_OS "linux" +#endif +#if defined(__aarch64__) || defined(_M_ARM64) +# define PSX_OL_ARCH "arm64" +#elif defined(__x86_64__) || defined(_M_X64) +# define PSX_OL_ARCH "x64" +#elif defined(__i386__) || defined(_M_IX86) +# define PSX_OL_ARCH "x86" +#else +# define PSX_OL_ARCH "unknown" +#endif +#define PSX_OVERLAY_ARCH_ABI PSX_OL_OS "-" PSX_OL_ARCH + /* "-" of this build (PSX_OVERLAY_ARCH_ABI), the cache layout's * arch-abi segment. Also keyed into the BIOS module cache. */ const char *overlay_loader_arch_abi(void); diff --git a/runtime/src/autocompile.c b/runtime/src/autocompile.c index ca31a29d..2dad0311 100644 --- a/runtime/src/autocompile.c +++ b/runtime/src/autocompile.c @@ -982,9 +982,9 @@ void autocompile_configure(const char *cmd, const char *cwd) { } #else s_out_lock_init = 1; - /* Pin WRITE cache + READ captures + flavor to the loader's canonical - * values in the environment every child inherits: same contract as the - * SetEnvironmentVariableA block in the Windows request path (see it for + /* Pin WRITE cache + READ captures + flavor + arch-abi to the loader's + * canonical values in the environment every child inherits: same contract + * as the SetEnvironmentVariableA block in the Windows request path (see it for * why each matters). Done once here, on the emulation thread before any * worker exists, because glibc's setenv is not safe against a concurrent * getenv on another thread. autocompile_set_cache_paths() runs first. */ @@ -996,6 +996,10 @@ void autocompile_configure(const char *cmd, const char *cwd) { snprintf(flavor_buf, sizeof flavor_buf, "%d", (int)PSX_OVERLAY_FLAVOR); setenv("PSX_OVERLAY_FLAVOR", flavor_buf, 1); } + /* The architecture too (see the Windows block): Xcode's clang is an + * arm64-only binary and defaults to arm64, so an x86_64 runtime under + * Rosetta used to get arm64 shards it could never load. */ + setenv("PSX_OVERLAY_ARCH_ABI", PSX_OVERLAY_ARCH_ABI, 1); if (s_cmd[0]) autocompile_report_interpreter(); #endif } @@ -1100,6 +1104,12 @@ int autocompile_request(void) { snprintf(flavor_buf, sizeof flavor_buf, "%d", (int)PSX_OVERLAY_FLAVOR); SetEnvironmentVariableA("PSX_OVERLAY_FLAVOR", flavor_buf); } + /* And the ARCHITECTURE: shards must be this runtime's "-" + * (overlay_loader.h), whatever the Python or the compiler binary runs as. + * On Windows the bundled Python, MinGW gcc and tcc are all x64, so this + * names what the tool derives anyway; on macOS it is what makes an x64 + * runtime under Rosetta get x86_64 shards from an arm64-only clang. */ + SetEnvironmentVariableA("PSX_OVERLAY_ARCH_ABI", PSX_OVERLAY_ARCH_ABI); /* cmd.exe /C resolves the command via PATH and supports the relative * paths in the configured line (cwd = project root). The WHOLE command is diff --git a/runtime/src/overlay_loader.c b/runtime/src/overlay_loader.c index 4edd8c0d..507e15cb 100644 --- a/runtime/src/overlay_loader.c +++ b/runtime/src/overlay_loader.c @@ -1830,29 +1830,8 @@ static void refresh_bios_resident_flags(void) { cache_path_is_bios_resident(s_cache_idx[i].path); } -/* Canonical cache arch-abi tag (caches are namespaced per backend AND per - * target so a Windows-x64 gcc DLL and, later, a same-OS arm64 build for the - * same fragment never comingle). compile_overlays.py - * computes the IDENTICAL string from platform.system()/machine(); keep the two - * mappings in lockstep ("-": win|linux|macos + x64|arm64|x86). */ -#if defined(_WIN32) -# define PSX_OL_OS "win" -#elif defined(__APPLE__) -# define PSX_OL_OS "macos" -#else -# define PSX_OL_OS "linux" -#endif -#if defined(__aarch64__) || defined(_M_ARM64) -# define PSX_OL_ARCH "arm64" -#elif defined(__x86_64__) || defined(_M_X64) -# define PSX_OL_ARCH "x64" -#elif defined(__i386__) || defined(_M_IX86) -# define PSX_OL_ARCH "x86" -#else -# define PSX_OL_ARCH "unknown" -#endif -#define PSX_OVERLAY_ARCH_ABI PSX_OL_OS "-" PSX_OL_ARCH - +/* PSX_OVERLAY_ARCH_ABI (the cache layout's "-") is defined in + * overlay_loader.h, so autocompile.c can hand it to the compile it spawns. */ const char *overlay_loader_arch_abi(void) { return PSX_OVERLAY_ARCH_ABI; } #ifndef _WIN32 diff --git a/runtime/src/psx_bios_module.c b/runtime/src/psx_bios_module.c index 765111ff..acd0990c 100644 --- a/runtime/src/psx_bios_module.c +++ b/runtime/src/psx_bios_module.c @@ -462,6 +462,10 @@ static int build_module(const BmToolchain *tk, const char *dump_path, !append_s(cmd, sizeof(cmd), " --stem ") || !append_s(cmd, sizeof(cmd), stem) || !append_s(cmd, sizeof(cmd), " --out ") || !append_q(cmd, sizeof(cmd), out_path) || !append_s(cmd, sizeof(cmd), " --flavor ") || !append_s(cmd, sizeof(cmd), flavor) || + /* This build's architecture, not the Python's or the compiler's (an + * x86_64 runtime under Rosetta, arm64-only Xcode clang). */ + !append_s(cmd, sizeof(cmd), " --arch-abi ") || + !append_s(cmd, sizeof(cmd), overlay_loader_arch_abi()) || !append_s(cmd, sizeof(cmd), " --compiler ") || !append_s(cmd, sizeof(cmd), compiler)) { set_err(err, err_cap, "build command too long"); return 0; diff --git a/runtime/tests/test_overlay_compile_arch.py b/runtime/tests/test_overlay_compile_arch.py new file mode 100644 index 00000000..a31bd27c --- /dev/null +++ b/runtime/tests/test_overlay_compile_arch.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""Overlay shards and BIOS modules are compiled for the RUNTIME's architecture. + +THE BUG +======= +An x86_64 macOS runtime running under Rosetta compiled every overlay shard for +arm64. The chain was: the runtime spawns `sh -c `; the shell +and the bundled Python inherit x86_64 from it, so compile_overlays.py named the +cache `macos-x64` (platform.machine() is the process's own architecture). But +it ran the compiler without `-arch`, and Xcode's clang is an arm64-only binary +whose default target is its own architecture. So arm64 shards landed in the +macos-x64 cache, the x64 runtime could not load one of them, and every overlay +ran on the interpreter. Nothing failed loudly. bios_module_build.py had the +same compile line. + +THE CONTRACT PINNED HERE +======================== +* The runtime exports its own "-" (overlay_loader.h's + PSX_OVERLAY_ARCH_ABI: the running slice, for a universal binary) to every + compile it spawns, as PSX_OVERLAY_ARCH_ABI, and passes it to + bios_module_build.py as --arch-abi. +* compile_overlays.py builds for that architecture: the cache directory names + it, and on macOS the compiler is told it with `-arch`. +* Shards are validated by loading them into the Python that built them, so a + Python that cannot load the runtime's architecture refuses up front instead + of writing shards nobody loads. The Python's architecture is its process's: + platform.machine() on macOS and Linux, but the interpreter's build on + Windows, where platform.machine() reports the machine (an x64 Python + emulated on Windows on ARM says ARM64; it must still build x64 shards). +* Windows and Linux compile exactly as before: no new flags, the same + directory. + +The hermetic cases run everywhere. The real-toolchain cases need a POSIX C +compiler (--cc); the per-slice cases need macOS, and the x86_64 slice needs +Rosetta on Apple silicon. + + python3 runtime/tests/test_overlay_compile_arch.py [--cc cc] + ctest -R overlay_compile_arch_test +""" + +import argparse +import contextlib +import io +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, '..', '..')) +TOOLS = os.path.join(ROOT, 'tools') +INCLUDE = os.path.join(ROOT, 'runtime', 'include') +sys.path.insert(0, TOOLS) + +import compile_overlays as co # noqa: E402 +import bios_module_build as bmb # noqa: E402 + +CC = 'cc' # replaced from --cc in __main__ + + +# What sysconfig.get_platform() says for a native Windows Python built for +# the machine it runs on (python.org spelling). +_WIN_BUILD = {'AMD64': 'win-amd64', 'ARM64': 'win-arm64', 'x86': 'win32'} + + +@contextlib.contextmanager +def host(system, machine, env_arch=None, build=None): + """Pretend this interpreter runs on `system`/`machine` (what + platform.system()/machine() report), optionally spawned by a runtime that + exported PSX_OVERLAY_ARCH_ABI=env_arch. On Windows platform.machine() is + the machine's, and `build` is what sysconfig.get_platform() reports for + the interpreter (default: built for that machine).""" + env = {k: v for k, v in os.environ.items() if k != 'PSX_OVERLAY_ARCH_ABI'} + if env_arch is not None: + env['PSX_OVERLAY_ARCH_ABI'] = env_arch + saved_os, saved_arch = co._TARGET_OS, co._TARGET_ARCH + co.set_target_os(None) + co.set_target_arch(None) + if build is None: + build = _WIN_BUILD.get(machine, 'win32') if system == 'Windows' else 'posix-build' + try: + with mock.patch.object(co.sysconfig, 'get_platform', return_value=build), \ + mock.patch.object(co.platform, 'system', return_value=system), \ + mock.patch.object(co.platform, 'machine', return_value=machine), \ + mock.patch.object(co.os, 'name', 'nt' if system == 'Windows' else 'posix'), \ + mock.patch.dict(os.environ, env, clear=True): + yield + finally: + co.set_target_os(saved_os) + co.set_target_arch(saved_arch) + + +class _Ran: + """Records the compiler command instead of running it.""" + def __init__(self): + self.cmds = [] + + def __call__(self, cmd, *a, **kw): + self.cmds.append(list(cmd)) + return subprocess.CompletedProcess(cmd, 0, '', '') + + +_FIXTURE = None + + +def fixture(): + """One shard source and include dir for every recorded command (the tcc + path reads both), so commands from different cases compare equal.""" + global _FIXTURE + if _FIXTURE is None: + root = tempfile.mkdtemp(prefix='psx_ovl_arch_fx_') + os.makedirs(os.path.join(root, 'inc')) + with open(os.path.join(root, 'inc', 'h.h'), 'w') as f: + f.write('\n') + _FIXTURE = root + return _FIXTURE + + +def shard_command(compiler='gcc', gcc='gcc'): + root = fixture() + src = os.path.join(root, 'shard.c') + with open(src, 'w') as f: + f.write('int overlay_abi(void) { return 0; }\n') + ran = _Ran() + with mock.patch.object(co.subprocess, 'run', ran), \ + contextlib.redirect_stdout(io.StringIO()): + co._compile_dll_direct(src, os.path.join(root, 'out.so'), + [os.path.join(root, 'inc')], + gcc=gcc, flavor=0, compiler=compiler, tcc='tcc') + return ran.cmds[-1] + + +def bios_module_command(gcc='gcc'): + ran = _Ran() + with mock.patch.object(bmb.subprocess, 'run', ran), \ + contextlib.redirect_stdout(io.StringIO()): + bmb.compile_module(['/x/a.c'], '/x/m.so', ['/x/inc'], + 'SCPH1001', 0, 'gcc', gcc, 'tcc') + return ran.cmds[-1] + + +def arch_args(cmd): + return [cmd[i + 1] for i, a in enumerate(cmd[:-1]) if a == '-arch'] + + +def run_main_until_guard(argv): + """compile_overlays.main() as the runtime spawns it, up to the arch guard. + Returns (exit code, stdout). The game.toml does not exist on purpose: a + run that gets past the guard fails there instead (FileNotFoundError).""" + out = io.StringIO() + err = io.StringIO() + with mock.patch.object(sys, 'argv', ['compile_overlays.py'] + argv), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + co.main() + except SystemExit as exc: + return exc.code, out.getvalue() + err.getvalue() + except FileNotFoundError: + return 'past-guard', out.getvalue() + err.getvalue() + return 0, out.getvalue() + + +MAIN_ARGV = ['--game-toml', '/nonexistent/game.toml', '--recompiler', '/nonexistent/r', + '--runtime-include', INCLUDE, '--captures', '/nonexistent/c.json'] + + +class ToolTargetsRuntimeArch(unittest.TestCase): + """compile_overlays.py / bios_module_build.py, with the host mocked.""" + + def test_rosetta_x64_runtime_gets_x86_64_shards(self): + # The reported case: x86_64 runtime, x86_64 Python (Rosetta). + with host('Darwin', 'x86_64', 'macos-x64'): + self.assertIsNone(co.apply_runtime_arch_abi( + os.environ['PSX_OVERLAY_ARCH_ABI'], 'PSX_OVERLAY_ARCH_ABI')) + self.assertEqual(co.cache_arch_abi(), 'macos-x64') + self.assertIsNone(co.interpreter_arch_mismatch()) + self.assertEqual(arch_args(shard_command()), ['x86_64']) + self.assertEqual(arch_args(bios_module_command()), ['x86_64']) + + def test_arm64_runtime_gets_arm64_shards(self): + with host('Darwin', 'arm64', 'macos-arm64'): + self.assertIsNone(co.apply_runtime_arch_abi('macos-arm64', 'env')) + self.assertEqual(co.cache_arch_abi(), 'macos-arm64') + self.assertEqual(arch_args(shard_command()), ['arm64']) + self.assertEqual(arch_args(bios_module_command()), ['arm64']) + + def test_manual_run_on_macos_names_the_interpreters_arch(self): + # No runtime pin (offline/AOT use): build for this interpreter, and + # still say so, so an x86_64 Python never gets arm64 output. + with host('Darwin', 'x86_64'): + self.assertEqual(co.cache_arch_abi(), 'macos-x64') + self.assertEqual(arch_args(shard_command()), ['x86_64']) + with host('Darwin', 'arm64'): + self.assertEqual(co.cache_arch_abi(), 'macos-arm64') + self.assertEqual(arch_args(shard_command()), ['arm64']) + + def test_main_adopts_the_runtime_pin(self): + with host('Darwin', 'x86_64', 'macos-x64'): + code, out = run_main_until_guard(MAIN_ARGV) + self.assertEqual(code, 'past-guard', out) + self.assertEqual(co.cache_arch_abi(), 'macos-x64') + # The env wins over --arch-abi, like the cache dir and flavor pins. + with host('Darwin', 'x86_64', 'macos-x64'): + code, out = run_main_until_guard(MAIN_ARGV + ['--arch-abi', 'macos-arm64']) + self.assertEqual(code, 'past-guard', out) + self.assertIn('PSX_OVERLAY_ARCH_ABI overrides --arch-abi', out) + self.assertEqual(co.cache_arch_abi(), 'macos-x64') + + def test_python_that_cannot_load_the_runtimes_arch_refuses(self): + # A universal runtime's arm64 slice spawning an x86_64-only Python: + # the shards could never be validated here, so build none, loudly. + with host('Darwin', 'x86_64', 'macos-arm64'): + code, out = run_main_until_guard(MAIN_ARGV) + self.assertEqual(code, 2, out) + self.assertIn('cannot build overlay shards', out) + self.assertIn('macos-arm64', out) + # --static writes C only and loads nothing: no guard. + with host('Darwin', 'x86_64', 'macos-arm64'): + code, out = run_main_until_guard(MAIN_ARGV + ['--static']) + self.assertEqual(code, 'past-guard', out) + with host('Darwin', 'x86_64', 'macos-arm64'), \ + contextlib.redirect_stdout(io.StringIO()) as out: + with self.assertRaises(SystemExit): + bmb.main(['--dump', '/nonexistent', '--stem', 'SCPH1001', + '--out', '/nonexistent/m.so']) + self.assertIn('cannot build a BIOS module', out.getvalue()) + + def test_bad_or_foreign_pins_are_rejected(self): + with host('Darwin', 'arm64'): + for bad in ('macos', 'macos-ppc', 'test-arch', 'darwin-arm64'): + self.assertIsNotNone(co.apply_runtime_arch_abi(bad, 'env'), bad) + self.assertIn('built for macos', co.apply_runtime_arch_abi('win-x64', 'env')) + self.assertEqual(co.cache_arch_abi(), 'macos-arm64') # unchanged + with host('Darwin', 'arm64', 'linux-x64'): + code, out = run_main_until_guard(MAIN_ARGV) + self.assertEqual(code, 2, out) + self.assertIn('PSX_OVERLAY_ARCH_ABI', out) + + def test_windows_is_unaffected(self): + for system, machine, gcc in (('Windows', 'AMD64', r'C:\msys64\mingw64\bin\gcc.exe'), + ('MSYS_NT-10.0-26100', 'x86_64', 'gcc')): + with host(system, machine): + before = (co.cache_arch_abi(), shard_command(gcc=gcc), + shard_command('tcc'), bios_module_command(gcc)) + with host(system, machine, 'win-x64'): + self.assertIsNone(co.apply_runtime_arch_abi('win-x64', 'env')) + after = (co.cache_arch_abi(), shard_command(gcc=gcc), + shard_command('tcc'), bios_module_command(gcc)) + self.assertIsNone(co.interpreter_arch_mismatch()) + self.assertEqual(before, after) + self.assertEqual(after[0], 'win-x64') + for cmd in after[1:]: + self.assertEqual(arch_args(cmd), [], cmd) + + def test_windows_process_arch_comes_from_the_build(self): + # platform.machine() on Windows is the MACHINE's (WMI since 3.12): + # an x64 Python emulated on Windows on ARM says ARM64, a 32-bit one + # on x64 Windows says AMD64. The interpreter's build is the process. + for machine, build, want in (('ARM64', 'win-amd64', 'x64'), + ('AMD64', 'win32', 'x86'), + ('ARM64', 'win-arm64', 'arm64'), + ('AMD64', 'mingw_x86_64_msvcrt_gnu', 'x64'), + ('AMD64', 'mingw_x86_64_ucrt_llvm', 'x64'), + ('ARM64', 'mingw_aarch64_ucrt_llvm', 'arm64'), + ('AMD64', 'mingw_i686_msvcrt_gnu', 'x86'), + ('ARM64', 'win-arm32', 'unknown')): + with host('Windows', machine, build=build): + self.assertEqual(co.interpreter_arch(), want, (machine, build)) + + def test_windows_on_arm_runs_the_x64_zip_unchanged(self): + # The x64 zip on Windows on ARM: runtime, Python and tcc all x64, + # emulated. Same commands and directory as on x64 Windows; neither + # tool refuses. + with host('Windows', 'AMD64', 'win-x64'): + self.assertIsNone(co.apply_runtime_arch_abi('win-x64', 'env')) + want = (co.cache_arch_abi(), shard_command(gcc='gcc'), + shard_command('tcc'), bios_module_command('gcc')) + with host('Windows', 'ARM64', 'win-x64', build='win-amd64'): + self.assertIsNone(co.apply_runtime_arch_abi('win-x64', 'env')) + self.assertIsNone(co.interpreter_arch_mismatch()) + got = (co.cache_arch_abi(), shard_command(gcc='gcc'), + shard_command('tcc'), bios_module_command('gcc')) + self.assertEqual(got, want) + self.assertEqual(got[0], 'win-x64') + with host('Windows', 'ARM64', 'win-x64', build='win-amd64'): + code, out = run_main_until_guard(MAIN_ARGV) + self.assertEqual(code, 'past-guard', out) + with host('Windows', 'ARM64', 'win-x64', build='win-amd64'), \ + contextlib.redirect_stdout(io.StringIO()) as out: + with self.assertRaises(SystemExit): # past the guard: no dump + bmb.main(['--dump', '/nonexistent', '--stem', 'SCPH1001', + '--out', '/nonexistent/m.dll', '--arch-abi', 'win-x64']) + self.assertNotIn('cannot build a BIOS module', out.getvalue()) + self.assertIn('not found', out.getvalue()) + # A real mismatch on Windows still refuses: an arm64 runtime cannot + # use what an x64 Python validates. + with host('Windows', 'ARM64', 'win-arm64', build='win-amd64'): + code, out = run_main_until_guard(MAIN_ARGV) + self.assertEqual(code, 2, out) + self.assertIn('runs as x64', out) + + def test_linux_is_unaffected(self): + for machine, tag in (('x86_64', 'linux-x64'), ('aarch64', 'linux-arm64')): + with host('Linux', machine): + before = (co.cache_arch_abi(), shard_command(), bios_module_command()) + with host('Linux', machine, tag): + self.assertIsNone(co.apply_runtime_arch_abi(tag, 'env')) + after = (co.cache_arch_abi(), shard_command(), bios_module_command()) + self.assertEqual(before, after) + self.assertEqual(after[0], tag) + self.assertEqual(arch_args(after[1]), []) + + def test_tcc_never_gets_arch_flags(self): + with host('Darwin', 'x86_64', 'macos-x64'): + co.apply_runtime_arch_abi('macos-x64', 'env') + self.assertEqual(arch_args(shard_command('tcc')), []) + + +def _runnable_macos_arches(): + """Mach-O arches this Mac can execute: its own, plus x86_64 via Rosetta.""" + if sys.platform != 'darwin': + return [] + arches = [] + for name in ('arm64', 'x86_64'): + if subprocess.run(['arch', '-' + name, '/usr/bin/true'], + capture_output=True).returncode == 0: + arches.append(name) + return arches + + +def _lipo_archs(path): + return subprocess.run(['lipo', '-archs', path], capture_output=True, + text=True, check=True).stdout.split() + + +@unittest.skipIf(os.name == 'nt', 'POSIX toolchain cases') +class RealToolchain(unittest.TestCase): + """The real C compiler and the real runtime sources.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix='psx_ovl_arch_') + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _cc(self, *args): + r = subprocess.run([CC, *args], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr or r.stdout) + + def _write(self, name, text): + path = os.path.join(self.tmp, name) + with open(path, 'w') as f: + f.write(text) + return path + + def test_runtime_exports_its_own_arch_to_spawned_compiles(self): + # autocompile.c + the four loader entry points it calls, nothing else; + # on macOS built universal and run as each slice the Mac can execute. + probe = self._write('probe.c', ''' +#include "autocompile.h" +#include "overlay_loader.h" +#include +#include +#include +struct OverlayPreparedImage { unsigned id; }; +OverlayPreparedImage *overlay_loader_prepare_published(const char *p) { (void)p; return NULL; } +int overlay_loader_commit_published(OverlayPreparedImage *i) { (void)i; return 0; } +void overlay_loader_discard_prepared(OverlayPreparedImage *i) { (void)i; } +void overlay_loader_rescan(void) { } +int main(void) { + const char *v; + autocompile_set_cache_paths("cache", "captures.json"); + autocompile_configure("true", "."); + v = getenv("PSX_OVERLAY_ARCH_ABI"); + printf("%s\\n", v ? v : "(unset)"); + return !(v && strcmp(v, PSX_OVERLAY_ARCH_ABI) == 0); +} +''') + exe = os.path.join(self.tmp, 'probe') + arches = _runnable_macos_arches() + flags = [a for name in arches for a in ('-arch', name)] + self._cc(*flags, '-I' + INCLUDE, probe, + os.path.join(ROOT, 'runtime', 'src', 'autocompile.c'), + '-o', exe, '-lpthread') + runs = [(['arch', '-' + name, exe], name) for name in arches] or [([exe], None)] + for argv, name in runs: + r = subprocess.run(argv, capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + got = r.stdout.split()[-1] if r.stdout.split() else '' + if name is None: + with host(platform.system(), platform.machine()): + want = co.cache_arch_abi() # same OS, same arch as this host + self.assertEqual(got, want) + else: + self.assertEqual(got, {'arm64': 'macos-arm64', + 'x86_64': 'macos-x64'}[name]) + + @unittest.skipUnless(sys.platform == 'darwin', 'macOS -arch') + def test_shards_are_the_target_arch_and_load_in_it(self): + # compile_overlays' real compile line, for each arch this Mac runs, + # whatever this Python is; then load the shard in a process of that + # architecture the way the runtime does (dlopen + overlay_abi). + shard = self._write('shard.c', 'int overlay_abi(void) { return 0x5A5A; }\n') + loader = self._write('loader.c', ''' +#include +#include +int main(int argc, char **argv) { + void *h = dlopen(argv[1], RTLD_NOW); + int (*abi)(void); + if (!h) { printf("dlopen: %s\\n", dlerror()); return 1; } + abi = (int (*)(void))dlsym(h, "overlay_abi"); + return !(abi && abi() == 0x5A5A); +} +''') + loader_exe = os.path.join(self.tmp, 'loader') + arches = _runnable_macos_arches() + self._cc(*[a for n in arches for a in ('-arch', n)], loader, '-o', loader_exe) + for name in arches: + tag = {'arm64': 'macos-arm64', 'x86_64': 'macos-x64'}[name] + out = os.path.join(self.tmp, f'{name}.so') + with host('Darwin', platform.machine(), tag): + co.apply_runtime_arch_abi(tag, 'env') + with contextlib.redirect_stdout(io.StringIO()): + ok = co._compile_dll_direct(shard, out, [INCLUDE], gcc=CC) + self.assertTrue(ok, name) + self.assertEqual(_lipo_archs(out), [name]) + r = subprocess.run(['arch', '-' + name, loader_exe, out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, f'{name}: {r.stdout}{r.stderr}') + + +def tearDownModule(): + if _FIXTURE: + shutil.rmtree(_FIXTURE, ignore_errors=True) + + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + ap.add_argument('--cc', default=os.environ.get('CC', 'cc')) + opts, rest = ap.parse_known_args() + CC = opts.cc + unittest.main(argv=[sys.argv[0]] + rest, verbosity=2) diff --git a/tools/bios_module_build.py b/tools/bios_module_build.py index 1b1769dd..ce507261 100644 --- a/tools/bios_module_build.py +++ b/tools/bios_module_build.py @@ -33,7 +33,7 @@ the exact dump its code describes. USAGE (as the runtime invokes it) python3 bios_module_build.py --dump --toolchain \ --stem SCPH1001 --out /SCPH1001_.so --flavor 0 \ - --compiler gcc|tcc [--gcc ] [--tcc ] + --compiler gcc|tcc [--gcc ] [--tcc ] [--arch-abi -] The toolchain dir is overlay_toolchain/ as tools/release_stage.py stages it (python, psxrecomp-bios, include/, bios/*.toml, recompiler/seeds/). Each @@ -143,7 +143,7 @@ def compile_module(sources: list[str], out: str, include_dirs: list[str], else: env, tc_dir = co._toolchain_env(gcc) pic = [] if co.is_windows() else ["-fPIC"] - cmd = [gcc, "-shared", *pic, "-O2", *defines, + cmd = [gcc, "-shared", *pic, *co.target_arch_flags("gcc"), "-O2", *defines, *[co.native_path(s) for s in sources], "-o", co.native_path(out), *[f"-I{co.native_path(d)}" for d in include_dirs], "-lm"] @@ -187,6 +187,9 @@ def main(argv=None) -> int: ap.add_argument("--stem", required=True, help="profile stem, e.g. SCPH1001 (bios/.toml)") ap.add_argument("--out", required=True, help="module path to publish (.so/.dll)") ap.add_argument("--flavor", type=int, default=0, help="PSX_OVERLAY_FLAVOR of the host build") + ap.add_argument("--arch-abi", default=None, + help="\"-\" of the host build (overlay_loader.h's PSX_OVERLAY_ARCH_ABI); " + "default: PSX_OVERLAY_ARCH_ABI from the environment, else this interpreter's") ap.add_argument("--compiler", choices=("gcc", "tcc"), default="gcc") ap.add_argument("--gcc", default="gcc") ap.add_argument("--tcc", default="tcc") @@ -197,6 +200,19 @@ def main(argv=None) -> int: ap.add_argument("--keep-work", action="store_true", help="keep the temporary emit tree (debugging)") args = ap.parse_args(argv) + # Build for the HOST's architecture, which on macOS need not be the + # compiler binary's default (an x86_64 runtime under Rosetta, arm64-only + # Xcode clang). The self-check loads the module into this interpreter, so + # the interpreter has to run as that architecture as well. + arch_err = co.apply_runtime_arch_abi( + args.arch_abi or os.environ.get("PSX_OVERLAY_ARCH_ABI"), + "--arch-abi" if args.arch_abi else "PSX_OVERLAY_ARCH_ABI") + if arch_err: + die(arch_err) + arch_mismatch = co.interpreter_arch_mismatch() + if arch_mismatch: + die(f"cannot build a BIOS module: {arch_mismatch}") + tk = os.path.abspath(args.toolchain) # Everything absolute: the emitter runs with cwd inside the staged work # tree, and the compiler is handed paths from more than one directory. diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 7f9aade8..a43630c8 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -56,6 +56,7 @@ except ImportError: import json import platform import re +import sysconfig def codegen_ver(runtime_include: str) -> int: @@ -236,28 +237,136 @@ def native_path(p: str) -> str: return os.path.abspath(p) +# The arch the shards are built for: the RUNTIME's, not this interpreter's. +# None means "this interpreter's" (manual/offline use). The spawning runtime +# pins it through PSX_OVERLAY_ARCH_ABI (autocompile.c), the same way it pins +# the cache dir and the flavor, because the interpreter, the compiler and the +# runtime need not share an architecture on macOS: an x86_64 runtime under +# Rosetta runs an x86_64 Python, but Xcode's clang is an arm64-only binary and +# defaults to arm64. Without -arch every shard it built was arm64, landed in +# the macos-x64 cache and never loaded, so the x64 runtime ran every overlay +# on the interpreter. +_TARGET_ARCH = None +_ARCH_ABI_OSES = ('win', 'linux', 'macos') +_ARCH_ABI_ARCHES = ('x64', 'arm64', 'x86', 'unknown') +# Apple's driver names for the arch-abi arches (clang -arch / gcc -arch). +_DARWIN_ARCH_NAMES = {'x64': 'x86_64', 'arm64': 'arm64', 'x86': 'i386'} + + +def _arch_abi_arch(machine: str) -> str: + """A machine/arch name (uname, PE, MSYS2 tag) in arch-abi spelling.""" + m = machine.lower() + if m in ('amd64', 'x86_64', 'x64'): + return 'x64' + if m in ('arm64', 'aarch64'): + return 'arm64' + if m in ('i386', 'i686', 'x86'): + return 'x86' + return 'unknown' + + +def interpreter_arch() -> str: + """Arch of THIS Python process, in arch-abi spelling: the only + architecture ctypes can load here. + + macOS and Linux: platform.machine() is uname's, which is the process's + own (x86_64 for a Python running under Rosetta). + + Native Windows: platform.machine() is the MACHINE's, not the process's + (Win32_Processor through WMI since Python 3.12, PROCESSOR_ARCHITEW6432 + for a WOW64 process before), so an x64 Python emulated on Windows on ARM + says ARM64 and a 32-bit one on x64 Windows says AMD64. The interpreter's + own build names the process there: sysconfig.get_platform() is + win-amd64 / win-arm64 / win32 for python.org builds (from the compiler + tag in sys.version) and mingw___ for MSYS2's.""" + if os.name == 'nt': + plat = sysconfig.get_platform().lower() + if plat == 'win32': + return 'x86' + m = re.match(r'(?:win-|mingw_)(amd64|arm64|x86_64|aarch64|i686)(?:_|$)', plat) + return _arch_abi_arch(m.group(1)) if m else 'unknown' + return _arch_abi_arch(platform.machine()) + + +def target_os_tag() -> str: + """The "" half of the arch-abi tag the shards are built for.""" + if _TARGET_OS in _ARCH_ABI_OSES: + return _TARGET_OS + if is_windows(): + return 'win' + return {'Darwin': 'macos'}.get(platform.system(), 'linux') + + +def set_target_arch(arch: str | None) -> None: + global _TARGET_ARCH + _TARGET_ARCH = arch + + +def target_arch() -> str: + """The "" half: the runtime's when it said so, else this interpreter's.""" + return _TARGET_ARCH or interpreter_arch() + + def cache_arch_abi() -> str: - """Canonical cache arch-abi tag, IDENTICAL to overlay_loader.c's + """Canonical cache arch-abi tag, IDENTICAL to overlay_loader.h's PSX_OVERLAY_ARCH_ABI ("-": win|linux|macos + x64|arm64|x86). gcc DLLs are namespaced under /gcc// so same-OS different-arch caches never comingle. Keep this - mapping in lockstep with overlay_loader.c.""" - if _TARGET_OS in ('win', 'linux', 'macos'): - os_tag = _TARGET_OS - elif is_windows(): - os_tag = 'win' - else: - os_tag = {'Darwin': 'macos'}.get(platform.system(), 'linux') - m = platform.machine().lower() - if m in ('amd64', 'x86_64', 'x64'): - arch = 'x64' - elif m in ('arm64', 'aarch64'): - arch = 'arm64' - elif m in ('i386', 'i686', 'x86'): - arch = 'x86' - else: - arch = 'unknown' - return f'{os_tag}-{arch}' + mapping in lockstep with overlay_loader.h.""" + return f'{target_os_tag()}-{target_arch()}' + + +def apply_runtime_arch_abi(value: str | None, source: str) -> str | None: + """Build for the runtime's "-" (PSX_OVERLAY_ARCH_ABI or + --arch-abi). Returns None when adopted (or nothing was given), else why + not. The OS half must agree with the OS the shards are built for: a + disagreement means the cache would be written where no loader reads.""" + if not value: + return None + value = value.strip() + os_tag, sep, arch = value.partition('-') + if not sep or os_tag not in _ARCH_ABI_OSES or arch not in _ARCH_ABI_ARCHES: + return (f'{source}={value!r} is not "-" ' + f'(os: {"|".join(_ARCH_ABI_OSES)}; ' + f'arch: {"|".join(_ARCH_ABI_ARCHES)})') + if os_tag != target_os_tag(): + return (f'{source}={value!r} names {os_tag}, but these shards are ' + f'built for {target_os_tag()}') + set_target_arch(arch) + return None + + +def target_arch_flags(compiler: str = 'gcc') -> list[str]: + """Compiler flags that make a shard's architecture the target's. + + macOS only: Apple's cc defaults to the architecture of the compiler + binary, not of whoever runs it, so the target is always named. Other + platforms are left exactly as they were (their toolchains target the one + architecture the runtime and the bundled Python share).""" + if compiler != 'gcc' or target_os_tag() != 'macos': + return [] + name = _DARWIN_ARCH_NAMES.get(target_arch()) + return ['-arch', name] if name else [] + + +def interpreter_arch_mismatch() -> str | None: + """Why this interpreter cannot build shards for the target, or None. + + Every shard is validated by loading it into this interpreter (ctypes: + overlay_abi, pair id, exports) before it counts, and a process can only + load its own architecture. So a mismatch cannot produce a usable cache; + say so instead of writing shards nothing will load.""" + have = interpreter_arch() + want = target_arch() + if have == want or have == 'unknown': + # 'unknown': nothing proves a mismatch; each shard's load check + # still decides, as before this guard existed. + return None + return (f'the runtime is {cache_arch_abi()} but this Python runs as ' + f'{have} ({sys.executable}); compiled shards are validated by ' + f'loading them here, so the Python must run as {want} too. Use a ' + f'Python built for {want} (a universal one runs as its caller\'s ' + f'architecture).') # --------------------------------------------------------------------------- @@ -5013,7 +5122,7 @@ def _compile_dll_direct(c_path: str, out_dll: str, include_dirs: list[str], # that conflicts with the host process. Use -shared without -fPIC. pic_flag = [] if is_windows() else ['-fPIC'] cmd = [ - gcc, '-shared', *pic_flag, '-O2', + gcc, '-shared', *pic_flag, *target_arch_flags('gcc'), '-O2', '-DPSX_OVERLAY_DLL_BUILD', # Overlays mirror the runtime's no-debug-tools build: the emitter guards # debug_server_cyc_observe (and friends) behind PSX_NO_DEBUG_TOOLS, and the @@ -6511,6 +6620,11 @@ def main(): 'to the sequential path).') ap.add_argument('--target-os', choices=['auto', 'win', 'linux', 'macos'], default='auto', help='target operating system for compiled overlay shards (default: auto)') + ap.add_argument('--arch-abi', default=None, + help='"-" of the runtime that will load the shards ' + '(e.g. macos-x64). The spawning runtime injects ' + 'PSX_OVERLAY_ARCH_ABI, which wins. Default: this ' + "interpreter's architecture.") args = ap.parse_args() target_os = args.target_os if target_os == 'auto': @@ -6564,6 +6678,24 @@ def main(): if _fl is not None and _fl != args.flavor: print(f'[cache] PSX_OVERLAY_FLAVOR overrides --flavor: {_fl}') args.flavor = _fl + # Architecture is pinned by the spawning runtime too (overlay_loader.h's + # PSX_OVERLAY_ARCH_ABI): the shards must be the RUNTIME's architecture and + # land in its arch-abi directory, whatever this interpreter or the + # compiler binary happens to be. + _env_arch = os.environ.get('PSX_OVERLAY_ARCH_ABI') + if _env_arch: + if args.arch_abi and _env_arch != args.arch_abi: + print(f'[cache] PSX_OVERLAY_ARCH_ABI overrides --arch-abi: {_env_arch}') + _arch_err = apply_runtime_arch_abi(_env_arch, 'PSX_OVERLAY_ARCH_ABI') + else: + _arch_err = apply_runtime_arch_abi(args.arch_abi, '--arch-abi') + if _arch_err: + ap.error(_arch_err) + if not args.static: + _arch_mismatch = interpreter_arch_mismatch() + if _arch_mismatch: + print(f'ERROR: cannot build overlay shards: {_arch_mismatch}') + sys.exit(2) if not args.captures: ap.error('no captures file: set PSX_OVERLAY_CAPTURES (runtime injects it) ' 'or pass --captures for manual/offline use')