Merge pull request #227 from mstan/fix/selfbuild-toolchain-sysroot

Self-build: pack sysroot into CMake and require toolchain 1.0.14

host/psxrecomp_codegen_host.c

@@ -14,6 +14,7 @@
 #  include <errno.h>
 #  include <fcntl.h>
 #  include <spawn.h>
+#  include <strings.h>
 #  include <sys/stat.h>
 #  include <sys/wait.h>
 #  include <unistd.h>
@@ -63,6 +64,14 @@ static const char* cfg_or(const char* v, const char* d) {
     return (v && v[0]) ? v : d;
 }
 
+static int ascii_strcasecmp(const char* a, const char* b) {
+#if defined(_WIN32)
+    return _stricmp(a, b);
+#else
+    return strcasecmp(a, b);
+#endif
+}
+
 static int path_is_file(const char* path) {
 #if defined(_WIN32)
     DWORD attr = GetFileAttributesA(path);
@@ -1496,6 +1505,55 @@ static void handle_progress_line(const char* line,
     }
 }
 
+/* Last-lines capture so a failed CLI run can say WHY in the wizard, which has
+ * no console: the child's stderr shares the progress pipe, JSON progress rows
+ * contribute their "message", raw rows (compiler/traceback text) contribute
+ * as-is, and the most recent error-looking line is appended to err_msg. This
+ * is what turns "psxrecomp rebuild failed (exit 1)" into "… failed (exit 1):
+ * Could NOT find OpenGL (missing: OPENGL_INCLUDE_DIR)". */
+typedef struct {
+    char last[480];
+    char last_err[480];
+} CliTail;
+
+static int cli_tail_line_is_error(const char* s) {
+    return strstr(s, "rror") != NULL || strstr(s, "ailed") != NULL ||
+           strstr(s, "FAILED") != NULL || strstr(s, "Traceback") != NULL ||
+           strstr(s, "fatal") != NULL || strstr(s, "Fatal") != NULL;
+}
+
+static void cli_tail_note(CliTail* t, const char* line) {
+    char msg[480];
+    const char* rec = line;
+    int is_error_event = 0;
+    if (line[0] == '{') {
+        /* sdk_progress emits compact JSON: {"event":"error","message":…}. */
+        is_error_event = strstr(line, "\"event\":\"error\"") != NULL;
+        if (!json_get_string(line, "message", msg, sizeof(msg)))
+            return; /* structured row without text (e.g. result) */
+        rec = msg;
+    }
+    if (!rec[0])
+        return;
+    snprintf(t->last, sizeof(t->last), "%s", rec);
+    if (is_error_event || cli_tail_line_is_error(rec))
+        snprintf(t->last_err, sizeof(t->last_err), "%s", rec);
+}
+
+static void cli_fail_msg(char* err_msg, size_t err_cap, const char* fail_label,
+                         long code, const CliTail* t) {
+    const char* why = t->last_err[0] ? t->last_err : t->last;
+    if (code == 3) {
+        snprintf(err_msg, err_cap, "Disc verification failed (wrong dump).");
+        return;
+    }
+    if (why[0])
+        snprintf(err_msg, err_cap, "%s failed (exit %ld): %s", fail_label,
+                 code, why);
+    else
+        snprintf(err_msg, err_cap, "%s failed (exit %ld).", fail_label, code);
+}
+
 #if defined(_WIN32)
 static int run_cli_win(const char* cmdline,
                        RecompLauncherCPrepareProgressFn on_progress,
@@ -1519,7 +1577,9 @@ static int run_cli_win(const char* cmdline,
     si.cb = sizeof(si);
     si.dwFlags = STARTF_USESTDHANDLES;
     si.hStdOutput = wr;
-    si.hStdError = GetStdHandle(STD_ERROR_HANDLE);
+    /* stderr shares the pipe: Python tracebacks and raw tool output feed the
+     * CliTail capture instead of vanishing (the wizard has no console). */
+    si.hStdError = wr;
     si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
 
     char mutable_cmd[4096];
@@ -1538,6 +1598,8 @@ static int run_cli_win(const char* cmdline,
     char line[16384];
     size_t line_len = 0;
     DWORD n = 0;
+    CliTail tail;
+    memset(&tail, 0, sizeof(tail));
     while (ReadFile(rd, buf, sizeof(buf), &n, NULL) && n > 0) {
         for (DWORD i = 0; i < n; ++i) {
             char c = buf[i];
@@ -1545,6 +1607,7 @@ static int run_cli_win(const char* cmdline,
             if (c == '\n') {
                 line[line_len] = '\0';
                 handle_progress_line(line, on_progress, progress_ctx);
+                cli_tail_note(&tail, line);
                 line_len = 0;
                 continue;
             }
@@ -1555,6 +1618,7 @@ static int run_cli_win(const char* cmdline,
     if (line_len) {
         line[line_len] = '\0';
         handle_progress_line(line, on_progress, progress_ctx);
+        cli_tail_note(&tail, line);
     }
     CloseHandle(rd);
     WaitForSingleObject(pi.hProcess, INFINITE);
@@ -1563,11 +1627,7 @@ static int run_cli_win(const char* cmdline,
     CloseHandle(pi.hThread);
     CloseHandle(pi.hProcess);
     if (code == 0) return 1;
-    if (code == 3)
-        snprintf(err_msg, err_cap, "Disc verification failed (wrong dump).");
-    else
-        snprintf(err_msg, err_cap, "%s failed (exit %lu).", fail_label,
-                 (unsigned long)code);
+    cli_fail_msg(err_msg, err_cap, fail_label, (long)code, &tail);
     return 0;
 }
 #else
@@ -1585,6 +1645,9 @@ static int run_cli_posix(char* const argv[],
     posix_spawn_file_actions_init(&actions);
     posix_spawn_file_actions_addclose(&actions, pipefd[0]);
     posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDOUT_FILENO);
+    /* stderr shares the pipe: Python tracebacks and raw tool output feed the
+     * CliTail capture instead of vanishing (the wizard has no console). */
+    posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDERR_FILENO);
     posix_spawn_file_actions_addclose(&actions, pipefd[1]);
 
     pid_t pid = 0;
@@ -1606,11 +1669,14 @@ static int run_cli_posix(char* const argv[],
         return 0;
     }
     char line[16384];
+    CliTail tail;
+    memset(&tail, 0, sizeof(tail));
     while (fgets(line, sizeof(line), out)) {
         size_t n = strlen(line);
         while (n && (line[n - 1] == '\n' || line[n - 1] == '\r'))
             line[--n] = '\0';
         handle_progress_line(line, on_progress, progress_ctx);
+        cli_tail_note(&tail, line);
     }
     fclose(out);
 
@@ -1621,10 +1687,7 @@ static int run_cli_posix(char* const argv[],
     }
     int code = WIFEXITED(status) ? WEXITSTATUS(status) : 1;
     if (code == 0) return 1;
-    if (code == 3)
-        snprintf(err_msg, err_cap, "Disc verification failed (wrong dump).");
-    else
-        snprintf(err_msg, err_cap, "%s failed (exit %d).", fail_label, code);
+    cli_fail_msg(err_msg, err_cap, fail_label, (long)code, &tail);
     return 0;
 }
 #endif
@@ -1643,14 +1706,25 @@ static const char* toolchain_zip_asset_name(void) {
 #endif
 }
 
-/* Optional floor via RETCOMM_TOOLCHAIN_MIN_VERSION. Wizard/default is empty:
- * download GitHub /releases/latest and accept any usable pack (no per-title
- * version pinning to maintain). */
+/* Framework floor for accepted packs, not a per-title pin. v1.0.14 is the
+ * first cmake-clang-v1 with the Linux build sysroot; older cached packs
+ * compile against host headers (absent on stock SteamOS) and mis-link after a
+ * pack upgrade (__isoc23_strtoul). The cache is otherwise accepted forever
+ * once any cmake runs, so the floor is what makes a stale pre-sysroot pack
+ * get replaced. RETCOMM_TOOLCHAIN_MIN_VERSION overrides in either direction;
+ * "0" / "off" / "none" disables the floor (pre-floor behavior). Mirrors
+ * tools/toolchain_pack.py default_min_version(). */
+#define TOOLCHAIN_DEFAULT_MIN_VERSION "1.0.14"
+
 static const char* toolchain_min_version(void) {
     const char* env = getenv("RETCOMM_TOOLCHAIN_MIN_VERSION");
-    if (env && env[0])
+    if (env && env[0]) {
+        if (strcmp(env, "0") == 0 || ascii_strcasecmp(env, "off") == 0 ||
+            ascii_strcasecmp(env, "none") == 0)
+            return "";
         return env;
-    return "";
+    }
+    return TOOLCHAIN_DEFAULT_MIN_VERSION;
 }
 
 /* Parse leading dotted integers from a version / tag (optional leading 'v'). */

psxrecomp_cli.py

@@ -679,6 +679,9 @@ def _build_recompiler_targets(
     # no MSYS2 GCC DLLs (same as tools/ci/build_emitters.sh).
     if clang_c is not None or clang_cxx is not None:
         cmake_args.append("-DPSXRECOMP_STATIC_CLI=ON")
+    cmake_args.extend(
+        _pack_sysroot_cmake_args(clang_c, (*cmake_args, *extra_cmake_args))
+    )
     cmake_args.extend(extra_cmake_args)
 
     progress.log(" ".join(cmake_args))
@@ -1199,6 +1202,30 @@ def _tool_in_dir(bin_dir: Optional[Path], name: str) -> Optional[Path]:
     return None
 
 
+def _pack_sysroot_cmake_args(
+    clang_path: Optional[Path], existing: list[str] | tuple[str, ...] = ()
+) -> list[str]:
+    """CMake args that wire the portable pack's bundled sysroot (Linux only).
+
+    clang.cfg already gives the *compiler* ``--sysroot``, but CMake's find_*
+    still searches the host, so on a headerless host (stock SteamOS) recomp-ui's
+    ``find_package(OpenGL REQUIRED)`` fails at configure with "Could NOT find
+    OpenGL (missing: OPENGL_INCLUDE_DIR)". CMAKE_SYSROOT re-roots find_* into
+    the pack. OpenGL_GL_PREFERENCE=LEGACY is required with it: the sysroot
+    ships legacy ``libGL.so`` + ``GL/gl.h`` but no GLVND dev files
+    (``glx.h`` / ``libOpenGL.so`` / ``libGLX.so``), so the GLVND path cannot
+    complete inside the sysroot and ``OpenGL::GL`` would never be created.
+    """
+    if not sys.platform.startswith("linux") or clang_path is None:
+        return []
+    if any("CMAKE_SYSROOT" in str(a) for a in existing):
+        return []
+    sysroot = Path(clang_path).resolve().parent.parent / "sysroot"
+    if not (sysroot / "usr" / "include").is_dir():
+        return []
+    return [f"-DCMAKE_SYSROOT={sysroot}", "-DOpenGL_GL_PREFERENCE=LEGACY"]
+
+
 def _read_cmake_cache_generator(cache_file: Path) -> str:
     if not cache_file.is_file():
         return ""
@@ -1252,6 +1279,7 @@ def _cmake_configure(
         compiler_args.append(f"-DCMAKE_C_COMPILER={clang_c}")
     if clang_cxx is not None:
         compiler_args.append(f"-DCMAKE_CXX_COMPILER={clang_cxx}")
+    compiler_args.extend(_pack_sysroot_cmake_args(clang_c, extra))
 
     cmd = [
         str(cmake),

tools/toolchain_pack.py

@@ -331,12 +331,29 @@ def version_satisfies(have: str, need: str) -> bool:
     return parse_version_tuple(have) >= parse_version_tuple(need)
 
 
-def default_min_version() -> str:
-    """Optional semver floor. Empty by default — wizard downloads /releases/latest.
+# Framework floor, not a per-title pin: v1.0.14 is the first pack with the
+# Linux build sysroot (glibc/libstdc++/GL headers + stubs). Older cached packs
+# compile against host headers, which do not exist on stock SteamOS, and their
+# objects mis-link after a pack upgrade (__isoc23_strtoul). Because the cache
+# is accepted forever once any cmake runs, a floor is the only way a stale
+# pre-sysroot pack ever gets replaced.
+DEFAULT_MIN_VERSION = "1.0.14"
 
-    Set RETCOMM_TOOLCHAIN_MIN_VERSION only when you need to reject stale caches.
+# RETCOMM_TOOLCHAIN_MIN_VERSION values that disable the floor entirely.
+_MIN_VERSION_DISABLE = {"0", "off", "none"}
+
+
+def default_min_version() -> str:
+    """Semver floor for accepted packs (DEFAULT_MIN_VERSION unless overridden).
+
+    RETCOMM_TOOLCHAIN_MIN_VERSION overrides in either direction: a version
+    raises/lowers the floor, and "0" / "off" / "none" disables it (accept any
+    usable pack, matching the pre-floor behavior).
     """
-    return (os.environ.get("RETCOMM_TOOLCHAIN_MIN_VERSION") or "").strip()
+    env = (os.environ.get("RETCOMM_TOOLCHAIN_MIN_VERSION") or "").strip()
+    if env:
+        return "" if env.lower() in _MIN_VERSION_DISABLE else env
+    return DEFAULT_MIN_VERSION
 
 
 def pack_satisfies_min(root: Path, min_version: str = "") -> bool:
@@ -974,7 +991,8 @@ def ensure_toolchain(
         )
 
     # Usable pack exists but is too old — fall through to download/replace.
-    stale = resolve_toolchain_bin(project_root, min_version="")
+    # min_version="0" probes without the default floor ("" would re-apply it).
+    stale = resolve_toolchain_bin(project_root, min_version="0")
     if stale and need and not existing and log:
         log(
             f"Cached toolchain does not meet min_version {need}; "