mirror/psxrecomp
1
0
Fork 0
mirror of https://github.com/RetroPortingToolKit/psxrecomp synced 2026-10-25 18:05:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

Windows: embed UTF-8 code-page manifest; wide picker for pre-1903 (#371)

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matthew Stanley
2026-09-26 22:05:14 -07:00
parent 5c1839675d
commit ff63ee9bde
8 changed files with 381 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assets/windows/psxrecomp.manifest
+45
View File
@@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Application manifest embedded in every shipped psxrecomp executable
(cmake/psx_windows_manifest.cmake).
activeCodePage=UTF-8 makes the process ANSI code page UTF-8 on Windows 10
1903 and later: every narrow Win32 call (CreateFileA, FindFirstFileA,
CreateProcessA, ...), the CRT's fopen and argv then speak the same UTF-8
that SDL, tinyfiledialogs, TOML files, our string literals and libstdc++'s
std::filesystem already use. Without it those calls decode bytes in the
legacy system code page (CP932 on Japanese Windows), which garbles UI text
and breaks non-ASCII paths (psxrecomp issue #371). Older Windows ignores
the element; runtime/include/psx_win_utf8.h keeps dialog text correct there.
The trustInfo and compatibility sections restate what the MinGW toolchain's
default manifest supplies, because an embedded manifest replaces it.
-->
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
<security>
<requestedPrivileges>
<requestedExecutionLevel level="asInvoker" uiAccess="false"/>
</requestedPrivileges>
</security>
</trustInfo>
<compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
<application>
<!-- Windows Vista -->
<supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
<!-- Windows 7 -->
<supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
<!-- Windows 8 -->
<supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
<!-- Windows 8.1 -->
<supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
<!-- Windows 10 / 11 -->
<supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
</application>
</compatibility>
<application xmlns="urn:schemas-microsoft-com:asm.v3">
<windowsSettings>
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
</windowsSettings>
</application>
</assembly>
cmake/psx_windows_manifest.cmake
+62
View File
@@ -0,0 +1,62 @@
# Windows application manifest for every shipped psxrecomp executable.
#
# The manifest (assets/windows/psxrecomp.manifest) sets the process ANSI code
# page to UTF-8 on Windows 10 1903+, so narrow Win32 calls, the CRT and argv
# agree with the UTF-8 that SDL, tinyfiledialogs and our literals use
# (issue #371). Include this at file scope (it enables the RC language), then
# call psxrecomp_windows_manifest(<target>) on each executable. DLLs do not
# take one: the process code page comes from the exe's manifest alone.
include_guard(DIRECTORY)
set(PSXRECOMP_WINDOWS_MANIFEST
"${CMAKE_CURRENT_LIST_DIR}/../assets/windows/psxrecomp.manifest")
get_filename_component(PSXRECOMP_WINDOWS_MANIFEST
"${PSXRECOMP_WINDOWS_MANIFEST}" ABSOLUTE)
if(WIN32)
# clang/llvm-mingw CI needs an explicit RC compiler hint; MSVC and MSYS2
# GCC find theirs on their own.
if(NOT MSVC AND NOT CMAKE_RC_COMPILER)
find_program(CMAKE_RC_COMPILER
NAMES llvm-rc llvm-windres windres
HINTS
"$ENV{RETCOMM_TOOLCHAIN}/bin"
"$ENV{CMAKE_CLANG_V1}/bin"
DOC "Windows resource compiler (manifest, app icon)")
endif()
enable_language(RC)
if(NOT CMAKE_RC_COMPILER)
message(FATAL_ERROR
"psxrecomp: no Windows resource compiler (llvm-rc/windres/rc). "
"It is required to embed the UTF-8 code-page manifest; without it "
"non-ASCII paths and dialog text break on non-UTF-8 Windows locales.")
endif()
endif()
function(psxrecomp_windows_manifest target)
if(NOT WIN32)
return()
endif()
if(MSVC)
# A .manifest source is merged by the MSVC linker via /MANIFESTINPUT.
target_sources(${target} PRIVATE "${PSXRECOMP_WINDOWS_MANIFEST}")
else()
# MinGW: RT_MANIFEST (24) resource with CREATEPROCESS_MANIFEST_RESOURCE_ID
# (1). binutils and lld let it replace the toolchain's default-manifest.o.
string(REPLACE "\\" "/" _manifest_fwd "${PSXRECOMP_WINDOWS_MANIFEST}")
set(_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_manifest.rc")
set(_content "1 24 \"${_manifest_fwd}\"\n")
if(EXISTS "${_rc}")
file(READ "${_rc}" _old)
else()
set(_old "")
endif()
if(NOT _old STREQUAL _content)
file(WRITE "${_rc}" "${_content}")
endif()
target_sources(${target} PRIVATE "${_rc}")
set_property(SOURCE "${_rc}" APPEND PROPERTY
OBJECT_DEPENDS "${PSXRECOMP_WINDOWS_MANIFEST}")
endif()
endfunction()
recompiler/CMakeLists.txt
+9
View File
@@ -918,6 +918,15 @@ if(BUILD_TESTING AND Python3_Interpreter_FOUND AND NOT MSVC)
--cc ${CMAKE_C_COMPILER})
endif()
# UTF-8 process code page on Windows for every shipped tool (issue #371): they
# take disc/BIOS/output paths from argv and hand them to fopen/Win32 calls.
include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/psx_windows_manifest.cmake)
foreach(_psx_tool psxrecomp-game psxrecomp-toml psxrecomp psxrecomp-analyze psxrecomp-bios)
if(TARGET ${_psx_tool})
psxrecomp_windows_manifest(${_psx_tool})
endif()
endforeach()
# Reads both CMakeLists.txt files as text, so it catches an orphan in either
# project from whichever one is being configured. Placed before the collision
# guard, which must stay last because it needs every target declared; this one
runtime/CMakeLists.txt
+13
View File
@@ -85,6 +85,19 @@ if(BUILD_TESTING)
else()
message(STATUS "gl_readback_region_test: not registered (requires MinGW, SDL3 headers/static library in PSX_GL_READBACK_SDL_ROOT and gcc.exe/g++.exe in PSX_GL_READBACK_COMPILER_BIN)")
endif()
# Windows UTF-8 text boundary (issue #371): the helpers legacy Windows
# relies on, and — with the shipped manifest — a UTF-8 process code page.
if(WIN32)
add_executable(win_utf8_helpers_test tests/test_win_utf8.cpp)
target_include_directories(win_utf8_helpers_test PRIVATE include)
add_test(NAME win_utf8_helpers_test COMMAND win_utf8_helpers_test)
add_executable(win_utf8_manifest_test tests/test_win_utf8.cpp)
target_include_directories(win_utf8_manifest_test PRIVATE include)
psxrecomp_windows_manifest(win_utf8_manifest_test)
add_test(NAME win_utf8_manifest_test
COMMAND win_utf8_manifest_test --expect-utf8-acp)
endif()
add_executable(launcher_device_roundtrip_test
tests/test_launcher_device_roundtrip.cpp)
target_include_directories(launcher_device_roundtrip_test PRIVATE include)
runtime/include/psx_win_utf8.h
+70
View File
@@ -0,0 +1,70 @@
/* psx_win_utf8.h — UTF-8 -> Win32 wide text for dialog APIs (issue #371).
*
* The runtime's narrow strings are UTF-8: source literals ("Step 2 of 2 —"),
* SDL, tinyfiledialogs, TOML, and libstdc++'s std::filesystem, which reads
* and writes narrow paths as UTF-8 whatever the code page. Narrow ("A") Win32
* calls and the CRT decode bytes in the process ANSI code page instead. Every
* shipped exe embeds a manifest that makes that code page UTF-8
* (assets/windows/psxrecomp.manifest), which settles it on Windows 10 1903
* and later.
*
* Older Windows ignores the manifest and keeps the legacy code page (CP932
* on Japanese installs). UI text we hand to Win32 therefore goes through the
* "W" API via these helpers, which is right on every Windows version, and a
* path a W API returns is kept wide (std::filesystem::path(wchar_t*)) rather
* than squeezed through the legacy code page.
*
* Header-only C++; empty on non-Windows builds. */
#pragma once
#if defined(_WIN32) && defined(__cplusplus)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#include <cstddef>
#include <cstring>
#include <string>
namespace psx_win_utf8 {
/* True when the manifest's UTF-8 code page is in effect (Windows 10 1903+). */
inline bool utf8_code_page_active() { return GetACP() == CP_UTF8; }
/* UTF-8 bytes (explicit length, may contain NULs) -> UTF-16. Invalid
* sequences become U+FFFD rather than failing: this feeds display text. */
inline std::wstring from_utf8(const char* s, std::size_t n) {
if (!s || n == 0) return std::wstring();
const int wlen = MultiByteToWideChar(CP_UTF8, 0, s, (int)n, nullptr, 0);
if (wlen <= 0) return std::wstring();
std::wstring w((std::size_t)wlen, L'\0');
MultiByteToWideChar(CP_UTF8, 0, s, (int)n, &w[0], wlen);
return w;
}
inline std::wstring from_utf8(const char* s) {
return s ? from_utf8(s, std::strlen(s)) : std::wstring();
}
inline std::wstring from_utf8(const std::string& s) {
return from_utf8(s.data(), s.size());
}
/* A double-NUL-terminated UTF-8 multi-string (OPENFILENAME lpstrFilter) ->
* the same shape in UTF-16, terminators included. */
inline std::wstring multi_sz_from_utf8(const char* s) {
if (!s) return std::wstring(2, L'\0');
const char* p = s;
while (*p) p += std::strlen(p) + 1;
/* p is at the final NUL; convert through it so both terminators survive. */
std::wstring w = from_utf8(s, (std::size_t)(p - s) + 1);
if (w.empty() || w.back() != L'\0') w.push_back(L'\0');
if (w.size() < 2 || w[w.size() - 2] != L'\0') w.push_back(L'\0');
return w;
}
} // namespace psx_win_utf8
#endif /* _WIN32 && __cplusplus */
runtime/runtime.cmake
+10 -23
View File
@@ -17,6 +17,7 @@ set_property(CACHE PSX_PGO PROPERTY STRINGS "" generate use)
include("${PSXRECOMP_ROOT}/cmake/psx_runtime_ipo.cmake")
include("${PSXRECOMP_ROOT}/cmake/psx_dependency_archive.cmake")
include("${PSXRECOMP_ROOT}/cmake/psx_windows_manifest.cmake")
include("${PSXRECOMP_ROOT}/runtime/chd_dependency.cmake")
include("${PSXRECOMP_ROOT}/runtime/overlay_static_sources.cmake")
@@ -1445,6 +1446,8 @@ function(psxrecomp_add_runtime_target target)
${PSXRT_EXTRAS_SOURCES}
)
target_link_libraries(${target} PRIVATE chdr-static)
# UTF-8 process code page on Windows (issue #371).
psxrecomp_windows_manifest(${target})
# audio_trace.c uses C11 atomics. Make the runtime's actual language
# requirement explicit instead of relying on a parent project's global
# CMAKE_C_STANDARD setting. cxx_std_17 likewise — game CMakeLists may omit
@@ -1593,29 +1596,13 @@ function(psxrecomp_add_runtime_target target)
endif()
if(PSXRT_APP_ICON AND EXISTS "${PSXRT_APP_ICON}")
if(WIN32)
# clang/llvm-mingw CI needs an RC compiler or the .rc is ignored and
# the PE ships without an embedded icon.
enable_language(RC)
if(NOT CMAKE_RC_COMPILER)
find_program(CMAKE_RC_COMPILER
NAMES llvm-rc llvm-windres windres
HINTS
"$ENV{RETCOMM_TOOLCHAIN}/bin"
"$ENV{CMAKE_CLANG_V1}/bin"
DOC "Windows resource compiler for APP_ICON .rc")
endif()
if(CMAKE_RC_COMPILER)
string(REPLACE "\\" "/" _psxrt_ico_fwd "${PSXRT_APP_ICON}")
set(_psxrt_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_app_icon.rc")
file(WRITE "${_psxrt_rc}" "IDI_ICON1 ICON \"${_psxrt_ico_fwd}\"\n")
target_sources(${target} PRIVATE "${_psxrt_rc}")
message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (RC=${CMAKE_RC_COMPILER})")
else()
message(WARNING
"psxrecomp ${target}: APP_ICON set but no RC compiler "
"(llvm-rc/windres) — PE will have no embedded icon; "
"runtime still loads assets/psxrecomp.png via SDL")
endif()
# The RC language is enabled (and its compiler required) by
# cmake/psx_windows_manifest.cmake.
string(REPLACE "\\" "/" _psxrt_ico_fwd "${PSXRT_APP_ICON}")
set(_psxrt_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_app_icon.rc")
file(WRITE "${_psxrt_rc}" "IDI_ICON1 ICON \"${_psxrt_ico_fwd}\"\n")
target_sources(${target} PRIVATE "${_psxrt_rc}")
message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (RC=${CMAKE_RC_COMPILER})")
else()
message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (window icon via PNG)")
endif()
runtime/src/main.cpp
+30 -7
View File
@@ -150,6 +150,7 @@ extern "C" void psx_game_codegen_forward_if_built(int argc, char** argv);
#include <iphlpapi.h>
#include <windows.h>
#include <commdlg.h>
#include "psx_win_utf8.h"
#else
#include <arpa/inet.h>
#include <fcntl.h>
@@ -2248,21 +2249,31 @@ static bool pick_runtime_file(const char* title, const char* filter,
return false;
}
#ifdef _WIN32
char path_buf[4096];
// Wide API on purpose: `title` is UTF-8 ("... — Step 2 of 2 ..."), and the
// ANSI entry point decodes it in the process code page. The manifest makes
// that UTF-8 on Windows 10 1903+, but older Windows keeps the legacy code
// page and the em dash renders as kanji on CP932 (issue #371).
const std::wstring wtitle = psx_win_utf8::from_utf8(title);
const std::wstring wfilter = psx_win_utf8::multi_sz_from_utf8(filter);
wchar_t path_buf[4096];
std::memset(path_buf, 0, sizeof(path_buf));
OPENFILENAMEA ofn;
OPENFILENAMEW ofn;
std::memset(&ofn, 0, sizeof(ofn));
ofn.lStructSize = sizeof(ofn);
ofn.hwndOwner = NULL;
ofn.lpstrFilter = filter;
ofn.lpstrFilter = wfilter.c_str();
ofn.lpstrFile = path_buf;
ofn.nMaxFile = (DWORD)sizeof(path_buf);
ofn.lpstrTitle = title;
ofn.nMaxFile = (DWORD)(sizeof(path_buf) / sizeof(path_buf[0]));
ofn.lpstrTitle = wtitle.c_str();
ofn.Flags = OFN_FILEMUSTEXIST | OFN_PATHMUSTEXIST | OFN_HIDEREADONLY;
if (!GetOpenFileNameA(&ofn)) return false;
out = path_buf;
if (!GetOpenFileNameW(&ofn)) return false;
// Keep the path wide. Narrowing it through a legacy code page would lose
// characters that page lacks, and libstdc++ reads narrow paths as UTF-8
// anyway, so Shift-JIS bytes would throw in std::filesystem::path.
out = std::filesystem::path(path_buf);
return true;
#else
(void)filter;
@@ -12440,6 +12451,18 @@ int main(int argc, char** argv) {
std::setvbuf(stdout, nullptr, _IOLBF, BUFSIZ);
std::setvbuf(stderr, nullptr, _IOLBF, BUFSIZ);
std::fprintf(stderr, "psxrecomp: main() entered\n");
#ifdef _WIN32
/* The embedded manifest's UTF-8 code page needs Windows 10 1903+. Say so
* when it is not in effect: narrow file calls then disagree with the
* UTF-8 paths SDL, the launcher and std::filesystem use, so a non-ASCII
* path may fail to open. First thing to know about a path bug. */
if (!psx_win_utf8::utf8_code_page_active()) {
std::fprintf(stderr,
"psxrecomp: UTF-8 code page not active (ANSI code page %u, "
"Windows 10 1903+ needed); non-ASCII file paths may fail\n",
(unsigned)GetACP());
}
#endif
std::fflush(stderr);
#if defined(RECOMP_LAUNCHER)
launcher_boot_timing_mark("host:main_enter");
runtime/tests/test_win_utf8.cpp
+142
View File
@@ -0,0 +1,142 @@
// Windows UTF-8 text boundary (issue #371).
//
// Built twice by runtime/CMakeLists.txt:
// win_utf8_helpers_test — no manifest: the psx_win_utf8.h conversions that
// keep dialog text right on Windows < 10 1903.
// win_utf8_manifest_test — with the shipped manifest, run with
// --expect-utf8-acp: the process code page must
// be UTF-8, so a UTF-8 path given to fopen or an
// A call lands on its real Unicode name.
#include "psx_win_utf8.h"
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <string>
static int g_failures = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, \
#cond); \
++g_failures; \
} \
} while (0)
// "Step 2 of 2 — game disc image" as UTF-8 bytes (em dash = E2 80 94).
static const char kTitleUtf8[] = "Step 2 of 2 \xE2\x80\x94 game disc image";
static void test_from_utf8() {
const std::wstring w = psx_win_utf8::from_utf8(kTitleUtf8);
CHECK(w == L"Step 2 of 2 \x2014 game disc image");
CHECK(psx_win_utf8::from_utf8("").empty());
CHECK(psx_win_utf8::from_utf8(static_cast<const char*>(nullptr)).empty());
// Japanese: 日本 = E6 97 A5 E6 9C AC.
CHECK(psx_win_utf8::from_utf8(std::string("\xE6\x97\xA5\xE6\x9C\xAC")) ==
L"\x65E5\x672C");
// Malformed input degrades to U+FFFD instead of dropping the string.
const std::wstring bad = psx_win_utf8::from_utf8("a\xFF" "b");
CHECK(bad.size() == 3 && bad[0] == L'a' && bad[1] == 0xFFFD && bad[2] == L'b');
}
static void test_multi_sz() {
static const char filter[] =
"PS1 Disc Images (*.cue;*.bin)\0*.cue;*.bin\0All Files (*.*)\0*.*\0";
const std::wstring w = psx_win_utf8::multi_sz_from_utf8(filter);
static const wchar_t expect[] =
L"PS1 Disc Images (*.cue;*.bin)\0*.cue;*.bin\0All Files (*.*)\0*.*\0";
// sizeof(expect) counts the literal's own terminator: double-NUL end.
CHECK(w.size() == sizeof(expect) / sizeof(expect[0]));
CHECK(std::memcmp(w.data(), expect, sizeof(expect)) == 0);
// Non-ASCII label survives; shape stays double-NUL terminated.
static const char u8filter[] = "Disc \xE2\x80\x94 images\0*.cue\0";
const std::wstring u = psx_win_utf8::multi_sz_from_utf8(u8filter);
static const wchar_t uexpect[] = L"Disc \x2014 images\0*.cue\0";
CHECK(u.size() == sizeof(uexpect) / sizeof(uexpect[0]));
CHECK(std::memcmp(u.data(), uexpect, sizeof(uexpect)) == 0);
const std::wstring empty = psx_win_utf8::multi_sz_from_utf8(nullptr);
CHECK(empty.size() == 2 && empty[0] == 0 && empty[1] == 0);
}
// The picker returns std::filesystem::path(wchar_t*), and the runtime moves
// paths around as path.string(). That is only lossless because libstdc++
// treats narrow paths as UTF-8 regardless of the code page — pin it, since
// the manifest exists to make Win32 agree with exactly this.
static void test_filesystem_narrow_is_utf8() {
#if defined(__GLIBCXX__)
const std::filesystem::path wide(L"C:\\\x65E5\x672C\\caf\x00E9 \x2014.cue");
CHECK(wide.string() ==
"C:\\\xE6\x97\xA5\xE6\x9C\xAC\\caf\xC3\xA9 \xE2\x80\x94.cue");
const std::filesystem::path narrow(wide.string());
CHECK(narrow.wstring() == wide.wstring());
#endif
}
typedef LONG(WINAPI* RtlGetVersionFn)(OSVERSIONINFOW*);
static DWORD windows_build() {
OSVERSIONINFOW vi;
std::memset(&vi, 0, sizeof(vi));
vi.dwOSVersionInfoSize = sizeof(vi);
HMODULE ntdll = GetModuleHandleW(L"ntdll.dll");
RtlGetVersionFn fn = ntdll ? reinterpret_cast<RtlGetVersionFn>(
reinterpret_cast<void*>(
GetProcAddress(ntdll, "RtlGetVersion")))
: nullptr;
if (!fn || fn(&vi) != 0) return 0;
return vi.dwMajorVersion > 10 ? 0xFFFFFFFFu
: vi.dwMajorVersion == 10 ? vi.dwBuildNumber : 0;
}
static void test_manifest_code_page() {
const DWORD build = windows_build();
if (build < 18362) {
std::printf("SKIP manifest checks: Windows build %lu predates 1903\n",
(unsigned long)build);
return;
}
CHECK(GetACP() == CP_UTF8);
CHECK(psx_win_utf8::utf8_code_page_active());
// A UTF-8 path through the CRT's narrow fopen and a narrow A call must
// resolve to the file's true Unicode name — the property every narrow
// call site in the runtime depends on.
wchar_t tmp_w[MAX_PATH];
const DWORD n = GetTempPathW(MAX_PATH, tmp_w);
CHECK(n > 0 && n < MAX_PATH);
if (n == 0 || n >= MAX_PATH) return;
const std::wstring name_w =
L"psx_win_utf8_\x65E5\x672C_\x2014_\x00E9_" +
std::to_wstring(GetCurrentProcessId()) + L".tmp";
const std::filesystem::path tmp_path(tmp_w);
const std::wstring wide = (tmp_path / name_w).wstring();
const std::string narrow = (tmp_path / name_w).string(); // UTF-8
std::FILE* f = std::fopen(narrow.c_str(), "wb");
CHECK(f != nullptr);
if (!f) return;
std::fputs("x", f);
std::fclose(f);
CHECK(GetFileAttributesW(wide.c_str()) != INVALID_FILE_ATTRIBUTES);
CHECK(GetFileAttributesA(narrow.c_str()) != INVALID_FILE_ATTRIBUTES);
DeleteFileW(wide.c_str());
}
int main(int argc, char** argv) {
test_from_utf8();
test_multi_sz();
test_filesystem_narrow_is_utf8();
if (argc > 1 && std::strcmp(argv[1], "--expect-utf8-acp") == 0) {
test_manifest_code_page();
}
if (g_failures) {
std::fprintf(stderr, "%d check(s) failed\n", g_failures);
return 1;
}
std::printf("win_utf8 tests passed (ACP=%u)\n", (unsigned)GetACP());
return 0;
}