diff --git a/assets/windows/psxrecomp.manifest b/assets/windows/psxrecomp.manifest new file mode 100644 index 00000000..214d33a5 --- /dev/null +++ b/assets/windows/psxrecomp.manifest @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UTF-8 + + + diff --git a/cmake/psx_windows_manifest.cmake b/cmake/psx_windows_manifest.cmake new file mode 100644 index 00000000..aa00e325 --- /dev/null +++ b/cmake/psx_windows_manifest.cmake @@ -0,0 +1,62 @@ +# Windows application manifest for every shipped psxrecomp executable. +# +# The manifest (assets/windows/psxrecomp.manifest) sets the process ANSI code +# page to UTF-8 on Windows 10 1903+, so narrow Win32 calls, the CRT and argv +# agree with the UTF-8 that SDL, tinyfiledialogs and our literals use +# (issue #371). Include this at file scope (it enables the RC language), then +# call psxrecomp_windows_manifest() on each executable. DLLs do not +# take one: the process code page comes from the exe's manifest alone. + +include_guard(DIRECTORY) + +set(PSXRECOMP_WINDOWS_MANIFEST + "${CMAKE_CURRENT_LIST_DIR}/../assets/windows/psxrecomp.manifest") +get_filename_component(PSXRECOMP_WINDOWS_MANIFEST + "${PSXRECOMP_WINDOWS_MANIFEST}" ABSOLUTE) + +if(WIN32) + # clang/llvm-mingw CI needs an explicit RC compiler hint; MSVC and MSYS2 + # GCC find theirs on their own. + if(NOT MSVC AND NOT CMAKE_RC_COMPILER) + find_program(CMAKE_RC_COMPILER + NAMES llvm-rc llvm-windres windres + HINTS + "$ENV{RETCOMM_TOOLCHAIN}/bin" + "$ENV{CMAKE_CLANG_V1}/bin" + DOC "Windows resource compiler (manifest, app icon)") + endif() + enable_language(RC) + if(NOT CMAKE_RC_COMPILER) + message(FATAL_ERROR + "psxrecomp: no Windows resource compiler (llvm-rc/windres/rc). " + "It is required to embed the UTF-8 code-page manifest; without it " + "non-ASCII paths and dialog text break on non-UTF-8 Windows locales.") + endif() +endif() + +function(psxrecomp_windows_manifest target) + if(NOT WIN32) + return() + endif() + if(MSVC) + # A .manifest source is merged by the MSVC linker via /MANIFESTINPUT. + target_sources(${target} PRIVATE "${PSXRECOMP_WINDOWS_MANIFEST}") + else() + # MinGW: RT_MANIFEST (24) resource with CREATEPROCESS_MANIFEST_RESOURCE_ID + # (1). binutils and lld let it replace the toolchain's default-manifest.o. + string(REPLACE "\\" "/" _manifest_fwd "${PSXRECOMP_WINDOWS_MANIFEST}") + set(_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_manifest.rc") + set(_content "1 24 \"${_manifest_fwd}\"\n") + if(EXISTS "${_rc}") + file(READ "${_rc}" _old) + else() + set(_old "") + endif() + if(NOT _old STREQUAL _content) + file(WRITE "${_rc}" "${_content}") + endif() + target_sources(${target} PRIVATE "${_rc}") + set_property(SOURCE "${_rc}" APPEND PROPERTY + OBJECT_DEPENDS "${PSXRECOMP_WINDOWS_MANIFEST}") + endif() +endfunction() diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index b58a379d..dea265ad 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -918,6 +918,15 @@ if(BUILD_TESTING AND Python3_Interpreter_FOUND AND NOT MSVC) --cc ${CMAKE_C_COMPILER}) endif() +# UTF-8 process code page on Windows for every shipped tool (issue #371): they +# take disc/BIOS/output paths from argv and hand them to fopen/Win32 calls. +include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/psx_windows_manifest.cmake) +foreach(_psx_tool psxrecomp-game psxrecomp-toml psxrecomp psxrecomp-analyze psxrecomp-bios) + if(TARGET ${_psx_tool}) + psxrecomp_windows_manifest(${_psx_tool}) + endif() +endforeach() + # Reads both CMakeLists.txt files as text, so it catches an orphan in either # project from whichever one is being configured. Placed before the collision # guard, which must stay last because it needs every target declared; this one diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 2cbf7107..bd83e665 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -85,6 +85,19 @@ if(BUILD_TESTING) else() message(STATUS "gl_readback_region_test: not registered (requires MinGW, SDL3 headers/static library in PSX_GL_READBACK_SDL_ROOT and gcc.exe/g++.exe in PSX_GL_READBACK_COMPILER_BIN)") endif() + # Windows UTF-8 text boundary (issue #371): the helpers legacy Windows + # relies on, and — with the shipped manifest — a UTF-8 process code page. + if(WIN32) + add_executable(win_utf8_helpers_test tests/test_win_utf8.cpp) + target_include_directories(win_utf8_helpers_test PRIVATE include) + add_test(NAME win_utf8_helpers_test COMMAND win_utf8_helpers_test) + add_executable(win_utf8_manifest_test tests/test_win_utf8.cpp) + target_include_directories(win_utf8_manifest_test PRIVATE include) + psxrecomp_windows_manifest(win_utf8_manifest_test) + add_test(NAME win_utf8_manifest_test + COMMAND win_utf8_manifest_test --expect-utf8-acp) + endif() + add_executable(launcher_device_roundtrip_test tests/test_launcher_device_roundtrip.cpp) target_include_directories(launcher_device_roundtrip_test PRIVATE include) diff --git a/runtime/include/psx_win_utf8.h b/runtime/include/psx_win_utf8.h new file mode 100644 index 00000000..4435b032 --- /dev/null +++ b/runtime/include/psx_win_utf8.h @@ -0,0 +1,70 @@ +/* psx_win_utf8.h — UTF-8 -> Win32 wide text for dialog APIs (issue #371). + * + * The runtime's narrow strings are UTF-8: source literals ("Step 2 of 2 —"), + * SDL, tinyfiledialogs, TOML, and libstdc++'s std::filesystem, which reads + * and writes narrow paths as UTF-8 whatever the code page. Narrow ("A") Win32 + * calls and the CRT decode bytes in the process ANSI code page instead. Every + * shipped exe embeds a manifest that makes that code page UTF-8 + * (assets/windows/psxrecomp.manifest), which settles it on Windows 10 1903 + * and later. + * + * Older Windows ignores the manifest and keeps the legacy code page (CP932 + * on Japanese installs). UI text we hand to Win32 therefore goes through the + * "W" API via these helpers, which is right on every Windows version, and a + * path a W API returns is kept wide (std::filesystem::path(wchar_t*)) rather + * than squeezed through the legacy code page. + * + * Header-only C++; empty on non-Windows builds. */ +#pragma once + +#if defined(_WIN32) && defined(__cplusplus) + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include +#include + +namespace psx_win_utf8 { + +/* True when the manifest's UTF-8 code page is in effect (Windows 10 1903+). */ +inline bool utf8_code_page_active() { return GetACP() == CP_UTF8; } + +/* UTF-8 bytes (explicit length, may contain NULs) -> UTF-16. Invalid + * sequences become U+FFFD rather than failing: this feeds display text. */ +inline std::wstring from_utf8(const char* s, std::size_t n) { + if (!s || n == 0) return std::wstring(); + const int wlen = MultiByteToWideChar(CP_UTF8, 0, s, (int)n, nullptr, 0); + if (wlen <= 0) return std::wstring(); + std::wstring w((std::size_t)wlen, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s, (int)n, &w[0], wlen); + return w; +} + +inline std::wstring from_utf8(const char* s) { + return s ? from_utf8(s, std::strlen(s)) : std::wstring(); +} + +inline std::wstring from_utf8(const std::string& s) { + return from_utf8(s.data(), s.size()); +} + +/* A double-NUL-terminated UTF-8 multi-string (OPENFILENAME lpstrFilter) -> + * the same shape in UTF-16, terminators included. */ +inline std::wstring multi_sz_from_utf8(const char* s) { + if (!s) return std::wstring(2, L'\0'); + const char* p = s; + while (*p) p += std::strlen(p) + 1; + /* p is at the final NUL; convert through it so both terminators survive. */ + std::wstring w = from_utf8(s, (std::size_t)(p - s) + 1); + if (w.empty() || w.back() != L'\0') w.push_back(L'\0'); + if (w.size() < 2 || w[w.size() - 2] != L'\0') w.push_back(L'\0'); + return w; +} + +} // namespace psx_win_utf8 + +#endif /* _WIN32 && __cplusplus */ diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 1dda84e7..ce64de16 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -17,6 +17,7 @@ set_property(CACHE PSX_PGO PROPERTY STRINGS "" generate use) include("${PSXRECOMP_ROOT}/cmake/psx_runtime_ipo.cmake") include("${PSXRECOMP_ROOT}/cmake/psx_dependency_archive.cmake") +include("${PSXRECOMP_ROOT}/cmake/psx_windows_manifest.cmake") include("${PSXRECOMP_ROOT}/runtime/chd_dependency.cmake") include("${PSXRECOMP_ROOT}/runtime/overlay_static_sources.cmake") @@ -1445,6 +1446,8 @@ function(psxrecomp_add_runtime_target target) ${PSXRT_EXTRAS_SOURCES} ) target_link_libraries(${target} PRIVATE chdr-static) + # UTF-8 process code page on Windows (issue #371). + psxrecomp_windows_manifest(${target}) # audio_trace.c uses C11 atomics. Make the runtime's actual language # requirement explicit instead of relying on a parent project's global # CMAKE_C_STANDARD setting. cxx_std_17 likewise — game CMakeLists may omit @@ -1593,29 +1596,13 @@ function(psxrecomp_add_runtime_target target) endif() if(PSXRT_APP_ICON AND EXISTS "${PSXRT_APP_ICON}") if(WIN32) - # clang/llvm-mingw CI needs an RC compiler or the .rc is ignored and - # the PE ships without an embedded icon. - enable_language(RC) - if(NOT CMAKE_RC_COMPILER) - find_program(CMAKE_RC_COMPILER - NAMES llvm-rc llvm-windres windres - HINTS - "$ENV{RETCOMM_TOOLCHAIN}/bin" - "$ENV{CMAKE_CLANG_V1}/bin" - DOC "Windows resource compiler for APP_ICON .rc") - endif() - if(CMAKE_RC_COMPILER) - string(REPLACE "\\" "/" _psxrt_ico_fwd "${PSXRT_APP_ICON}") - set(_psxrt_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_app_icon.rc") - file(WRITE "${_psxrt_rc}" "IDI_ICON1 ICON \"${_psxrt_ico_fwd}\"\n") - target_sources(${target} PRIVATE "${_psxrt_rc}") - message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (RC=${CMAKE_RC_COMPILER})") - else() - message(WARNING - "psxrecomp ${target}: APP_ICON set but no RC compiler " - "(llvm-rc/windres) — PE will have no embedded icon; " - "runtime still loads assets/psxrecomp.png via SDL") - endif() + # The RC language is enabled (and its compiler required) by + # cmake/psx_windows_manifest.cmake. + string(REPLACE "\\" "/" _psxrt_ico_fwd "${PSXRT_APP_ICON}") + set(_psxrt_rc "${CMAKE_CURRENT_BINARY_DIR}/${target}_app_icon.rc") + file(WRITE "${_psxrt_rc}" "IDI_ICON1 ICON \"${_psxrt_ico_fwd}\"\n") + target_sources(${target} PRIVATE "${_psxrt_rc}") + message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (RC=${CMAKE_RC_COMPILER})") else() message(STATUS "psxrecomp ${target}: APP_ICON=${PSXRT_APP_ICON} (window icon via PNG)") endif() diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 18eb5c97..ac0080bd 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -150,6 +150,7 @@ extern "C" void psx_game_codegen_forward_if_built(int argc, char** argv); #include #include #include +#include "psx_win_utf8.h" #else #include #include @@ -2248,21 +2249,31 @@ static bool pick_runtime_file(const char* title, const char* filter, return false; } #ifdef _WIN32 - char path_buf[4096]; + // Wide API on purpose: `title` is UTF-8 ("... — Step 2 of 2 ..."), and the + // ANSI entry point decodes it in the process code page. The manifest makes + // that UTF-8 on Windows 10 1903+, but older Windows keeps the legacy code + // page and the em dash renders as kanji on CP932 (issue #371). + const std::wstring wtitle = psx_win_utf8::from_utf8(title); + const std::wstring wfilter = psx_win_utf8::multi_sz_from_utf8(filter); + wchar_t path_buf[4096]; std::memset(path_buf, 0, sizeof(path_buf)); - OPENFILENAMEA ofn; + OPENFILENAMEW ofn; std::memset(&ofn, 0, sizeof(ofn)); ofn.lStructSize = sizeof(ofn); ofn.hwndOwner = NULL; - ofn.lpstrFilter = filter; + ofn.lpstrFilter = wfilter.c_str(); ofn.lpstrFile = path_buf; - ofn.nMaxFile = (DWORD)sizeof(path_buf); - ofn.lpstrTitle = title; + ofn.nMaxFile = (DWORD)(sizeof(path_buf) / sizeof(path_buf[0])); + ofn.lpstrTitle = wtitle.c_str(); ofn.Flags = OFN_FILEMUSTEXIST | OFN_PATHMUSTEXIST | OFN_HIDEREADONLY; - if (!GetOpenFileNameA(&ofn)) return false; - out = path_buf; + if (!GetOpenFileNameW(&ofn)) return false; + + // Keep the path wide. Narrowing it through a legacy code page would lose + // characters that page lacks, and libstdc++ reads narrow paths as UTF-8 + // anyway, so Shift-JIS bytes would throw in std::filesystem::path. + out = std::filesystem::path(path_buf); return true; #else (void)filter; @@ -12440,6 +12451,18 @@ int main(int argc, char** argv) { std::setvbuf(stdout, nullptr, _IOLBF, BUFSIZ); std::setvbuf(stderr, nullptr, _IOLBF, BUFSIZ); std::fprintf(stderr, "psxrecomp: main() entered\n"); +#ifdef _WIN32 + /* The embedded manifest's UTF-8 code page needs Windows 10 1903+. Say so + * when it is not in effect: narrow file calls then disagree with the + * UTF-8 paths SDL, the launcher and std::filesystem use, so a non-ASCII + * path may fail to open. First thing to know about a path bug. */ + if (!psx_win_utf8::utf8_code_page_active()) { + std::fprintf(stderr, + "psxrecomp: UTF-8 code page not active (ANSI code page %u, " + "Windows 10 1903+ needed); non-ASCII file paths may fail\n", + (unsigned)GetACP()); + } +#endif std::fflush(stderr); #if defined(RECOMP_LAUNCHER) launcher_boot_timing_mark("host:main_enter"); diff --git a/runtime/tests/test_win_utf8.cpp b/runtime/tests/test_win_utf8.cpp new file mode 100644 index 00000000..ed1fa65c --- /dev/null +++ b/runtime/tests/test_win_utf8.cpp @@ -0,0 +1,142 @@ +// Windows UTF-8 text boundary (issue #371). +// +// Built twice by runtime/CMakeLists.txt: +// win_utf8_helpers_test — no manifest: the psx_win_utf8.h conversions that +// keep dialog text right on Windows < 10 1903. +// win_utf8_manifest_test — with the shipped manifest, run with +// --expect-utf8-acp: the process code page must +// be UTF-8, so a UTF-8 path given to fopen or an +// A call lands on its real Unicode name. +#include "psx_win_utf8.h" + +#include +#include +#include +#include + +static int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, \ + #cond); \ + ++g_failures; \ + } \ + } while (0) + +// "Step 2 of 2 — game disc image" as UTF-8 bytes (em dash = E2 80 94). +static const char kTitleUtf8[] = "Step 2 of 2 \xE2\x80\x94 game disc image"; + +static void test_from_utf8() { + const std::wstring w = psx_win_utf8::from_utf8(kTitleUtf8); + CHECK(w == L"Step 2 of 2 \x2014 game disc image"); + CHECK(psx_win_utf8::from_utf8("").empty()); + CHECK(psx_win_utf8::from_utf8(static_cast(nullptr)).empty()); + // Japanese: 日本 = E6 97 A5 E6 9C AC. + CHECK(psx_win_utf8::from_utf8(std::string("\xE6\x97\xA5\xE6\x9C\xAC")) == + L"\x65E5\x672C"); + // Malformed input degrades to U+FFFD instead of dropping the string. + const std::wstring bad = psx_win_utf8::from_utf8("a\xFF" "b"); + CHECK(bad.size() == 3 && bad[0] == L'a' && bad[1] == 0xFFFD && bad[2] == L'b'); +} + +static void test_multi_sz() { + static const char filter[] = + "PS1 Disc Images (*.cue;*.bin)\0*.cue;*.bin\0All Files (*.*)\0*.*\0"; + const std::wstring w = psx_win_utf8::multi_sz_from_utf8(filter); + static const wchar_t expect[] = + L"PS1 Disc Images (*.cue;*.bin)\0*.cue;*.bin\0All Files (*.*)\0*.*\0"; + // sizeof(expect) counts the literal's own terminator: double-NUL end. + CHECK(w.size() == sizeof(expect) / sizeof(expect[0])); + CHECK(std::memcmp(w.data(), expect, sizeof(expect)) == 0); + + // Non-ASCII label survives; shape stays double-NUL terminated. + static const char u8filter[] = "Disc \xE2\x80\x94 images\0*.cue\0"; + const std::wstring u = psx_win_utf8::multi_sz_from_utf8(u8filter); + static const wchar_t uexpect[] = L"Disc \x2014 images\0*.cue\0"; + CHECK(u.size() == sizeof(uexpect) / sizeof(uexpect[0])); + CHECK(std::memcmp(u.data(), uexpect, sizeof(uexpect)) == 0); + + const std::wstring empty = psx_win_utf8::multi_sz_from_utf8(nullptr); + CHECK(empty.size() == 2 && empty[0] == 0 && empty[1] == 0); +} + +// The picker returns std::filesystem::path(wchar_t*), and the runtime moves +// paths around as path.string(). That is only lossless because libstdc++ +// treats narrow paths as UTF-8 regardless of the code page — pin it, since +// the manifest exists to make Win32 agree with exactly this. +static void test_filesystem_narrow_is_utf8() { +#if defined(__GLIBCXX__) + const std::filesystem::path wide(L"C:\\\x65E5\x672C\\caf\x00E9 \x2014.cue"); + CHECK(wide.string() == + "C:\\\xE6\x97\xA5\xE6\x9C\xAC\\caf\xC3\xA9 \xE2\x80\x94.cue"); + const std::filesystem::path narrow(wide.string()); + CHECK(narrow.wstring() == wide.wstring()); +#endif +} + +typedef LONG(WINAPI* RtlGetVersionFn)(OSVERSIONINFOW*); + +static DWORD windows_build() { + OSVERSIONINFOW vi; + std::memset(&vi, 0, sizeof(vi)); + vi.dwOSVersionInfoSize = sizeof(vi); + HMODULE ntdll = GetModuleHandleW(L"ntdll.dll"); + RtlGetVersionFn fn = ntdll ? reinterpret_cast( + reinterpret_cast( + GetProcAddress(ntdll, "RtlGetVersion"))) + : nullptr; + if (!fn || fn(&vi) != 0) return 0; + return vi.dwMajorVersion > 10 ? 0xFFFFFFFFu + : vi.dwMajorVersion == 10 ? vi.dwBuildNumber : 0; +} + +static void test_manifest_code_page() { + const DWORD build = windows_build(); + if (build < 18362) { + std::printf("SKIP manifest checks: Windows build %lu predates 1903\n", + (unsigned long)build); + return; + } + CHECK(GetACP() == CP_UTF8); + CHECK(psx_win_utf8::utf8_code_page_active()); + + // A UTF-8 path through the CRT's narrow fopen and a narrow A call must + // resolve to the file's true Unicode name — the property every narrow + // call site in the runtime depends on. + wchar_t tmp_w[MAX_PATH]; + const DWORD n = GetTempPathW(MAX_PATH, tmp_w); + CHECK(n > 0 && n < MAX_PATH); + if (n == 0 || n >= MAX_PATH) return; + const std::wstring name_w = + L"psx_win_utf8_\x65E5\x672C_\x2014_\x00E9_" + + std::to_wstring(GetCurrentProcessId()) + L".tmp"; + const std::filesystem::path tmp_path(tmp_w); + const std::wstring wide = (tmp_path / name_w).wstring(); + const std::string narrow = (tmp_path / name_w).string(); // UTF-8 + + std::FILE* f = std::fopen(narrow.c_str(), "wb"); + CHECK(f != nullptr); + if (!f) return; + std::fputs("x", f); + std::fclose(f); + CHECK(GetFileAttributesW(wide.c_str()) != INVALID_FILE_ATTRIBUTES); + CHECK(GetFileAttributesA(narrow.c_str()) != INVALID_FILE_ATTRIBUTES); + DeleteFileW(wide.c_str()); +} + +int main(int argc, char** argv) { + test_from_utf8(); + test_multi_sz(); + test_filesystem_narrow_is_utf8(); + if (argc > 1 && std::strcmp(argv[1], "--expect-utf8-acp") == 0) { + test_manifest_code_page(); + } + if (g_failures) { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("win_utf8 tests passed (ACP=%u)\n", (unsigned)GetACP()); + return 0; +}