diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json index 2aecdc11f2..b94ab2bb1b 100644 --- a/.vs/launch.vs.json +++ b/.vs/launch.vs.json @@ -168,8 +168,8 @@ "type": "default", "project": "CMakeLists.txt", "projectTarget": "extractor.exe (bin\\extractor.exe)", - "name": "Tools - Extractor - Extract", - "args": ["E:\\ISOs\\Jak\\Jak 1.iso"] + "name": "Tools - Extractor - Full", + "args": ["\"E:\\ISOs\\Jak\\Jak 1.iso\""] } ] } diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index a449be7a69..ae9f7d870c 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -49,7 +49,8 @@ add_library(common util/os.cpp util/print_float.cpp util/FontUtils.cpp - util/FrameLimiter.cpp "util/unicode_util.h" "util/unicode_util.cpp") + util/FrameLimiter.cpp + util/unicode_util.cpp) target_link_libraries(common fmt lzokay replxx libzstd_static) diff --git a/common/goos/Interpreter.cpp b/common/goos/Interpreter.cpp index 6ee88742bd..74d75ac89a 100644 --- a/common/goos/Interpreter.cpp +++ b/common/goos/Interpreter.cpp @@ -10,6 +10,7 @@ #include "ParseHelpers.h" #include "common/util/FileUtil.h" +#include "common/util/unicode_util.h" #include "third-party/fmt/core.h" @@ -1682,8 +1683,8 @@ Object Interpreter::eval_get_env(const Object& form, const std::shared_ptr&) { vararg_check(form, args, {ObjectType::STRING}, {{"default", {false, ObjectType::STRING}}}); const std::string var_name = args.unnamed.at(0).as_string()->data; - const char* env_p = std::getenv(var_name.c_str()); - if (env_p == NULL) { + auto env_p = get_env(var_name); + if (env_p.empty()) { if (args.has_named("default")) { return args.get_named("default"); } else { diff --git a/common/util/FileUtil.cpp b/common/util/FileUtil.cpp index d67aa87091..27e91751bf 100644 --- a/common/util/FileUtil.cpp +++ b/common/util/FileUtil.cpp @@ -15,6 +15,7 @@ #include "common/common_types.h" #include "common/util/BinaryReader.h" +#include "common/util/unicode_util.h" // This disables the use of PCLMULQDQ which is probably ok, but let's just be safe and disable it // because nobody will care if png compression is 10% slower. @@ -39,10 +40,10 @@ namespace file_util { fs::path get_user_home_dir() { #ifdef _WIN32 // NOTE - on older systems, this may case issues if it cannot be found! - std::string home_dir = std::getenv("USERPROFILE"); + std::string home_dir = get_env("USERPROFILE"); return fs::path(home_dir); #else - std::string home_dir = std::getenv("HOME"); + std::string home_dir = get_env("HOME"); return fs::path(home_dir); #endif } @@ -50,16 +51,16 @@ fs::path get_user_home_dir() { fs::path get_user_config_dir() { fs::path config_base_path; #ifdef _WIN32 - auto config_base_dir = std::getenv("APPDATA"); - config_base_path = fs::path(std::string(config_base_dir)); + auto config_base_dir = get_env("APPDATA"); + config_base_path = fs::path(config_base_dir); #elif __linux // Docs - https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html // Prefer XDG_CONFIG_HOME if available - auto config_base_dir = std::getenv("XDG_CONFIG_HOME"); - if (!config_base_dir) { + auto config_base_dir = get_env("XDG_CONFIG_HOME"); + if (config_base_dir.empty()) { config_base_path = get_user_home_dir() / ".config"; } else { - config_base_path = std::string(config_base_dir); + config_base_path = fs::path(config_base_dir); } #endif return config_base_path / "OpenGOAL"; @@ -85,9 +86,10 @@ struct { */ std::string get_current_executable_path() { #ifdef _WIN32 - char buffer[FILENAME_MAX]; - GetModuleFileNameA(NULL, buffer, FILENAME_MAX); - std::string file_path(buffer); + // NOTE - MAX_PATH is kinda wrong here as you can have a path longer than 260 in windows + wchar_t path[MAX_PATH]; + GetModuleFileNameW(NULL, path, MAX_PATH); + std::string file_path = wide_string_to_utf8_string(path); if (file_path.rfind("\\\\?\\", 0) == 0) { return file_path.substr(4); } @@ -280,7 +282,7 @@ std::vector read_binary_file(const fs::path& path) { } std::string read_text_file(const fs::path& path) { - std::ifstream file(path.string()); + fs::ifstream file(path); if (!file.good()) { throw std::runtime_error("couldn't open " + path.string()); } diff --git a/common/util/unicode_util.cpp b/common/util/unicode_util.cpp index 11f281ba03..27581efd4d 100644 --- a/common/util/unicode_util.cpp +++ b/common/util/unicode_util.cpp @@ -14,48 +14,66 @@ #endif // clang-format on -std::string utf8_from_utf16(const wchar_t* utf16_string) { #ifdef _WIN32 - if (utf16_string == nullptr) { - return std::string(); - } - int target_length = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, -1, nullptr, - 0, nullptr, nullptr); - if (target_length == 0) { - return std::string(); - } - std::string utf8_string; - utf8_string.resize(target_length); - int converted_length = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, -1, - utf8_string.data(), target_length, nullptr, nullptr); - if (converted_length == 0) { - return std::string(); - } - return utf8_string; -#else - return "don't call this on linux"; -#endif -} - -std::vector get_widechar_cli_args() { -#ifdef _WIN32 - // Convert the UTF-16 command line arguments to UTF-8 for the Engine to use. - int argc; - wchar_t** argv = CommandLineToArgvW(GetCommandLineW(), &argc); - if (argv == nullptr) { +std::wstring utf8_string_to_wide_string(const std::string_view& str) { + std::wstring ret; + if (!utf8_string_to_wide_string(ret, str)) return {}; + + return ret; +} + +bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str) { + int wlen = + MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast(str.length()), nullptr, 0); + if (wlen < 0) + return false; + + dest.resize(wlen); + if (wlen > 0 && MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast(str.length()), + dest.data(), wlen) < 0) + return false; + + return true; +} + +std::string wide_string_to_utf8_string(const std::wstring_view& str) { + std::string ret; + if (!wide_string_to_utf8_string(ret, str)) + ret.clear(); + + return ret; +} + +bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str) { + int mblen = WideCharToMultiByte(CP_UTF8, 0, str.data(), static_cast(str.length()), nullptr, + 0, nullptr, nullptr); + if (mblen < 0) + return false; + + dest.resize(mblen); + if (mblen > 0 && WideCharToMultiByte(CP_UTF8, 0, str.data(), static_cast(str.length()), + dest.data(), mblen, nullptr, nullptr) < 0) { + return false; } - std::vector command_line_arguments; + return true; +} +#endif - for (int i = 0; i < argc; i++) { - command_line_arguments.push_back(utf8_from_utf16(argv[i])); +std::string get_env(const std::string& name) { +#ifdef _WIN32 + auto name_wide = utf8_string_to_wide_string(name); + auto val = _wgetenv(name_wide.data()); + if (!val) { + return ""; } - - LocalFree(argv); - - return command_line_arguments; + return wide_string_to_utf8_string(val); #else - return {"don't call this on linux"}; + auto val = std::getenv(name.data()); + if (!val) { + return ""; + } + return val; #endif } diff --git a/common/util/unicode_util.h b/common/util/unicode_util.h index cd79df1fe7..165ce4279b 100644 --- a/common/util/unicode_util.h +++ b/common/util/unicode_util.h @@ -3,5 +3,11 @@ #include #include -std::string utf8_from_utf16(const wchar_t* utf16_string); -std::vector get_widechar_cli_args(); +#ifdef _WIN32 +std::wstring utf8_string_to_wide_string(const std::string_view& str); +bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str); +std::string wide_string_to_utf8_string(const std::wstring_view& str); +bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str); +#endif + +std::string get_env(const std::string& name); diff --git a/decompiler/data/TextureDB.cpp b/decompiler/data/TextureDB.cpp index ec90e2bd4f..99a1409e92 100644 --- a/decompiler/data/TextureDB.cpp +++ b/decompiler/data/TextureDB.cpp @@ -3,6 +3,7 @@ #include "common/util/Assert.h" #include "third-party/fmt/core.h" +#define STBI_WINDOWS_UTF8 #include "third-party/stb_image/stb_image.h" namespace decompiler { diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 5c0e61de43..a881521a28 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -234,14 +234,11 @@ int main(int argc, char** argv) { bool flag_folder = false; std::string game_name = "jak1"; -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + lg::error("Bad encoding, needs UTF-8"); + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif lg::initialize(); @@ -267,7 +264,7 @@ int main(int argc, char** argv) { // If no flag is set, we default to running everything if (!flag_extract && !flag_decompile && !flag_compile && !flag_play) { - fmt::print("Running all steps, no flags provided!\n"); + lg::info("Running all steps, no flags provided!"); flag_runall = true; } if (flag_runall) { @@ -284,11 +281,22 @@ int main(int argc, char** argv) { lg::error("Error: project path override '{}' does not exist", project_path_override.string()); return static_cast(ExtractorErrorCode::INVALID_CLI_INPUT); } - file_util::setup_project_path(project_path_override); + auto ok = file_util::setup_project_path(project_path_override); + if (!ok) { + lg::error("Could not setup project path!"); + return 1; + } } else { - file_util::setup_project_path({}); + auto ok = file_util::setup_project_path({}); + if (!ok) { + lg::error("Could not setup project path!"); + return 1; + } } + auto log_path = file_util::get_jak_project_dir() / "extractor.log"; + lg::set_file(log_path.string()); + fs::path iso_data_path; // - INPUT VALIDATION diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 642ea08581..cda17f71d9 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -18,14 +18,10 @@ #include "decompiler/level_extractor/extract_level.h" int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif Timer decomp_timer; diff --git a/game/graphics/pipelines/opengl.cpp b/game/graphics/pipelines/opengl.cpp index 273b6c646d..be0badd265 100644 --- a/game/graphics/pipelines/opengl.cpp +++ b/game/graphics/pipelines/opengl.cpp @@ -29,6 +29,7 @@ #include "third-party/imgui/imgui.h" #include "third-party/imgui/imgui_impl_glfw.h" #include "third-party/imgui/imgui_impl_opengl3.h" +#define STBI_WINDOWS_UTF8 #include "third-party/stb_image/stb_image.h" namespace { @@ -182,43 +183,14 @@ static std::shared_ptr gl_make_display(int width, (file_util::get_jak_project_dir() / "game" / "assets" / "appicon.png").string(); GLFWimage images[1]; - images[0].pixels = - stbi_load(image_path.c_str(), &images[0].width, &images[0].height, 0, 4); // rgba channels - glfwSetWindowIcon(window, 1, images); - stbi_image_free(images[0].pixels); - - // init framerate settings - /* - if (GLFWmonitor* primary_monitor = glfwGetPrimaryMonitor()) { - auto primary_monitor_video_mode = glfwGetVideoMode(primary_monitor); - - if (primary_monitor_video_mode && primary_monitor_video_mode->refreshRate > 60) { - // Use the framelimiter by default and disable vsync - Gfx::g_global_settings.framelimiter = true; - Gfx::g_global_settings.vsync = false; - glfwSwapInterval(0); - if (primary_monitor_video_mode->refreshRate > 100) { - BootVideoMode = VideoMode::FPS150; - Gfx::g_global_settings.target_fps = 150; - } else if (primary_monitor_video_mode->refreshRate > 60) { - BootVideoMode = VideoMode::FPS100; - Gfx::g_global_settings.target_fps = 100; - } - } else { - // enable vsync - Gfx::g_global_settings.framelimiter = false; - Gfx::g_global_settings.vsync = true; - // glfwSwapInterval(1); - glfwSwapInterval(settings.vsync); - } + auto load_result = stbi_load(image_path.c_str(), &images[0].width, &images[0].height, 0, 4); + if (load_result) { + images[0].pixels = load_result; // rgba channels + glfwSetWindowIcon(window, 1, images); + stbi_image_free(images[0].pixels); } else { - // enable vsync - Gfx::g_global_settings.framelimiter = false; - Gfx::g_global_settings.vsync = true; - // glfwSwapInterval(1); - glfwSwapInterval(settings.vsync); + lg::error("Could not load icon for OpenGL window"); } - */ SetDisplayCallbacks(window); Pad::initialize(); diff --git a/game/main.cpp b/game/main.cpp index 8a63b65779..7fcc081b9a 100644 --- a/game/main.cpp +++ b/game/main.cpp @@ -3,6 +3,8 @@ * Main for the game. Launches the runtime. */ +#define STBI_WINDOWS_UTF8 + #include #include "runtime.h" @@ -40,14 +42,13 @@ void setup_logging(bool verbose) { * Entry point for the game. */ int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif + + // TODO - replace with CLI11 and just propagate args through + // - https://github.com/CLIUtils/CLI11/issues/744 // Figure out if the CPU has AVX2 to enable higher performance AVX2 versions of functions. setup_cpu_info(); @@ -125,17 +126,21 @@ int main(int argc, char** argv) { // run the runtime in a loop so we can reset the game and have it restart cleanly lg::info("OpenGOAL Runtime {}.{}", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); - auto exit_status = exec_runtime(ptrs.size(), ptrs.data()); - - switch (exit_status) { - case RuntimeExitStatus::EXIT: - return 0; - case RuntimeExitStatus::RESTART_RUNTIME: - case RuntimeExitStatus::RUNNING: - break; - case RuntimeExitStatus::RESTART_IN_DEBUG: - force_debug_next_time = true; - break; + try { + auto exit_status = exec_runtime(ptrs.size(), ptrs.data()); + switch (exit_status) { + case RuntimeExitStatus::EXIT: + return 0; + case RuntimeExitStatus::RESTART_RUNTIME: + case RuntimeExitStatus::RUNNING: + break; + case RuntimeExitStatus::RESTART_IN_DEBUG: + force_debug_next_time = true; + break; + } + } catch (std::exception& ex) { + lg::error("Unexpected exception occurred - {}", ex.what()); + throw ex; } } return 0; diff --git a/test/offline/offline_test_main.cpp b/test/offline/offline_test_main.cpp index 7ced28357f..86f3f15a93 100644 --- a/test/offline/offline_test_main.cpp +++ b/test/offline/offline_test_main.cpp @@ -389,14 +389,11 @@ std::optional parse_config(const std::string_view& game_name) } int main(int argc, char* argv[]) { -#ifdef _WIN32 - auto utf8_args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : utf8_args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + std::cerr << "Bad encoding, needs UTF-8." << std::endl; + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif lg::initialize(); diff --git a/test/test_main.cpp b/test/test_main.cpp index 2029e30dff..2be6d8b2f5 100644 --- a/test/test_main.cpp +++ b/test/test_main.cpp @@ -19,14 +19,11 @@ // to make it easier to test a subset of tests int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + std::cerr << "Bad encoding, needs UTF-8." << std::endl; + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif // hopefully get a debug print on github actions setup_cpu_info(); diff --git a/third-party/fpng/fpng.cpp b/third-party/fpng/fpng.cpp index 8452e2a9fc..49e490e012 100644 --- a/third-party/fpng/fpng.cpp +++ b/third-party/fpng/fpng.cpp @@ -1707,6 +1707,31 @@ do_literals: } #ifndef FPNG_NO_STDIO +#ifdef _WIN32 + bool utf8_string_to_wide_string(std::wstring& dest, const std::string& str) { + int wlen = MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast(str.length()), + nullptr, 0); + if (wlen < 0) + return false; + + dest.resize(wlen); + if (wlen > 0 && + MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast(str.length()), + dest.data(), wlen) < 0) + return false; + + return true; + } + + std::wstring utf8_string_to_wide_string(const std::string& str) { + std::wstring ret; + if (!utf8_string_to_wide_string(ret, str)) + return {}; + + return ret; + } +#endif + bool fpng_encode_image_to_file(const char* pFilename, const void* pImage, uint32_t w, uint32_t h, uint32_t num_chans, uint32_t flags) { std::vector out_buf; @@ -1715,7 +1740,14 @@ do_literals: FILE* pFile = nullptr; #ifdef _MSC_VER - fopen_s(&pFile, pFilename, "wb"); + // NOTE - Manual fix by us to support unicode.... + std::wstring converted_path = utf8_string_to_wide_string(pFilename); + if (converted_path.empty()) { + printf("bad path - %s", converted_path.data()); + return false; + } + + pFile = _wfopen(converted_path.data(), L"wb"); #else pFile = fopen(pFilename, "wb"); #endif diff --git a/third-party/stb_image/stb_image.cpp b/third-party/stb_image/stb_image.cpp index 80d8f9af64..d5cd88a0a5 100644 --- a/third-party/stb_image/stb_image.cpp +++ b/third-party/stb_image/stb_image.cpp @@ -1,4 +1,5 @@ +#define STBI_WINDOWS_UTF8 #define STB_IMAGE_IMPLEMENTATION #define STB_IMAGE_WRITE_IMPLEMENTATION #include "stb_image_write.h" -#include "stb_image.h" \ No newline at end of file +#include "stb_image.h" diff --git a/third-party/tiny_gltf/tiny_gltf.h b/third-party/tiny_gltf/tiny_gltf.h index 98cc4e42fc..ce66297ec6 100644 --- a/third-party/tiny_gltf/tiny_gltf.h +++ b/third-party/tiny_gltf/tiny_gltf.h @@ -1554,6 +1554,7 @@ class TinyGLTF { #ifndef TINYGLTF_NO_STB_IMAGE #ifndef TINYGLTF_NO_INCLUDE_STB_IMAGE +#define STBI_WINDOWS_UTF8 #include "stb_image.h" #endif #endif diff --git a/tools/MemoryDumpTool/main.cpp b/tools/MemoryDumpTool/main.cpp index b0ad64f72e..51c433f158 100644 --- a/tools/MemoryDumpTool/main.cpp +++ b/tools/MemoryDumpTool/main.cpp @@ -558,14 +558,10 @@ void inspect_symbols(const Ram& ram, } int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif fmt::print("MemoryDumpTool\n"); diff --git a/tools/dgo_packer.cpp b/tools/dgo_packer.cpp index 797c2723ec..059670a66e 100644 --- a/tools/dgo_packer.cpp +++ b/tools/dgo_packer.cpp @@ -6,14 +6,10 @@ #include int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); printf("DGO Packing Tool\n"); diff --git a/tools/dgo_unpacker.cpp b/tools/dgo_unpacker.cpp index 8d8a0d99bf..2b188bdca2 100644 --- a/tools/dgo_unpacker.cpp +++ b/tools/dgo_unpacker.cpp @@ -49,14 +49,10 @@ int run(int argc, char** argv) { } // namespace int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif try { return run(argc, argv); diff --git a/tools/level_tools/level_dump/main.cpp b/tools/level_tools/level_dump/main.cpp index d179453704..fdc61c8314 100644 --- a/tools/level_tools/level_dump/main.cpp +++ b/tools/level_tools/level_dump/main.cpp @@ -56,14 +56,10 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) { } int main(int argc, char** argv) { -#ifdef _WIN32 - auto args = get_widechar_cli_args(); - std::vector string_ptrs; - for (auto& str : args) { - string_ptrs.push_back(str.data()); + fs::u8arguments u8guard(argc, argv); + if (!u8guard.valid()) { + exit(EXIT_FAILURE); } - argv = string_ptrs.data(); -#endif try { fmt::print("Level Dump Tool\n"); diff --git a/vendor.yaml b/vendor.yaml index 682b905e8d..c2d7a56dfe 100644 --- a/vendor.yaml +++ b/vendor.yaml @@ -8,9 +8,17 @@ third-party/discord-rpc: sha: 963aa9f3e5ce81a4682c6ca3d136cddda614db33 third-party/fpng: sha: bfe5f9c69e93b99b31268c10db8e645c9125a07f + modifications: + - "PR #1632 - Fixes Unicode path saving on windows" third-party/CLI11.hpp: git: https://github.com/CLIUtils/CLI11/tree/v2.2.0 third-party/xxhash.hpp: git: https://github.com/RedSpah/xxhash_cpp/tree/0.7.3 third-party/xdelta3: sha: 7508fd2a823443b1f0173ca361620f21d62a7d37 +third-party/stb_image: + modifications: + - "PR #1632 - Uses UTF-8 on windows" +third-party/tiny_gltf: + modifications: + - "PR #1632 - Ensure stb_image is using UTF-8 on windows"