From 0e96cdb6c5b408487f730acb837484369eada6c4 Mon Sep 17 00:00:00 2001 From: ManDude <7569514+ManDude@users.noreply.github.com> Date: Tue, 19 Oct 2021 05:02:18 +0100 Subject: [PATCH] [de/compiler] New text tool (#918) * fix `citb-drop-plat` a bit and PAL `fisher` * Clean up `pc-pad-utils`. Looks so clean! * Increase process stacks by ~2x * Convert `game_text` custom encoding to and from a readable one (UTF-8) * clang * add missing characters * support all diacritic variants * fix a character * remaining cases * fix tests * fix memory leak? * clang * add custom characters w/ diacritics * Update all-types.gc * robustness * minor bug * move custom font decoding function to `FontUtils.cpp` * Move valid source chars patching to Reader constructor --- common/CMakeLists.txt | 1 + common/goos/Reader.cpp | 75 ++- common/goos/Reader.h | 9 +- common/util/FontUtils.cpp | 478 ++++++++++++++++++ common/util/FontUtils.h | 40 ++ decompiler/config/all-types.gc | 25 +- .../jak1_pal/anonymous_function_types.jsonc | 8 +- decompiler/config/jak1_pal/label_types.jsonc | 6 +- .../config/jak1_pal/stack_structures.jsonc | 1 + decompiler/data/game_text.cpp | 6 +- goal_src/engine/gfx/font-h.gc | 12 +- goal_src/kernel/gkernel-h.gc | 6 +- goal_src/levels/citadel/citb-drop-plat-CIT.gc | 32 +- goal_src/pc_debug/font-encode-test.gc | 104 ++++ goal_src/pc_debug/pc-pad-utils.gc | 13 +- goalc/data_compiler/game_text.cpp | 7 +- .../levels/citadel/citb-drop-plat-CIT_REF.gc | 32 +- 17 files changed, 765 insertions(+), 90 deletions(-) create mode 100644 common/util/FontUtils.cpp create mode 100644 common/util/FontUtils.h create mode 100644 goal_src/pc_debug/font-encode-test.gc diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f8c4f59676..55bcffe62b 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(common util/json_util.cpp util/Timer.cpp util/print_float.cpp + util/FontUtils.cpp ) target_link_libraries(common fmt lzokay replxx libzstd_static) diff --git a/common/goos/Reader.cpp b/common/goos/Reader.cpp index 0fde44c607..ea6b75607e 100644 --- a/common/goos/Reader.cpp +++ b/common/goos/Reader.cpp @@ -11,6 +11,7 @@ #include "Reader.h" #include "common/util/FileUtil.h" +#include "common/util/FontUtils.h" #include "third-party/fmt/core.h" #include #include "ReplUtils.h" @@ -102,6 +103,27 @@ void TextStream::seek_past_whitespace_and_comments() { } } +/*! + * Read encoding bytes on a TextStream and check if it's UTF-8. + * If it's not, you can choose to throw or not. + * If UTF-8 encoding is not detected, the stream is not advanced. + */ +void TextStream::read_utf8_encoding(bool throw_on_error) { + if (text_remains(2)) { + if ((u8)peek(0) == 0xEF && (u8)peek(1) == 0xBB && (u8)peek(2) == 0xBF) { + read(); + read(); + read(); + return; + } + } + + if (throw_on_error) { + throw std::runtime_error( + fmt::format("UTF-8 encoding not detected in {}", text->get_description())); + } +} + Reader::Reader() { // add default macros add_reader_macro("'", "quote"); @@ -142,6 +164,26 @@ Reader::Reader() { m_valid_source_text_chars[(int)'\n'] = true; m_valid_source_text_chars[(int)'\t'] = true; m_valid_source_text_chars[(int)'\r'] = true; + + // allow every character that gets transformed to something else + for (auto& remap : g_font_large_char_remap) { + for (auto rc : remap.chars) { + m_valid_source_text_chars[(u8)rc] = true; + } + } + for (auto& remap : g_font_large_string_replace) { + for (auto rc : remap.to) { + m_valid_source_text_chars[(u8)rc] = true; + } + for (auto rc : remap.from) { + m_valid_source_text_chars[(u8)rc] = true; + } + } + m_valid_source_text_chars[0] = false; +} + +bool Reader::is_valid_source_char(char c) const { + return m_valid_source_text_chars[(u8)c]; } /*! @@ -163,7 +205,7 @@ std::optional Reader::read_from_stdin(const std::string& prompt, ReplWra db.insert(textFrag); // perform read - auto result = internal_read(textFrag); + auto result = internal_read(textFrag, false); db.link(result, textFrag, 0); return result; } else { @@ -180,7 +222,7 @@ Object Reader::read_from_string(const std::string& str, bool add_top_level) { db.insert(textFrag); // perform read - auto result = internal_read(textFrag, add_top_level); + auto result = internal_read(textFrag, false, add_top_level); db.link(result, textFrag, 0); return result; } @@ -188,7 +230,7 @@ Object Reader::read_from_string(const std::string& str, bool add_top_level) { /*! * Read a file */ -Object Reader::read_from_file(const std::vector& file_path) { +Object Reader::read_from_file(const std::vector& file_path, bool check_encoding) { std::string joined_name; for (const auto& thing : file_path) { @@ -202,7 +244,7 @@ Object Reader::read_from_file(const std::vector& file_path) { auto textFrag = std::make_shared(file_util::get_file_path(file_path), joined_name); db.insert(textFrag); - auto result = internal_read(textFrag); + auto result = internal_read(textFrag, check_encoding); db.link(result, textFrag, 0); return result; } @@ -210,21 +252,34 @@ Object Reader::read_from_file(const std::vector& file_path) { /*! * Common read for a SourceText */ -Object Reader::internal_read(std::shared_ptr text, bool add_top_level) { - // validate the input; - for (int offset = 0; offset < text->get_size(); offset++) { - if (!m_valid_source_text_chars[(u8)text->get_text()[offset]]) { +Object Reader::internal_read(std::shared_ptr text, + bool check_encoding, + bool add_top_level) { + // verify UTF-8 encoding + if (check_encoding && (text->get_size() < 3 || (u8)text->get_text()[0] != 0xEF || + (u8)text->get_text()[1] != 0xBB || (u8)text->get_text()[2] != 0xBF)) { + throw std::runtime_error( + fmt::format("Text file {} has invalid encoding", text->get_description())); + } + + // validate the input + for (int offset = check_encoding ? 3 : 0; offset < text->get_size(); offset++) { + if (!is_valid_source_char(text->get_text()[offset])) { // failed. int line_number = text->get_line_idx(offset) + 1; throw std::runtime_error(fmt::format("Invalid character found on line {} of {}: 0x{:x}", line_number, text->get_description(), - (u32)text->get_text()[offset])); + (u8)text->get_text()[offset])); } } // first create stream TextStream ts(text); + if (check_encoding) { + // discard the UTF-8 encoding bytes + ts.read_utf8_encoding(true); + } // clean up first whitespace ts.seek_past_whitespace_and_comments(); @@ -239,7 +294,7 @@ Object Reader::internal_read(std::shared_ptr text, bool add_top_leve bool Reader::check_string_is_valid(const std::string& str) const { for (auto c : str) { - if (!m_valid_source_text_chars[(u8)c]) { + if (!is_valid_source_char(c)) { return false; } } diff --git a/common/goos/Reader.h b/common/goos/Reader.h index 12e9ddab84..2578fb76ea 100644 --- a/common/goos/Reader.h +++ b/common/goos/Reader.h @@ -55,6 +55,7 @@ struct TextStream { bool text_remains() { return seek < text->get_size(); } bool text_remains(int i) { return seek + i < text->get_size(); } void seek_past_whitespace_and_comments(); + void read_utf8_encoding(bool throw_on_error); }; /*! @@ -72,7 +73,7 @@ class Reader { Reader(); Object read_from_string(const std::string& str, bool add_top_level = true); std::optional read_from_stdin(const std::string& prompt, ReplWrapper& repl); - Object read_from_file(const std::vector& file_path); + Object read_from_file(const std::vector& file_path, bool check_encoding = false); bool check_string_is_valid(const std::string& str) const; std::string get_source_dir(); @@ -80,7 +81,9 @@ class Reader { TextDb db; private: - Object internal_read(std::shared_ptr text, bool add_top_level = true); + Object internal_read(std::shared_ptr text, + bool check_encoding, + bool add_top_level = true); Object read_list(TextStream& stream, bool expect_close_paren = true); bool read_object(Token& tok, TextStream& ts, Object& obj); bool read_array(TextStream& stream, Object& o); @@ -100,6 +103,8 @@ class Reader { bool m_valid_symbols_chars[256]; bool m_valid_source_text_chars[256]; + bool is_valid_source_char(char c) const; + std::unordered_map m_reader_macros; }; diff --git a/common/util/FontUtils.cpp b/common/util/FontUtils.cpp new file mode 100644 index 0000000000..dd80856bb5 --- /dev/null +++ b/common/util/FontUtils.cpp @@ -0,0 +1,478 @@ +/*! + * @file FontUtils.cpp + * + * Code for handling text and strings in Jak 1's "large font" format. + * + * MAKE SURE THIS FILE IS ENCODED IN UTF-8!!! The various strings here depend on it. + * Always verify the encoding if string detection suddenly goes awry. + */ + +#include "FontUtils.h" +#include "third-party/fmt/core.h" + +#include +#include +#include +#include + +/*! + * Remaps UTF-8 characters to the appropriate character fit for the game's large font. + * It is unfortunately quite large. + */ +std::vector g_font_large_char_remap = { + // random + {"ˇ", {0x10}}, // caron + {"`", {0x11}}, // grave accent + {"'", {0x12}}, // apostrophe + {"^", {0x13}}, // circumflex + {"", {0x14}}, // tilde + {"¨", {0x15}}, // umlaut + {"º", {0x16}}, // numero/overring + {"¡", {0x17}}, // inverted exclamation mark + {"¿", {0x18}}, // inverted question mark + + {"海", {0x1a}}, // umi + {"Æ", {0x1b}}, // aesc + {"界", {0x1c}}, // kai + {"Ç", {0x1d}}, // c-cedilla + {"学", {0x1e}}, // gaku + {"ß", {0x1f}}, // eszett + + {"\"", {0x22}}, // double-quotes + + {"ワ", {0x24}}, // wa + + {"ヲ", {0x26}}, // wo + {"ン", {0x27}}, // -n + + {"岩", {0x5c}}, // iwa + {"旧", {0x5d}}, // kyuu + {"空", {0x5e}}, // sora + //{"掘", {0x5f}}, // horu + + {"ヮ", {0x60}}, // -wa + {"撃", {0x61}}, // utsu + {"賢", {0x62}}, // kashikoi + {"湖", {0x63}}, // mizuumi + {"口", {0x64}}, // kuchi + {"行", {0x65}}, // iku + {"合", {0x66}}, // ai + {"士", {0x67}}, // shi + {"寺", {0x68}}, // tera + {"山", {0x69}}, // yama + {"者", {0x6a}}, // mono + {"所", {0x6b}}, // tokoro + {"書", {0x6c}}, // kaku + {"小", {0x6d}}, // shou + {"沼", {0x6e}}, // numa + {"上", {0x6f}}, // ue + {"城", {0x70}}, // shiro + {"場", {0x71}}, // ba + {"出", {0x72}}, // shutsu + {"闇", {0x73}}, // yami + {"遺", {0x74}}, // nokosu + {"黄", {0x75}}, // ki + {"屋", {0x76}}, // ya + {"下", {0x77}}, // shita + {"家", {0x78}}, // ie + {"火", {0x79}}, // hi + {"花", {0x7a}}, // hana + {"レ", {0x7b}}, // re + {"Œ", {0x7c}}, // oe + {"ロ", {0x7d}}, // ro + + {"青", {0x7f}}, // ao + + {"・", {0x90}}, // nakaguro + {"゛", {0x91}}, // dakuten + {"゜", {0x92}}, // handakuten + {"ー", {0x93}}, // chouompu + {"『", {0x94}}, // nijuukagikakko left + {"』", {0x95}}, // nijuukagikakko right + // hiragana + {"ぁ", {0x96}}, // -a + {"あ", {0x97}}, // a + {"ぃ", {0x98}}, // -i + {"い", {0x99}}, // i + {"ぅ", {0x9a}}, // -u + {"う", {0x9b}}, // u + {"ぇ", {0x9c}}, // -e + {"え", {0x9d}}, // e + {"ぉ", {0x9e}}, // -o + {"お", {0x9f}}, // o + {"か", {0xa0}}, // ka + {"き", {0xa1}}, // ki + {"く", {0xa2}}, // ku + {"け", {0xa3}}, // ke + {"こ", {0xa4}}, // ko + {"さ", {0xa5}}, // sa + {"し", {0xa6}}, // shi + {"す", {0xa7}}, // su + {"せ", {0xa8}}, // se + {"そ", {0xa9}}, // so + {"た", {0xaa}}, // ta + {"ち", {0xab}}, // chi + {"っ", {0xac}}, // sokuon + {"つ", {0xad}}, // tsu + {"て", {0xae}}, // te + {"と", {0xaf}}, // to + {"な", {0xb0}}, // na + {"に", {0xb1}}, // ni + {"ぬ", {0xb2}}, // nu + {"ね", {0xb3}}, // ne + {"の", {0xb4}}, // no + {"は", {0xb5}}, // ha + {"ひ", {0xb6}}, // hi + {"ふ", {0xb7}}, // hu + {"へ", {0xb8}}, // he + {"ほ", {0xb9}}, // ho + {"ま", {0xba}}, // ma + {"み", {0xbb}}, // mi + {"む", {0xbc}}, // mu + {"め", {0xbd}}, // me + {"も", {0xbe}}, // mo + {"ゃ", {0xbf}}, // youon ya + {"や", {0xc0}}, // ya + {"ゅ", {0xc1}}, // youon yu + {"ゆ", {0xc2}}, // yu + {"ょ", {0xc3}}, // youon yo + {"よ", {0xc4}}, // yo + {"ら", {0xc5}}, // ra + {"り", {0xc6}}, // ri + {"る", {0xc7}}, // ru + {"れ", {0xc8}}, // re + {"ろ", {0xc9}}, // ro + {"ゎ", {0xca}}, // -wa + {"わ", {0xcb}}, // wa + {"を", {0xcc}}, // wo + {"ん", {0xcd}}, // -n + // katakana + {"ァ", {0xce}}, // -a + {"ア", {0xcf}}, // a + {"ィ", {0xd0}}, // -i + {"イ", {0xd1}}, // i + {"ゥ", {0xd2}}, // -u + {"ウ", {0xd3}}, // u + {"ェ", {0xd4}}, // -e + {"エ", {0xd5}}, // e + {"ォ", {0xd6}}, // -o + {"オ", {0xd7}}, // o + {"カ", {0xd8}}, // ka + {"キ", {0xd9}}, // ki + {"ク", {0xda}}, // ku + {"ケ", {0xdb}}, // ke + {"コ", {0xdc}}, // ko + {"サ", {0xdd}}, // sa + {"シ", {0xde}}, // shi + {"ス", {0xdf}}, // su + {"セ", {0xe0}}, // se + {"ソ", {0xe1}}, // so + {"タ", {0xe2}}, // ta + {"チ", {0xe3}}, // chi + {"ッ", {0xe4}}, // sokuon + {"ツ", {0xe5}}, // tsu + {"テ", {0xe6}}, // te + {"ト", {0xe7}}, // to + {"ナ", {0xe8}}, // na + {"ニ", {0xe9}}, // ni + {"ヌ", {0xea}}, // nu + {"ネ", {0xeb}}, // ne + {"ノ", {0xec}}, // no + {"ハ", {0xed}}, // ha + {"ヒ", {0xee}}, // hi + {"フ", {0xef}}, // hu + {"ヘ", {0xf0}}, // he + {"ホ", {0xf1}}, // ho + {"マ", {0xf2}}, // ma + {"ミ", {0xf3}}, // mi + {"ム", {0xf4}}, // mu + {"メ", {0xf5}}, // me + {"モ", {0xf6}}, // mo + {"ャ", {0xf7}}, // youon ya + {"ヤ", {0xf8}}, // ya + {"ュ", {0xf9}}, // youon yu + {"ユ", {0xfa}}, // yu + {"ョ", {0xfb}}, // youon yo + {"ヨ", {0xfc}}, // yo + {"ラ", {0xfd}}, // ra + {"リ", {0xfe}}, // ri + {"ル", {0xff}}, // ru + // kanji 2 + {"宝", {1, 0x01}}, // takara + + {"石", {1, 0x10}}, // ishi + {"赤", {1, 0x11}}, // aka + {"跡", {1, 0x12}}, // ato + {"川", {1, 0x13}}, // kawa + {"戦", {1, 0x14}}, // ikusa + {"村", {1, 0x15}}, // mura + {"隊", {1, 0x16}}, // tai + {"台", {1, 0x17}}, // utena + {"長", {1, 0x18}}, // osa + {"鳥", {1, 0x19}}, // tori + {"艇", {1, 0x1a}}, // tei + {"洞", {1, 0x1b}}, // hora + {"道", {1, 0x1c}}, // michi + {"発", {1, 0x1d}}, // hatsu + {"飛", {1, 0x1e}}, // tobu + {"噴", {1, 0x1f}}, // fuku + + {"池", {1, 0xa0}}, // ike + {"中", {1, 0xa1}}, // naka + {"塔", {1, 0xa2}}, // tou + {"島", {1, 0xa3}}, // shima + {"部", {1, 0xa4}}, // bu + {"砲", {1, 0xa5}}, // hou + {"産", {1, 0xa6}}, // san + {"眷", {1, 0xa7}}, // kaerimiru + {"力", {1, 0xa8}}, // chikara + {"緑", {1, 0xa9}}, // midori + {"岸", {1, 0xaa}}, // kishi + {"像", {1, 0xab}}, // zou + {"谷", {1, 0xac}}, // tani + {"心", {1, 0xad}}, // kokoro + {"森", {1, 0xae}}, // mori + {"水", {1, 0xaf}}, // mizu + {"船", {1, 0xb0}}, // fune + {"™", {1, 0xb1}}, // trademark +}; +/*! + * Replaces specific UTF-8 strings with more readable variants. + */ +std::vector g_font_large_string_replace = { + // \" -> " (confusing) + {"\\\"", "\""}, + + // other + {"A~Y~-21H~-5Vº~Z", "Å"}, + {"N~Y~-6Hº~Z~+10H", "Nº"}, + + // tildes + {"N~Y~-22H~-4V~Z", "Ñ"}, + {"A~Y~-21H~-5V~Z", "Ã"}, // custom + {"O~Y~-22H~-4V~Z", "Õ"}, // custom + + // acute accents + {"A~Y~-21H~-5V'~Z", "Á"}, + {"E~Y~-22H~-5V'~Z", "É"}, + {"I~Y~-19H~-5V'~Z", "Í"}, + {"O~Y~-22H~-4V'~Z", "Ó"}, + {"U~Y~-24H~-3V'~Z", "Ú"}, + + // circumflex + {"A~Y~-20H~-4V^~Z", "Â"}, // custom + {"E~Y~-20H~-5V^~Z", "Ê"}, + {"I~Y~-19H~-5V^~Z", "Î"}, + {"O~Y~-20H~-4V^~Z", "Ô"}, // custom + {"U~Y~-24H~-3V^~Z", "Û"}, + + // grave accents + {"A~Y~-21H~-5V`~Z", "À"}, + {"E~Y~-22H~-5V`~Z", "È"}, + {"I~Y~-19H~-5V`~Z", "Ì"}, + {"O~Y~-22H~-4V`~Z", "Ò"}, // custom + {"U~Y~-24H~-3V`~Z", "Ù"}, + + // umlaut + {"A~Y~-21H~-5V¨~Z", "Ä"}, + {"E~Y~-20H~-5V¨~Z", "Ë"}, + {"I~Y~-19H~-5V¨~Z", "Ï"}, // custom + {"O~Y~-22H~-4V¨~Z", "Ö"}, + {"O~Y~-22H~-3V¨~Z", "ö"}, // dumb + {"U~Y~-22H~-3V¨~Z", "Ü"}, + + // dakuten katakana + {"~Yウ~Z゛", "ヴ"}, + {"~Yカ~Z゛", "ガ"}, + {"~Yキ~Z゛", "ギ"}, + {"~Yク~Z゛", "グ"}, + {"~Yケ~Z゛", "ゲ"}, + {"~Yコ~Z゛", "ゴ"}, + {"~Yサ~Z゛", "ザ"}, + {"~Yシ~Z゛", "ジ"}, + {"~Yス~Z゛", "ズ"}, + {"~Yセ~Z゛", "ゼ"}, + {"~Yソ~Z゛", "ゾ"}, + {"~Yタ~Z゛", "ダ"}, + {"~Yチ~Z゛", "ヂ"}, + {"~Yツ~Z゛", "ヅ"}, + {"~Yテ~Z゛", "デ"}, + {"~Yト~Z゛", "ド"}, + {"~Yハ~Z゛", "バ"}, + {"~Yヒ~Z゛", "ビ"}, + {"~Yフ~Z゛", "ブ"}, + {"~Yヘ~Z゛", "ベ"}, + {"~Yホ~Z゛", "ボ"}, + // handakuten katakana + {"~Yハ~Z゜", "パ"}, + {"~Yヒ~Z゜", "ピ"}, + {"~Yフ~Z゜", "プ"}, + {"~Yヘ~Z゜", "ペ"}, + {"~Yホ~Z゜", "ポ"}, + // dakuten hiragana + {"~Yか~Z゛", "が"}, + {"~Yき~Z゛", "ぎ"}, + {"~Yく~Z゛", "ぐ"}, + {"~Yけ~Z゛", "げ"}, + {"~Yこ~Z゛", "ご"}, + {"~Yさ~Z゛", "ざ"}, + {"~Yし~Z゛", "じ"}, + {"~Yす~Z゛", "ず"}, + {"~Yせ~Z゛", "ぜ"}, + {"~Yそ~Z゛", "ぞ"}, + {"~Yた~Z゛", "だ"}, + {"~Yち~Z゛", "ぢ"}, + {"~Yつ~Z゛", "づ"}, + {"~Yて~Z゛", "で"}, + {"~Yと~Z゛", "ど"}, + {"~Yは~Z゛", "ば"}, + {"~Yひ~Z゛", "び"}, + {"~Yふ~Z゛", "ぶ"}, + {"~Yへ~Z゛", "べ"}, + {"~Yほ~Z゛", "ぼ"}, + // handakuten hiragana + {"~Yは~Z゜", "ぱ"}, + {"~Yひ~Z゜", "ぴ"}, + {"~Yふ~Z゜", "ぷ"}, + {"~Yへ~Z゜", "ぺ"}, + {"~Yほ~Z゜", "ぽ"}, + // japanese punctuation + {",~+8H", "、"}, + {"~+8H ", "　"}, + + // (hack) special case kanji + {"~~", "世"}, + + // playstation buttons + {"~Y~22L<~Z~Y~27L*~Z~Y~1L>~Z~Y~23L[~Z~+26H", ""}, + {"~Y~22L<~Z~Y~26L;~Z~Y~1L>~Z~Y~23L[~Z~+26H", ""}, + {"~Y~22L<~Z~Y~25L@~Z~Y~1L>~Z~Y~23L[~Z~+26H", ""}, + {"~Y~22L<~Z~Y~24L#~Z~Y~1L>~Z~Y~23L[~Z~+26H", ""}, // custom +}; + +static bool remaps_inited = false; +static void init_remaps() { + if (!remaps_inited) { + std::sort( + g_font_large_char_remap.begin(), g_font_large_char_remap.end(), + [](const RemapInfo& a, const RemapInfo& b) { return a.bytes.size() > b.bytes.size(); }); + std::sort( + g_font_large_string_replace.begin(), g_font_large_string_replace.end(), + [](const ReplaceInfo& a, const ReplaceInfo& b) { return a.from.size() > b.from.size(); }); + remaps_inited = true; + } +} + +/*! + * Convert Jak 1 character encoding to something readable. + */ +RemapInfo* jak1_bytes_to_utf8(const char* in) { + init_remaps(); + for (auto& info : g_font_large_char_remap) { + if (info.bytes.size() == 0) + continue; + bool found = true; + for (int i = 0; found && i < info.bytes.size(); ++i) { + if (uint8_t(in[i]) != info.bytes.at(i)) { + found = false; + } + } + if (found) { + return &info; + } + } + return nullptr; +} + +/*! + * Try to replace specific substrings with better variants. + * These are for hiding confusing text transforms. + */ +std::string& jak1_trans_to_utf8(std::string& str) { + init_remaps(); + for (auto& info : g_font_large_string_replace) { + auto pos = str.find(info.from); + while (pos != std::string::npos) { + str.replace(pos, info.from.size(), info.to); + pos = str.find(info.from, pos + info.to.size()); + } + } + return str; +} + +std::string& utf8_trans_to_jak1(std::string& str) { + init_remaps(); + for (auto& info : g_font_large_string_replace) { + auto pos = str.find(info.to); + while (pos != std::string::npos) { + str.replace(pos, info.to.size(), info.from); + pos = str.find(info.to, pos + info.from.size()); + } + } + return str; +} + +std::string& utf8_bytes_to_jak1(std::string& str) { + // find all instances of characters and save them + std::map> remap_cache; + for (auto& info : g_font_large_char_remap) { + auto pos = str.find(info.chars); + while (pos != std::string::npos) { + remap_cache[pos] = &info; + pos = str.find(info.chars, pos + info.chars.size()); + } + } + + // go through the string backwards and replace saved chars + for (auto& remap : remap_cache) { + std::string temp; + for (auto b : remap.second->bytes) { + temp.push_back(b); + } + str.replace(remap.first, remap.second->chars.size(), temp); + } + + return str; +} + +/*! + * Turn a normal readable string into a string readable in the Jak 1 font encoding. + */ +std::string convert_to_jak1_encoding(std::string str) { + utf8_trans_to_jak1(str); + utf8_bytes_to_jak1(str); + return str; +} + +static const std::unordered_set passthrus = {'~', ' ', ',', '.', '-', '+', '(', ')', + '!', ':', '?', '=', '%', '*', '/', '#', + ';', '<', '>', '@', '[', '_'}; + +/*! + * Convert a string from the Jak 1 large font encoding to something normal. + * Unprintable characters become escape sequences, including tab and newline. + */ +std::string convert_from_jak1_encoding(const char* in) { + std::string result; + while (*in) { + auto remap = jak1_bytes_to_utf8(in); + if (remap != nullptr) { + result.append(remap->chars); + in += remap->bytes.size() - 1; + } else if (((*in >= '0' && *in <= '9') || (*in >= 'A' && *in <= 'Z') || + passthrus.find(*in) != passthrus.end()) && + *in != '\\') { + result.push_back(*in); + } else if (*in == '\n') { + result += "\\n"; + } else if (*in == '\t') { + result += "\\t"; + } else if (*in == '\\') { + result += "\\\\"; + } else { + result += fmt::format("\\c{:02x}", uint8_t(*in)); + } + in++; + } + return jak1_trans_to_utf8(result); +} diff --git a/common/util/FontUtils.h b/common/util/FontUtils.h new file mode 100644 index 0000000000..09151f8df3 --- /dev/null +++ b/common/util/FontUtils.h @@ -0,0 +1,40 @@ +#pragma once + +/*! + * @file FontUtils.h + * + * Code for handling text and strings in Jak 1's "large font" format. + * + * MAKE SURE THIS FILE IS ENCODED IN UTF-8!!! The various strings here depend on it. + * Always verify the encoding if string detection suddenly goes awry. + */ + +#include "common/common_types.h" + +#include +#include + +struct RemapInfo { + std::string chars; + std::vector bytes; +}; +struct ReplaceInfo { + std::string from; + std::string to; +}; + +/*! + * Remaps UTF-8 characters to the appropriate character fit for the game's large font. + * It is unfortunately quite large. + */ +extern std::vector g_font_large_char_remap; +/*! + * Replaces specific UTF-8 strings with more readable variants. + */ +extern std::vector g_font_large_string_replace; + +RemapInfo* jak1_bytes_to_utf8(const char* in); +std::string& jak1_trans_to_utf8(std::string& str); + +std::string convert_to_jak1_encoding(std::string str); +std::string convert_from_jak1_encoding(const char* in); diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index 7409718c5c..eeff4ac980 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -5358,12 +5358,12 @@ (flat-dark-purple 19) (flat-yellow 20) (blue-white 21) - (flat-dark-gray 22) - (flat-gray 23) - (flat-pink 24) - (flat-red 25) - (flat-green 26) - (flat-purple 27) + (pad-back 22) + (pad-shine 23) + (pad-square 24) + (pad-circle 25) + (pad-triangle 26) + (pad-x 27) (lighter-lighter-blue 28) (yellow-orange 29) (yellow-green-2 30) @@ -5666,7 +5666,7 @@ (define-extern *text-group-names* (array string)) (define-extern kheap type) (define-extern *common-text-heap* kheap) -(define-extern *common-text* game-text-info) ; TODO guess, but its definitely not just a symbol! +(define-extern *common-text* game-text-info) ;; ---------------------- @@ -5897,7 +5897,6 @@ (define-extern adgif-shader-login-no-remap (function adgif-shader texture)) (define-extern adgif-shader-login-fast (function adgif-shader texture)) (define-extern adgif-shader-login-no-remap-fast (function adgif-shader texture)) -;;;; unknown type (define-extern adgif-shader<-texture-simple! (function adgif-shader texture adgif-shader)) ;; - Symbols @@ -25992,8 +25991,8 @@ (spin-angle float :offset-assert 192) (spin-speed float :offset-assert 196) (interp float :offset-assert 200) - (duration uint64 :offset-assert 208) - (delay uint64 :offset-assert 216) + (duration int64 :offset-assert 208) + (delay int64 :offset-assert 216) (color int8 :offset-assert 224) ) :method-count-assert 22 @@ -26026,8 +26025,8 @@ (x-spacing float :offset-assert 256) (z-spacing float :offset-assert 260) (idle-distance float :offset-assert 264) - (duration uint64 :offset-assert 272) - (drop-time uint64 :offset-assert 280) + (duration int64 :offset-assert 272) + (drop-time int64 :offset-assert 280) ) :method-count-assert 20 :size-assert #x120 @@ -26040,7 +26039,7 @@ (define-extern citb-drop-plat-spawn-children (function none :behavior citb-drop-plat)) (define-extern citb-drop-plat-drop-children (function int none :behavior citb-drop-plat)) (define-extern citb-drop-plat-drop-all-children (function symbol :behavior citb-drop-plat)) -(define-extern drop-plat-init-by-other (function vector uint uint int none :behavior drop-plat)) +(define-extern drop-plat-init-by-other (function vector int int int none :behavior drop-plat)) (define-extern drop-plat-set-fade (function none :behavior drop-plat)) ;; - Unknowns diff --git a/decompiler/config/jak1_pal/anonymous_function_types.jsonc b/decompiler/config/jak1_pal/anonymous_function_types.jsonc index 55ee7b498b..dd06f46574 100644 --- a/decompiler/config/jak1_pal/anonymous_function_types.jsonc +++ b/decompiler/config/jak1_pal/anonymous_function_types.jsonc @@ -761,13 +761,7 @@ [29, "(function none)"] ], - "fisher-JUN": [ - [27, "(function none :behavior process)"], - [28, "(function none :behavior fisher)"], - [20, "(function none :behavior fisher-fish)"], - [1, "(function none :behavior target)"] - ], - "fisher-JUNGLE-L1": [ + "fisher": [ [27, "(function none :behavior process)"], [28, "(function none :behavior fisher)"], [20, "(function none :behavior fisher-fish)"], diff --git a/decompiler/config/jak1_pal/label_types.jsonc b/decompiler/config/jak1_pal/label_types.jsonc index a937e89fe0..e31781ce92 100644 --- a/decompiler/config/jak1_pal/label_types.jsonc +++ b/decompiler/config/jak1_pal/label_types.jsonc @@ -872,11 +872,7 @@ ["L670", "uint64", true] ], - "fisher-JUN": [ - ["L262", "vector"], - ["L463", "(array (inline-array fisher-params))"] - ], - "fisher-JUNGLE-L1": [ + "fisher": [ ["L262", "vector"], ["L463", "(array (inline-array fisher-params))"] ], diff --git a/decompiler/config/jak1_pal/stack_structures.jsonc b/decompiler/config/jak1_pal/stack_structures.jsonc index e4e01fa588..59ce0b6146 100644 --- a/decompiler/config/jak1_pal/stack_structures.jsonc +++ b/decompiler/config/jak1_pal/stack_structures.jsonc @@ -2820,6 +2820,7 @@ [32, "vector"] ], "fisher-draw-display": [[16, "font-context"]], + "(trans fisher-done)": [[16, "font-context"]], "(method 10 torus)": [ [16, "vector"], diff --git a/decompiler/data/game_text.cpp b/decompiler/data/game_text.cpp index 96c47a4568..aa30d4a46e 100644 --- a/decompiler/data/game_text.cpp +++ b/decompiler/data/game_text.cpp @@ -7,6 +7,7 @@ #include "decompiler/ObjectFile/ObjectFileDB.h" #include "common/goos/Reader.h" #include "common/util/BitUtils.h" +#include "common/util/FontUtils.h" namespace decompiler { namespace { @@ -98,7 +99,7 @@ GameTextResult process_game_text(ObjectFileData& data) { } // escape characters - result.text[text_id] = goos::get_readable_string(text.c_str()); + result.text[text_id] = convert_from_jak1_encoding(text.c_str()); // remember what we read (-1 for the type tag) auto string_start = (text_label.offset / 4) - 1; @@ -146,7 +147,8 @@ std::string write_game_text( } // write! - std::string result = fmt::format("(language-count {})\n", langauges.size()); + std::string result; // = "\xEF\xBB\xBF"; // UTF-8 encode (don't need this anymore) + result += fmt::format("(language-count {})\n", langauges.size()); result += "(group-name \"common\")\n"; for (auto& x : text_by_id) { result += fmt::format("(#x{:04x}\n ", x.first); diff --git a/goal_src/engine/gfx/font-h.gc b/goal_src/engine/gfx/font-h.gc index 8878e0fb90..78a83193fc 100644 --- a/goal_src/engine/gfx/font-h.gc +++ b/goal_src/engine/gfx/font-h.gc @@ -32,12 +32,12 @@ (flat-dark-purple 19) (flat-yellow 20) (blue-white 21) - (flat-dark-gray 22) - (flat-gray 23) - (flat-pink 24) - (flat-red 25) - (flat-green 26) - (flat-purple 27) + (pad-back 22) + (pad-shine 23) + (pad-square 24) + (pad-circle 25) + (pad-triangle 26) + (pad-x 27) (lighter-lighter-blue 28) (yellow-orange 29) (yellow-green-2 30) diff --git a/goal_src/kernel/gkernel-h.gc b/goal_src/kernel/gkernel-h.gc index a80ca3f374..fb0b040ae1 100644 --- a/goal_src/kernel/gkernel-h.gc +++ b/goal_src/kernel/gkernel-h.gc @@ -33,10 +33,12 @@ ;; -memory- ;; the size of the execution stack (~14 kB) shared by all threads -(defconstant DPROCESS_STACK_SIZE #x3800) +;; OpenGOAL NOTE: increased to 32kB +(defconstant DPROCESS_STACK_SIZE #x8000) ;; another stack size used as a maximum for temporary threads -(defconstant PROCESS_STACK_SIZE #x1c00) +;; OpenGOAL NOTE: increased to 16kB +(defconstant PROCESS_STACK_SIZE #x4000) ;; default size of stack to backup for a process (defconstant PROCESS_STACK_SAVE_SIZE 256) diff --git a/goal_src/levels/citadel/citb-drop-plat-CIT.gc b/goal_src/levels/citadel/citb-drop-plat-CIT.gc index e35b945829..76d815c3f6 100644 --- a/goal_src/levels/citadel/citb-drop-plat-CIT.gc +++ b/goal_src/levels/citadel/citb-drop-plat-CIT.gc @@ -124,8 +124,8 @@ (spin-angle float :offset-assert 192) (spin-speed float :offset-assert 196) (interp float :offset-assert 200) - (duration uint64 :offset-assert 208) - (delay uint64 :offset-assert 216) + (duration int64 :offset-assert 208) + (delay int64 :offset-assert 216) (color int8 :offset-assert 224) ) :heap-base #x80 @@ -171,7 +171,7 @@ (if (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (-> self duration)) + (-> self duration) ) (go drop-plat-drop) ) @@ -230,7 +230,7 @@ (when (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (-> self delay)) + (-> self delay) ) (let ((v1-14 (logand -3 (-> self draw status))) (a0-5 (-> self draw)) @@ -503,7 +503,7 @@ (defbehavior drop-plat-init-by-other drop-plat - ((arg0 vector) (arg1 uint) (arg2 uint) (arg3 int)) + ((arg0 vector) (arg1 int) (arg2 int) (arg3 int)) (set! (-> self color) arg3) (set! (-> self delay) arg1) (set! (-> self duration) arg2) @@ -542,8 +542,8 @@ (x-spacing float :offset-assert 256) (z-spacing float :offset-assert 260) (idle-distance float :offset-assert 264) - (duration uint64 :offset-assert 272) - (drop-time uint64 :offset-assert 280) + (duration int64 :offset-assert 272) + (drop-time int64 :offset-assert 280) ) :heap-base #xb0 :method-count-assert 20 @@ -654,7 +654,10 @@ uint sv-64 ) - t0-0 + (the-as + uint + t0-0 + ) ) ) (-> @@ -676,7 +679,7 @@ ) ) ) - (set! (-> self drop-time) (the-as uint (-> *display* base-frame-counter))) + (set! (-> self drop-time) (-> *display* base-frame-counter)) 0 (none) ) @@ -741,15 +744,12 @@ ((= v1-0 'player-stepped) (when (>= - (- - (-> *display* base-frame-counter) - (the-as int (-> self drop-time)) - ) + (- (-> *display* base-frame-counter) (-> self drop-time)) 60 ) (set! (-> self drop-time) - (the-as uint (-> *display* base-frame-counter)) + (-> *display* base-frame-counter) ) (citb-drop-plat-drop-children (the-as int (-> arg3 param 0)) @@ -772,7 +772,7 @@ (or (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (+ (-> self duration) 600)) + (+ (-> self duration) 600) ) (or (not *target*) @@ -825,7 +825,7 @@ ) (set! (-> obj duration) - (the-as uint (the int (* 300.0 (+ 2.0 (the float (-> obj z-count)))))) + (the int (* 300.0 (+ 2.0 (the float (-> obj z-count))))) ) (let ((f0-7 (res-lump-float arg0 'rotoffset))) (quaternion-rotate-y! (-> obj root quat) (-> obj root quat) f0-7) diff --git a/goal_src/pc_debug/font-encode-test.gc b/goal_src/pc_debug/font-encode-test.gc new file mode 100644 index 0000000000..69cdc4342c --- /dev/null +++ b/goal_src/pc_debug/font-encode-test.gc @@ -0,0 +1,104 @@ +;;-*-Lisp-*- +(in-package goal) + + +;; This file is used for debugging and testing the large font encoding. +;; This file should *not* be included as part of any packages, it should be manually loaded by the user. + +;; To run this: + +#| +(make-group "iso") ;; build the game +(lt) ;; connect to the runtime +(lg) ;; have the runtime load the game engine +(test-play) ;; start the game loop +(ml "goal_src/pc_debug/font-encode-test.gc") ;; build and load this file. +|# + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; constants +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defconstant FONT_ENCODE_TEXT_LEFT 56) +(defconstant FONT_ENCODE_TEXT_Y 80) + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; functions +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(define *font-string* (new 'global 'string 64 (the-as string #f))) +(define *font-string-ex* "") +(define *font-string-val* #x96) + +(defun-debug font-encode-test-start () + "start the encode test proc" + + (unless (process-by-name 'font-encode *active-pool*) + (make-function-process process :name 'font-encode + (lambda :behavior process () + + (stack-size-set! (-> self main-thread) 768) + + (let ((fnt (new 'stack 'font-context *font-default-matrix* FONT_ENCODE_TEXT_LEFT FONT_ENCODE_TEXT_Y 0.0 + (font-color orange-red) (font-flags shadow kerning large middle))) + ) + + (set-width! fnt 400) + (set-height! fnt 100) + + (loop + (suspend) + + (if (or (cpad-pressed? 0 left) (cpad-hold? 0 l1)) + (-! *font-string-val* 1) + ) + (if (or (cpad-pressed? 0 right) (cpad-hold? 0 r1)) + (+! *font-string-val* 1) + ) + (if (< *font-string-val* 1) + (set! *font-string-val* 1) + ) + (if (> *font-string-val* #x1ff) + (set! *font-string-val* #x1ff) + ) + + (clear *font-string*) + (cond + ((>= *font-string-val* #x100) + (set! (-> *font-string* data 0) (/ *font-string-val* 256)) + (set! (-> *font-string* data 1) (mod *font-string-val* 256)) + (set! (-> *font-string* data 2) 0) + ) + (else + (set! (-> *font-string* data 0) (mod *font-string-val* 256)) + (set! (-> *font-string* data 1) 0) + ) + ) + + (set-origin! fnt FONT_ENCODE_TEXT_LEFT FONT_ENCODE_TEXT_Y) + (set-flags! fnt (font-flags shadow kerning large middle)) + (print-game-text *font-string* fnt #f 128 24) + (set-origin! fnt FONT_ENCODE_TEXT_LEFT (+ FONT_ENCODE_TEXT_Y 32)) + (print-game-text *font-string-ex* fnt #f 128 24) + (set-origin! fnt FONT_ENCODE_TEXT_LEFT (- FONT_ENCODE_TEXT_Y 16)) + (set-flags! fnt (font-flags shadow kerning middle)) + (print-game-text (string-format "#x~X" *font-string-val*) fnt #f 128 12) + ) + ) + + ) + ) + ) + ) + +(defun-debug font-encode-test-stop () + "stop the encode test proc" + + (kill-by-name 'font-encode *active-pool*) + ) + + diff --git a/goal_src/pc_debug/pc-pad-utils.gc b/goal_src/pc_debug/pc-pad-utils.gc index 4d6d77057c..6b5aff5ad9 100644 --- a/goal_src/pc_debug/pc-pad-utils.gc +++ b/goal_src/pc_debug/pc-pad-utils.gc @@ -28,6 +28,7 @@ (input handle) ) ) + (define *pc-pad-proc-list* (new 'static 'pc-pad-proc-list)) (set! (-> *pc-pad-proc-list* show) (the handle #f)) (set! (-> *pc-pad-proc-list* input) (the handle #f)) @@ -39,6 +40,9 @@ (pad-idx uint64) ) :heap-base #x20 + (:states + pc-pi-mapping-button + ) ) (define *pc-pad-button-names* @@ -64,7 +68,7 @@ "SQUARE" )) -;; there is a matching enum in newpad +;; there is a matching enum in newpad.h (defenum pc-pad-input-status (disabled) (enabled) @@ -72,13 +76,6 @@ ) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;; forward declarations -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(define-extern pc-pi-mapping-button (state pc-pad-proc)) - - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; constants diff --git a/goalc/data_compiler/game_text.cpp b/goalc/data_compiler/game_text.cpp index 92fad52a6d..3919f6c62a 100644 --- a/goalc/data_compiler/game_text.cpp +++ b/goalc/data_compiler/game_text.cpp @@ -12,6 +12,7 @@ #include "common/goos/Reader.h" #include "DataObjectGenerator.h" #include "common/util/FileUtil.h" +#include "common/util/FontUtils.h" #include "third-party/fmt/core.h" namespace { @@ -56,7 +57,7 @@ std::string get_string(const goos::Object& x) { if (x.is_string()) { return x.as_string()->data; } - throw std::runtime_error(x.print() + " was supposed to be an string, but isn't"); + throw std::runtime_error(x.print() + " was supposed to be a string, but isn't"); } std::string uppercase(const std::string& in) { @@ -89,7 +90,7 @@ std::vector> parse(const goos::Object& data for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) { if (obj.is_pair()) { - auto head = obj.as_pair()->car; + auto& head = obj.as_pair()->car; if (head.is_symbol() && head.as_symbol()->name == "language-count") { if (languages_set) { throw std::runtime_error("Languages has been set multiple times."); @@ -126,7 +127,7 @@ std::vector> parse(const goos::Object& data throw std::runtime_error("Entry appears more than once"); } - map[id] = entry.as_string()->data; + map[id] = convert_to_jak1_encoding(entry.as_string()->data); } else { throw std::runtime_error("Each entry must be a string"); } diff --git a/test/decompiler/reference/levels/citadel/citb-drop-plat-CIT_REF.gc b/test/decompiler/reference/levels/citadel/citb-drop-plat-CIT_REF.gc index fa4b22acbe..d22655f4cd 100644 --- a/test/decompiler/reference/levels/citadel/citb-drop-plat-CIT_REF.gc +++ b/test/decompiler/reference/levels/citadel/citb-drop-plat-CIT_REF.gc @@ -108,8 +108,8 @@ (spin-angle float :offset-assert 192) (spin-speed float :offset-assert 196) (interp float :offset-assert 200) - (duration uint64 :offset-assert 208) - (delay uint64 :offset-assert 216) + (duration int64 :offset-assert 208) + (delay int64 :offset-assert 216) (color int8 :offset-assert 224) ) :heap-base #x80 @@ -170,7 +170,7 @@ (if (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (-> self duration)) + (-> self duration) ) (go drop-plat-drop) ) @@ -232,7 +232,7 @@ (when (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (-> self delay)) + (-> self delay) ) (let ((v1-14 (logand -3 (-> self draw status))) (a0-5 (-> self draw)) @@ -514,7 +514,7 @@ ;; Used lq/sq (defbehavior drop-plat-init-by-other drop-plat - ((arg0 vector) (arg1 uint) (arg2 uint) (arg3 int)) + ((arg0 vector) (arg1 int) (arg2 int) (arg3 int)) (set! (-> self color) arg3) (set! (-> self delay) arg1) (set! (-> self duration) arg2) @@ -564,8 +564,8 @@ (x-spacing float :offset-assert 256) (z-spacing float :offset-assert 260) (idle-distance float :offset-assert 264) - (duration uint64 :offset-assert 272) - (drop-time uint64 :offset-assert 280) + (duration int64 :offset-assert 272) + (drop-time int64 :offset-assert 280) ) :heap-base #xb0 :method-count-assert 20 @@ -701,7 +701,10 @@ uint sv-64 ) - t0-0 + (the-as + uint + t0-0 + ) ) ) (-> @@ -723,7 +726,7 @@ ) ) ) - (set! (-> self drop-time) (the-as uint (-> *display* base-frame-counter))) + (set! (-> self drop-time) (-> *display* base-frame-counter)) 0 (none) ) @@ -793,15 +796,12 @@ ((= v1-0 'player-stepped) (when (>= - (- - (-> *display* base-frame-counter) - (the-as int (-> self drop-time)) - ) + (- (-> *display* base-frame-counter) (-> self drop-time)) 60 ) (set! (-> self drop-time) - (the-as uint (-> *display* base-frame-counter)) + (-> *display* base-frame-counter) ) (citb-drop-plat-drop-children (the-as int (-> arg3 param 0)) @@ -824,7 +824,7 @@ (or (>= (- (-> *display* base-frame-counter) (-> self state-time)) - (the-as int (+ (-> self duration) 600)) + (+ (-> self duration) 600) ) (or (not *target*) @@ -879,7 +879,7 @@ ) (set! (-> obj duration) - (the-as uint (the int (* 300.0 (+ 2.0 (the float (-> obj z-count)))))) + (the int (* 300.0 (+ 2.0 (the float (-> obj z-count))))) ) (let ((f0-7 (res-lump-float arg0 'rotoffset))) (quaternion-rotate-y! (-> obj root quat) (-> obj root quat) f0-7)