[de/compiler] New text tool (#918)

* fix `citb-drop-plat` a bit and PAL `fisher`

* Clean up `pc-pad-utils`. Looks so clean!

* Increase process stacks by ~2x

* Convert `game_text` custom encoding to and from a readable one (UTF-8)

* clang

* add missing characters

* support all diacritic variants

* fix a character

* remaining cases

* fix tests

* fix memory leak?

* clang

* add custom characters w/ diacritics

* Update all-types.gc

* robustness

* minor bug

* move custom font decoding function to `FontUtils.cpp`

* Move valid source chars patching to Reader constructor
This commit is contained in:
ManDude
2021-10-19 05:02:18 +01:00
committed by GitHub
parent 4c480e6970
commit 0e96cdb6c5
17 changed files with 765 additions and 90 deletions
+1
View File
@@ -29,6 +29,7 @@ add_library(common
util/json_util.cpp
util/Timer.cpp
util/print_float.cpp
util/FontUtils.cpp
)
target_link_libraries(common fmt lzokay replxx libzstd_static)
+65 -10
View File
@@ -11,6 +11,7 @@
#include "Reader.h"
#include "common/util/FileUtil.h"
#include "common/util/FontUtils.h"
#include "third-party/fmt/core.h"
#include <filesystem>
#include "ReplUtils.h"
@@ -102,6 +103,27 @@ void TextStream::seek_past_whitespace_and_comments() {
}
}
/*!
* Read encoding bytes on a TextStream and check if it's UTF-8.
* If it's not, you can choose to throw or not.
* If UTF-8 encoding is not detected, the stream is not advanced.
*/
void TextStream::read_utf8_encoding(bool throw_on_error) {
if (text_remains(2)) {
if ((u8)peek(0) == 0xEF && (u8)peek(1) == 0xBB && (u8)peek(2) == 0xBF) {
read();
read();
read();
return;
}
}
if (throw_on_error) {
throw std::runtime_error(
fmt::format("UTF-8 encoding not detected in {}", text->get_description()));
}
}
Reader::Reader() {
// add default macros
add_reader_macro("'", "quote");
@@ -142,6 +164,26 @@ Reader::Reader() {
m_valid_source_text_chars[(int)'\n'] = true;
m_valid_source_text_chars[(int)'\t'] = true;
m_valid_source_text_chars[(int)'\r'] = true;
// allow every character that gets transformed to something else
for (auto& remap : g_font_large_char_remap) {
for (auto rc : remap.chars) {
m_valid_source_text_chars[(u8)rc] = true;
}
}
for (auto& remap : g_font_large_string_replace) {
for (auto rc : remap.to) {
m_valid_source_text_chars[(u8)rc] = true;
}
for (auto rc : remap.from) {
m_valid_source_text_chars[(u8)rc] = true;
}
}
m_valid_source_text_chars[0] = false;
}
bool Reader::is_valid_source_char(char c) const {
return m_valid_source_text_chars[(u8)c];
}
/*!
@@ -163,7 +205,7 @@ std::optional<Object> Reader::read_from_stdin(const std::string& prompt, ReplWra
db.insert(textFrag);
// perform read
auto result = internal_read(textFrag);
auto result = internal_read(textFrag, false);
db.link(result, textFrag, 0);
return result;
} else {
@@ -180,7 +222,7 @@ Object Reader::read_from_string(const std::string& str, bool add_top_level) {
db.insert(textFrag);
// perform read
auto result = internal_read(textFrag, add_top_level);
auto result = internal_read(textFrag, false, add_top_level);
db.link(result, textFrag, 0);
return result;
}
@@ -188,7 +230,7 @@ Object Reader::read_from_string(const std::string& str, bool add_top_level) {
/*!
* Read a file
*/
Object Reader::read_from_file(const std::vector<std::string>& file_path) {
Object Reader::read_from_file(const std::vector<std::string>& file_path, bool check_encoding) {
std::string joined_name;
for (const auto& thing : file_path) {
@@ -202,7 +244,7 @@ Object Reader::read_from_file(const std::vector<std::string>& file_path) {
auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_name);
db.insert(textFrag);
auto result = internal_read(textFrag);
auto result = internal_read(textFrag, check_encoding);
db.link(result, textFrag, 0);
return result;
}
@@ -210,21 +252,34 @@ Object Reader::read_from_file(const std::vector<std::string>& file_path) {
/*!
* Common read for a SourceText
*/
Object Reader::internal_read(std::shared_ptr<SourceText> text, bool add_top_level) {
// validate the input;
for (int offset = 0; offset < text->get_size(); offset++) {
if (!m_valid_source_text_chars[(u8)text->get_text()[offset]]) {
Object Reader::internal_read(std::shared_ptr<SourceText> text,
bool check_encoding,
bool add_top_level) {
// verify UTF-8 encoding
if (check_encoding && (text->get_size() < 3 || (u8)text->get_text()[0] != 0xEF ||
(u8)text->get_text()[1] != 0xBB || (u8)text->get_text()[2] != 0xBF)) {
throw std::runtime_error(
fmt::format("Text file {} has invalid encoding", text->get_description()));
}
// validate the input
for (int offset = check_encoding ? 3 : 0; offset < text->get_size(); offset++) {
if (!is_valid_source_char(text->get_text()[offset])) {
// failed.
int line_number = text->get_line_idx(offset) + 1;
throw std::runtime_error(fmt::format("Invalid character found on line {} of {}: 0x{:x}",
line_number, text->get_description(),
(u32)text->get_text()[offset]));
(u8)text->get_text()[offset]));
}
}
// first create stream
TextStream ts(text);
if (check_encoding) {
// discard the UTF-8 encoding bytes
ts.read_utf8_encoding(true);
}
// clean up first whitespace
ts.seek_past_whitespace_and_comments();
@@ -239,7 +294,7 @@ Object Reader::internal_read(std::shared_ptr<SourceText> text, bool add_top_leve
bool Reader::check_string_is_valid(const std::string& str) const {
for (auto c : str) {
if (!m_valid_source_text_chars[(u8)c]) {
if (!is_valid_source_char(c)) {
return false;
}
}
+7 -2
View File
@@ -55,6 +55,7 @@ struct TextStream {
bool text_remains() { return seek < text->get_size(); }
bool text_remains(int i) { return seek + i < text->get_size(); }
void seek_past_whitespace_and_comments();
void read_utf8_encoding(bool throw_on_error);
};
/*!
@@ -72,7 +73,7 @@ class Reader {
Reader();
Object read_from_string(const std::string& str, bool add_top_level = true);
std::optional<Object> read_from_stdin(const std::string& prompt, ReplWrapper& repl);
Object read_from_file(const std::vector<std::string>& file_path);
Object read_from_file(const std::vector<std::string>& file_path, bool check_encoding = false);
bool check_string_is_valid(const std::string& str) const;
std::string get_source_dir();
@@ -80,7 +81,9 @@ class Reader {
TextDb db;
private:
Object internal_read(std::shared_ptr<SourceText> text, bool add_top_level = true);
Object internal_read(std::shared_ptr<SourceText> text,
bool check_encoding,
bool add_top_level = true);
Object read_list(TextStream& stream, bool expect_close_paren = true);
bool read_object(Token& tok, TextStream& ts, Object& obj);
bool read_array(TextStream& stream, Object& o);
@@ -100,6 +103,8 @@ class Reader {
bool m_valid_symbols_chars[256];
bool m_valid_source_text_chars[256];
bool is_valid_source_char(char c) const;
std::unordered_map<std::string, std::string> m_reader_macros;
};
+478
View File
@@ -0,0 +1,478 @@
/*!
* @file FontUtils.cpp
*
* Code for handling text and strings in Jak 1's "large font" format.
*
* MAKE SURE THIS FILE IS ENCODED IN UTF-8!!! The various strings here depend on it.
* Always verify the encoding if string detection suddenly goes awry.
*/
#include "FontUtils.h"
#include "third-party/fmt/core.h"
#include <algorithm>
#include <functional>
#include <map>
#include <unordered_set>
/*!
* Remaps UTF-8 characters to the appropriate character fit for the game's large font.
* It is unfortunately quite large.
*/
std::vector<RemapInfo> g_font_large_char_remap = {
// random
{"ˇ", {0x10}}, // caron
{"`", {0x11}}, // grave accent
{"'", {0x12}}, // apostrophe
{"^", {0x13}}, // circumflex
{"<TIL>", {0x14}}, // tilde
{"¨", {0x15}}, // umlaut
{"º", {0x16}}, // numero/overring
{"¡", {0x17}}, // inverted exclamation mark
{"¿", {0x18}}, // inverted question mark
{"", {0x1a}}, // umi
{"Æ", {0x1b}}, // aesc
{"", {0x1c}}, // kai
{"Ç", {0x1d}}, // c-cedilla
{"", {0x1e}}, // gaku
{"ß", {0x1f}}, // eszett
{"\"", {0x22}}, // double-quotes
{"", {0x24}}, // wa
{"", {0x26}}, // wo
{"", {0x27}}, // -n
{"", {0x5c}}, // iwa
{"", {0x5d}}, // kyuu
{"", {0x5e}}, // sora
//{"掘", {0x5f}}, // horu
{"", {0x60}}, // -wa
{"", {0x61}}, // utsu
{"", {0x62}}, // kashikoi
{"", {0x63}}, // mizuumi
{"", {0x64}}, // kuchi
{"", {0x65}}, // iku
{"", {0x66}}, // ai
{"", {0x67}}, // shi
{"", {0x68}}, // tera
{"", {0x69}}, // yama
{"", {0x6a}}, // mono
{"", {0x6b}}, // tokoro
{"", {0x6c}}, // kaku
{"", {0x6d}}, // shou
{"", {0x6e}}, // numa
{"", {0x6f}}, // ue
{"", {0x70}}, // shiro
{"", {0x71}}, // ba
{"", {0x72}}, // shutsu
{"", {0x73}}, // yami
{"", {0x74}}, // nokosu
{"", {0x75}}, // ki
{"", {0x76}}, // ya
{"", {0x77}}, // shita
{"", {0x78}}, // ie
{"", {0x79}}, // hi
{"", {0x7a}}, // hana
{"", {0x7b}}, // re
{"Œ", {0x7c}}, // oe
{"", {0x7d}}, // ro
{"", {0x7f}}, // ao
{"", {0x90}}, // nakaguro
{"", {0x91}}, // dakuten
{"", {0x92}}, // handakuten
{"", {0x93}}, // chouompu
{"", {0x94}}, // nijuukagikakko left
{"", {0x95}}, // nijuukagikakko right
// hiragana
{"", {0x96}}, // -a
{"", {0x97}}, // a
{"", {0x98}}, // -i
{"", {0x99}}, // i
{"", {0x9a}}, // -u
{"", {0x9b}}, // u
{"", {0x9c}}, // -e
{"", {0x9d}}, // e
{"", {0x9e}}, // -o
{"", {0x9f}}, // o
{"", {0xa0}}, // ka
{"", {0xa1}}, // ki
{"", {0xa2}}, // ku
{"", {0xa3}}, // ke
{"", {0xa4}}, // ko
{"", {0xa5}}, // sa
{"", {0xa6}}, // shi
{"", {0xa7}}, // su
{"", {0xa8}}, // se
{"", {0xa9}}, // so
{"", {0xaa}}, // ta
{"", {0xab}}, // chi
{"", {0xac}}, // sokuon
{"", {0xad}}, // tsu
{"", {0xae}}, // te
{"", {0xaf}}, // to
{"", {0xb0}}, // na
{"", {0xb1}}, // ni
{"", {0xb2}}, // nu
{"", {0xb3}}, // ne
{"", {0xb4}}, // no
{"", {0xb5}}, // ha
{"", {0xb6}}, // hi
{"", {0xb7}}, // hu
{"", {0xb8}}, // he
{"", {0xb9}}, // ho
{"", {0xba}}, // ma
{"", {0xbb}}, // mi
{"", {0xbc}}, // mu
{"", {0xbd}}, // me
{"", {0xbe}}, // mo
{"", {0xbf}}, // youon ya
{"", {0xc0}}, // ya
{"", {0xc1}}, // youon yu
{"", {0xc2}}, // yu
{"", {0xc3}}, // youon yo
{"", {0xc4}}, // yo
{"", {0xc5}}, // ra
{"", {0xc6}}, // ri
{"", {0xc7}}, // ru
{"", {0xc8}}, // re
{"", {0xc9}}, // ro
{"", {0xca}}, // -wa
{"", {0xcb}}, // wa
{"", {0xcc}}, // wo
{"", {0xcd}}, // -n
// katakana
{"", {0xce}}, // -a
{"", {0xcf}}, // a
{"", {0xd0}}, // -i
{"", {0xd1}}, // i
{"", {0xd2}}, // -u
{"", {0xd3}}, // u
{"", {0xd4}}, // -e
{"", {0xd5}}, // e
{"", {0xd6}}, // -o
{"", {0xd7}}, // o
{"", {0xd8}}, // ka
{"", {0xd9}}, // ki
{"", {0xda}}, // ku
{"", {0xdb}}, // ke
{"", {0xdc}}, // ko
{"", {0xdd}}, // sa
{"", {0xde}}, // shi
{"", {0xdf}}, // su
{"", {0xe0}}, // se
{"", {0xe1}}, // so
{"", {0xe2}}, // ta
{"", {0xe3}}, // chi
{"", {0xe4}}, // sokuon
{"", {0xe5}}, // tsu
{"", {0xe6}}, // te
{"", {0xe7}}, // to
{"", {0xe8}}, // na
{"", {0xe9}}, // ni
{"", {0xea}}, // nu
{"", {0xeb}}, // ne
{"", {0xec}}, // no
{"", {0xed}}, // ha
{"", {0xee}}, // hi
{"", {0xef}}, // hu
{"", {0xf0}}, // he
{"", {0xf1}}, // ho
{"", {0xf2}}, // ma
{"", {0xf3}}, // mi
{"", {0xf4}}, // mu
{"", {0xf5}}, // me
{"", {0xf6}}, // mo
{"", {0xf7}}, // youon ya
{"", {0xf8}}, // ya
{"", {0xf9}}, // youon yu
{"", {0xfa}}, // yu
{"", {0xfb}}, // youon yo
{"", {0xfc}}, // yo
{"", {0xfd}}, // ra
{"", {0xfe}}, // ri
{"", {0xff}}, // ru
// kanji 2
{"", {1, 0x01}}, // takara
{"", {1, 0x10}}, // ishi
{"", {1, 0x11}}, // aka
{"", {1, 0x12}}, // ato
{"", {1, 0x13}}, // kawa
{"", {1, 0x14}}, // ikusa
{"", {1, 0x15}}, // mura
{"", {1, 0x16}}, // tai
{"", {1, 0x17}}, // utena
{"", {1, 0x18}}, // osa
{"", {1, 0x19}}, // tori
{"", {1, 0x1a}}, // tei
{"", {1, 0x1b}}, // hora
{"", {1, 0x1c}}, // michi
{"", {1, 0x1d}}, // hatsu
{"", {1, 0x1e}}, // tobu
{"", {1, 0x1f}}, // fuku
{"", {1, 0xa0}}, // ike
{"", {1, 0xa1}}, // naka
{"", {1, 0xa2}}, // tou
{"", {1, 0xa3}}, // shima
{"", {1, 0xa4}}, // bu
{"", {1, 0xa5}}, // hou
{"", {1, 0xa6}}, // san
{"", {1, 0xa7}}, // kaerimiru
{"", {1, 0xa8}}, // chikara
{"", {1, 0xa9}}, // midori
{"", {1, 0xaa}}, // kishi
{"", {1, 0xab}}, // zou
{"", {1, 0xac}}, // tani
{"", {1, 0xad}}, // kokoro
{"", {1, 0xae}}, // mori
{"", {1, 0xaf}}, // mizu
{"", {1, 0xb0}}, // fune
{"", {1, 0xb1}}, // trademark
};
/*!
* Replaces specific UTF-8 strings with more readable variants.
*/
std::vector<ReplaceInfo> g_font_large_string_replace = {
// \" -> " (confusing)
{"\\\"", "\""},
// other
{"A~Y~-21H~-5Vº~Z", "Å"},
{"N~Y~-6Hº~Z~+10H", ""},
// tildes
{"N~Y~-22H~-4V<TIL>~Z", "Ñ"},
{"A~Y~-21H~-5V<TIL>~Z", "Ã"}, // custom
{"O~Y~-22H~-4V<TIL>~Z", "Õ"}, // custom
// acute accents
{"A~Y~-21H~-5V'~Z", "Á"},
{"E~Y~-22H~-5V'~Z", "É"},
{"I~Y~-19H~-5V'~Z", "Í"},
{"O~Y~-22H~-4V'~Z", "Ó"},
{"U~Y~-24H~-3V'~Z", "Ú"},
// circumflex
{"A~Y~-20H~-4V^~Z", "Â"}, // custom
{"E~Y~-20H~-5V^~Z", "Ê"},
{"I~Y~-19H~-5V^~Z", "Î"},
{"O~Y~-20H~-4V^~Z", "Ô"}, // custom
{"U~Y~-24H~-3V^~Z", "Û"},
// grave accents
{"A~Y~-21H~-5V`~Z", "À"},
{"E~Y~-22H~-5V`~Z", "È"},
{"I~Y~-19H~-5V`~Z", "Ì"},
{"O~Y~-22H~-4V`~Z", "Ò"}, // custom
{"U~Y~-24H~-3V`~Z", "Ù"},
// umlaut
{"A~Y~-21H~-5V¨~Z", "Ä"},
{"E~Y~-20H~-5V¨~Z", "Ë"},
{"I~Y~-19H~-5V¨~Z", "Ï"}, // custom
{"O~Y~-22H~-4V¨~Z", "Ö"},
{"O~Y~-22H~-3V¨~Z", "ö"}, // dumb
{"U~Y~-22H~-3V¨~Z", "Ü"},
// dakuten katakana
{"~Yウ~Z゛", ""},
{"~Yカ~Z゛", ""},
{"~Yキ~Z゛", ""},
{"~Yク~Z゛", ""},
{"~Yケ~Z゛", ""},
{"~Yコ~Z゛", ""},
{"~Yサ~Z゛", ""},
{"~Yシ~Z゛", ""},
{"~Yス~Z゛", ""},
{"~Yセ~Z゛", ""},
{"~Yソ~Z゛", ""},
{"~Yタ~Z゛", ""},
{"~Yチ~Z゛", ""},
{"~Yツ~Z゛", ""},
{"~Yテ~Z゛", ""},
{"~Yト~Z゛", ""},
{"~Yハ~Z゛", ""},
{"~Yヒ~Z゛", ""},
{"~Yフ~Z゛", ""},
{"~Yヘ~Z゛", ""},
{"~Yホ~Z゛", ""},
// handakuten katakana
{"~Yハ~Z゜", ""},
{"~Yヒ~Z゜", ""},
{"~Yフ~Z゜", ""},
{"~Yヘ~Z゜", ""},
{"~Yホ~Z゜", ""},
// dakuten hiragana
{"~Yか~Z゛", ""},
{"~Yき~Z゛", ""},
{"~Yく~Z゛", ""},
{"~Yけ~Z゛", ""},
{"~Yこ~Z゛", ""},
{"~Yさ~Z゛", ""},
{"~Yし~Z゛", ""},
{"~Yす~Z゛", ""},
{"~Yせ~Z゛", ""},
{"~Yそ~Z゛", ""},
{"~Yた~Z゛", ""},
{"~Yち~Z゛", ""},
{"~Yつ~Z゛", ""},
{"~Yて~Z゛", ""},
{"~Yと~Z゛", ""},
{"~Yは~Z゛", ""},
{"~Yひ~Z゛", ""},
{"~Yふ~Z゛", ""},
{"~Yへ~Z゛", ""},
{"~Yほ~Z゛", ""},
// handakuten hiragana
{"~Yは~Z゜", ""},
{"~Yひ~Z゜", ""},
{"~Yふ~Z゜", ""},
{"~Yへ~Z゜", ""},
{"~Yほ~Z゜", ""},
// japanese punctuation
{",~+8H", ""},
{"~+8H ", " "},
// (hack) special case kanji
{"~~", ""},
// playstation buttons
{"~Y~22L<~Z~Y~27L*~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_X>"},
{"~Y~22L<~Z~Y~26L;~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_TRIANGLE>"},
{"~Y~22L<~Z~Y~25L@~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_CIRCLE>"},
{"~Y~22L<~Z~Y~24L#~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_SQUARE>"}, // custom
};
static bool remaps_inited = false;
static void init_remaps() {
if (!remaps_inited) {
std::sort(
g_font_large_char_remap.begin(), g_font_large_char_remap.end(),
[](const RemapInfo& a, const RemapInfo& b) { return a.bytes.size() > b.bytes.size(); });
std::sort(
g_font_large_string_replace.begin(), g_font_large_string_replace.end(),
[](const ReplaceInfo& a, const ReplaceInfo& b) { return a.from.size() > b.from.size(); });
remaps_inited = true;
}
}
/*!
* Convert Jak 1 character encoding to something readable.
*/
RemapInfo* jak1_bytes_to_utf8(const char* in) {
init_remaps();
for (auto& info : g_font_large_char_remap) {
if (info.bytes.size() == 0)
continue;
bool found = true;
for (int i = 0; found && i < info.bytes.size(); ++i) {
if (uint8_t(in[i]) != info.bytes.at(i)) {
found = false;
}
}
if (found) {
return &info;
}
}
return nullptr;
}
/*!
* Try to replace specific substrings with better variants.
* These are for hiding confusing text transforms.
*/
std::string& jak1_trans_to_utf8(std::string& str) {
init_remaps();
for (auto& info : g_font_large_string_replace) {
auto pos = str.find(info.from);
while (pos != std::string::npos) {
str.replace(pos, info.from.size(), info.to);
pos = str.find(info.from, pos + info.to.size());
}
}
return str;
}
std::string& utf8_trans_to_jak1(std::string& str) {
init_remaps();
for (auto& info : g_font_large_string_replace) {
auto pos = str.find(info.to);
while (pos != std::string::npos) {
str.replace(pos, info.to.size(), info.from);
pos = str.find(info.to, pos + info.from.size());
}
}
return str;
}
std::string& utf8_bytes_to_jak1(std::string& str) {
// find all instances of characters and save them
std::map<size_t, const RemapInfo*, std::greater<size_t>> remap_cache;
for (auto& info : g_font_large_char_remap) {
auto pos = str.find(info.chars);
while (pos != std::string::npos) {
remap_cache[pos] = &info;
pos = str.find(info.chars, pos + info.chars.size());
}
}
// go through the string backwards and replace saved chars
for (auto& remap : remap_cache) {
std::string temp;
for (auto b : remap.second->bytes) {
temp.push_back(b);
}
str.replace(remap.first, remap.second->chars.size(), temp);
}
return str;
}
/*!
* Turn a normal readable string into a string readable in the Jak 1 font encoding.
*/
std::string convert_to_jak1_encoding(std::string str) {
utf8_trans_to_jak1(str);
utf8_bytes_to_jak1(str);
return str;
}
static const std::unordered_set<char> passthrus = {'~', ' ', ',', '.', '-', '+', '(', ')',
'!', ':', '?', '=', '%', '*', '/', '#',
';', '<', '>', '@', '[', '_'};
/*!
* Convert a string from the Jak 1 large font encoding to something normal.
* Unprintable characters become escape sequences, including tab and newline.
*/
std::string convert_from_jak1_encoding(const char* in) {
std::string result;
while (*in) {
auto remap = jak1_bytes_to_utf8(in);
if (remap != nullptr) {
result.append(remap->chars);
in += remap->bytes.size() - 1;
} else if (((*in >= '0' && *in <= '9') || (*in >= 'A' && *in <= 'Z') ||
passthrus.find(*in) != passthrus.end()) &&
*in != '\\') {
result.push_back(*in);
} else if (*in == '\n') {
result += "\\n";
} else if (*in == '\t') {
result += "\\t";
} else if (*in == '\\') {
result += "\\\\";
} else {
result += fmt::format("\\c{:02x}", uint8_t(*in));
}
in++;
}
return jak1_trans_to_utf8(result);
}
+40
View File
@@ -0,0 +1,40 @@
#pragma once
/*!
* @file FontUtils.h
*
* Code for handling text and strings in Jak 1's "large font" format.
*
* MAKE SURE THIS FILE IS ENCODED IN UTF-8!!! The various strings here depend on it.
* Always verify the encoding if string detection suddenly goes awry.
*/
#include "common/common_types.h"
#include <string>
#include <vector>
struct RemapInfo {
std::string chars;
std::vector<u8> bytes;
};
struct ReplaceInfo {
std::string from;
std::string to;
};
/*!
* Remaps UTF-8 characters to the appropriate character fit for the game's large font.
* It is unfortunately quite large.
*/
extern std::vector<RemapInfo> g_font_large_char_remap;
/*!
* Replaces specific UTF-8 strings with more readable variants.
*/
extern std::vector<ReplaceInfo> g_font_large_string_replace;
RemapInfo* jak1_bytes_to_utf8(const char* in);
std::string& jak1_trans_to_utf8(std::string& str);
std::string convert_to_jak1_encoding(std::string str);
std::string convert_from_jak1_encoding(const char* in);
+12 -13
View File
@@ -5358,12 +5358,12 @@
(flat-dark-purple 19)
(flat-yellow 20)
(blue-white 21)
(flat-dark-gray 22)
(flat-gray 23)
(flat-pink 24)
(flat-red 25)
(flat-green 26)
(flat-purple 27)
(pad-back 22)
(pad-shine 23)
(pad-square 24)
(pad-circle 25)
(pad-triangle 26)
(pad-x 27)
(lighter-lighter-blue 28)
(yellow-orange 29)
(yellow-green-2 30)
@@ -5666,7 +5666,7 @@
(define-extern *text-group-names* (array string))
(define-extern kheap type)
(define-extern *common-text-heap* kheap)
(define-extern *common-text* game-text-info) ; TODO guess, but its definitely not just a symbol!
(define-extern *common-text* game-text-info)
;; ----------------------
@@ -5897,7 +5897,6 @@
(define-extern adgif-shader-login-no-remap (function adgif-shader texture))
(define-extern adgif-shader-login-fast (function adgif-shader texture))
(define-extern adgif-shader-login-no-remap-fast (function adgif-shader texture))
;;;; unknown type
(define-extern adgif-shader<-texture-simple! (function adgif-shader texture adgif-shader))
;; - Symbols
@@ -25992,8 +25991,8 @@
(spin-angle float :offset-assert 192)
(spin-speed float :offset-assert 196)
(interp float :offset-assert 200)
(duration uint64 :offset-assert 208)
(delay uint64 :offset-assert 216)
(duration int64 :offset-assert 208)
(delay int64 :offset-assert 216)
(color int8 :offset-assert 224)
)
:method-count-assert 22
@@ -26026,8 +26025,8 @@
(x-spacing float :offset-assert 256)
(z-spacing float :offset-assert 260)
(idle-distance float :offset-assert 264)
(duration uint64 :offset-assert 272)
(drop-time uint64 :offset-assert 280)
(duration int64 :offset-assert 272)
(drop-time int64 :offset-assert 280)
)
:method-count-assert 20
:size-assert #x120
@@ -26040,7 +26039,7 @@
(define-extern citb-drop-plat-spawn-children (function none :behavior citb-drop-plat))
(define-extern citb-drop-plat-drop-children (function int none :behavior citb-drop-plat))
(define-extern citb-drop-plat-drop-all-children (function symbol :behavior citb-drop-plat))
(define-extern drop-plat-init-by-other (function vector uint uint int none :behavior drop-plat))
(define-extern drop-plat-init-by-other (function vector int int int none :behavior drop-plat))
(define-extern drop-plat-set-fade (function none :behavior drop-plat))
;; - Unknowns
@@ -761,13 +761,7 @@
[29, "(function none)"]
],
"fisher-JUN": [
[27, "(function none :behavior process)"],
[28, "(function none :behavior fisher)"],
[20, "(function none :behavior fisher-fish)"],
[1, "(function none :behavior target)"]
],
"fisher-JUNGLE-L1": [
"fisher": [
[27, "(function none :behavior process)"],
[28, "(function none :behavior fisher)"],
[20, "(function none :behavior fisher-fish)"],
+1 -5
View File
@@ -872,11 +872,7 @@
["L670", "uint64", true]
],
"fisher-JUN": [
["L262", "vector"],
["L463", "(array (inline-array fisher-params))"]
],
"fisher-JUNGLE-L1": [
"fisher": [
["L262", "vector"],
["L463", "(array (inline-array fisher-params))"]
],
@@ -2820,6 +2820,7 @@
[32, "vector"]
],
"fisher-draw-display": [[16, "font-context"]],
"(trans fisher-done)": [[16, "font-context"]],
"(method 10 torus)": [
[16, "vector"],
+4 -2
View File
@@ -7,6 +7,7 @@
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "common/goos/Reader.h"
#include "common/util/BitUtils.h"
#include "common/util/FontUtils.h"
namespace decompiler {
namespace {
@@ -98,7 +99,7 @@ GameTextResult process_game_text(ObjectFileData& data) {
}
// escape characters
result.text[text_id] = goos::get_readable_string(text.c_str());
result.text[text_id] = convert_from_jak1_encoding(text.c_str());
// remember what we read (-1 for the type tag)
auto string_start = (text_label.offset / 4) - 1;
@@ -146,7 +147,8 @@ std::string write_game_text(
}
// write!
std::string result = fmt::format("(language-count {})\n", langauges.size());
std::string result; // = "\xEF\xBB\xBF"; // UTF-8 encode (don't need this anymore)
result += fmt::format("(language-count {})\n", langauges.size());
result += "(group-name \"common\")\n";
for (auto& x : text_by_id) {
result += fmt::format("(#x{:04x}\n ", x.first);
+6 -6
View File
@@ -32,12 +32,12 @@
(flat-dark-purple 19)
(flat-yellow 20)
(blue-white 21)
(flat-dark-gray 22)
(flat-gray 23)
(flat-pink 24)
(flat-red 25)
(flat-green 26)
(flat-purple 27)
(pad-back 22)
(pad-shine 23)
(pad-square 24)
(pad-circle 25)
(pad-triangle 26)
(pad-x 27)
(lighter-lighter-blue 28)
(yellow-orange 29)
(yellow-green-2 30)
+4 -2
View File
@@ -33,10 +33,12 @@
;; -memory-
;; the size of the execution stack (~14 kB) shared by all threads
(defconstant DPROCESS_STACK_SIZE #x3800)
;; OpenGOAL NOTE: increased to 32kB
(defconstant DPROCESS_STACK_SIZE #x8000)
;; another stack size used as a maximum for temporary threads
(defconstant PROCESS_STACK_SIZE #x1c00)
;; OpenGOAL NOTE: increased to 16kB
(defconstant PROCESS_STACK_SIZE #x4000)
;; default size of stack to backup for a process
(defconstant PROCESS_STACK_SAVE_SIZE 256)
+16 -16
View File
@@ -124,8 +124,8 @@
(spin-angle float :offset-assert 192)
(spin-speed float :offset-assert 196)
(interp float :offset-assert 200)
(duration uint64 :offset-assert 208)
(delay uint64 :offset-assert 216)
(duration int64 :offset-assert 208)
(delay int64 :offset-assert 216)
(color int8 :offset-assert 224)
)
:heap-base #x80
@@ -171,7 +171,7 @@
(if
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (-> self duration))
(-> self duration)
)
(go drop-plat-drop)
)
@@ -230,7 +230,7 @@
(when
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (-> self delay))
(-> self delay)
)
(let ((v1-14 (logand -3 (-> self draw status)))
(a0-5 (-> self draw))
@@ -503,7 +503,7 @@
(defbehavior
drop-plat-init-by-other drop-plat
((arg0 vector) (arg1 uint) (arg2 uint) (arg3 int))
((arg0 vector) (arg1 int) (arg2 int) (arg3 int))
(set! (-> self color) arg3)
(set! (-> self delay) arg1)
(set! (-> self duration) arg2)
@@ -542,8 +542,8 @@
(x-spacing float :offset-assert 256)
(z-spacing float :offset-assert 260)
(idle-distance float :offset-assert 264)
(duration uint64 :offset-assert 272)
(drop-time uint64 :offset-assert 280)
(duration int64 :offset-assert 272)
(drop-time int64 :offset-assert 280)
)
:heap-base #xb0
:method-count-assert 20
@@ -654,7 +654,10 @@
uint
sv-64
)
t0-0
(the-as
uint
t0-0
)
)
)
(->
@@ -676,7 +679,7 @@
)
)
)
(set! (-> self drop-time) (the-as uint (-> *display* base-frame-counter)))
(set! (-> self drop-time) (-> *display* base-frame-counter))
0
(none)
)
@@ -741,15 +744,12 @@
((= v1-0 'player-stepped)
(when
(>=
(-
(-> *display* base-frame-counter)
(the-as int (-> self drop-time))
)
(- (-> *display* base-frame-counter) (-> self drop-time))
60
)
(set!
(-> self drop-time)
(the-as uint (-> *display* base-frame-counter))
(-> *display* base-frame-counter)
)
(citb-drop-plat-drop-children
(the-as int (-> arg3 param 0))
@@ -772,7 +772,7 @@
(or
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (+ (-> self duration) 600))
(+ (-> self duration) 600)
)
(or
(not *target*)
@@ -825,7 +825,7 @@
)
(set!
(-> obj duration)
(the-as uint (the int (* 300.0 (+ 2.0 (the float (-> obj z-count))))))
(the int (* 300.0 (+ 2.0 (the float (-> obj z-count)))))
)
(let ((f0-7 (res-lump-float arg0 'rotoffset)))
(quaternion-rotate-y! (-> obj root quat) (-> obj root quat) f0-7)
+104
View File
@@ -0,0 +1,104 @@
;;-*-Lisp-*-
(in-package goal)
;; This file is used for debugging and testing the large font encoding.
;; This file should *not* be included as part of any packages, it should be manually loaded by the user.
;; To run this:
#|
(make-group "iso") ;; build the game
(lt) ;; connect to the runtime
(lg) ;; have the runtime load the game engine
(test-play) ;; start the game loop
(ml "goal_src/pc_debug/font-encode-test.gc") ;; build and load this file.
|#
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; constants
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defconstant FONT_ENCODE_TEXT_LEFT 56)
(defconstant FONT_ENCODE_TEXT_Y 80)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; functions
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define *font-string* (new 'global 'string 64 (the-as string #f)))
(define *font-string-ex* "")
(define *font-string-val* #x96)
(defun-debug font-encode-test-start ()
"start the encode test proc"
(unless (process-by-name 'font-encode *active-pool*)
(make-function-process process :name 'font-encode
(lambda :behavior process ()
(stack-size-set! (-> self main-thread) 768)
(let ((fnt (new 'stack 'font-context *font-default-matrix* FONT_ENCODE_TEXT_LEFT FONT_ENCODE_TEXT_Y 0.0
(font-color orange-red) (font-flags shadow kerning large middle)))
)
(set-width! fnt 400)
(set-height! fnt 100)
(loop
(suspend)
(if (or (cpad-pressed? 0 left) (cpad-hold? 0 l1))
(-! *font-string-val* 1)
)
(if (or (cpad-pressed? 0 right) (cpad-hold? 0 r1))
(+! *font-string-val* 1)
)
(if (< *font-string-val* 1)
(set! *font-string-val* 1)
)
(if (> *font-string-val* #x1ff)
(set! *font-string-val* #x1ff)
)
(clear *font-string*)
(cond
((>= *font-string-val* #x100)
(set! (-> *font-string* data 0) (/ *font-string-val* 256))
(set! (-> *font-string* data 1) (mod *font-string-val* 256))
(set! (-> *font-string* data 2) 0)
)
(else
(set! (-> *font-string* data 0) (mod *font-string-val* 256))
(set! (-> *font-string* data 1) 0)
)
)
(set-origin! fnt FONT_ENCODE_TEXT_LEFT FONT_ENCODE_TEXT_Y)
(set-flags! fnt (font-flags shadow kerning large middle))
(print-game-text *font-string* fnt #f 128 24)
(set-origin! fnt FONT_ENCODE_TEXT_LEFT (+ FONT_ENCODE_TEXT_Y 32))
(print-game-text *font-string-ex* fnt #f 128 24)
(set-origin! fnt FONT_ENCODE_TEXT_LEFT (- FONT_ENCODE_TEXT_Y 16))
(set-flags! fnt (font-flags shadow kerning middle))
(print-game-text (string-format "#x~X" *font-string-val*) fnt #f 128 12)
)
)
)
)
)
)
(defun-debug font-encode-test-stop ()
"stop the encode test proc"
(kill-by-name 'font-encode *active-pool*)
)
+5 -8
View File
@@ -28,6 +28,7 @@
(input handle)
)
)
(define *pc-pad-proc-list* (new 'static 'pc-pad-proc-list))
(set! (-> *pc-pad-proc-list* show) (the handle #f))
(set! (-> *pc-pad-proc-list* input) (the handle #f))
@@ -39,6 +40,9 @@
(pad-idx uint64)
)
:heap-base #x20
(:states
pc-pi-mapping-button
)
)
(define *pc-pad-button-names*
@@ -64,7 +68,7 @@
"SQUARE"
))
;; there is a matching enum in newpad
;; there is a matching enum in newpad.h
(defenum pc-pad-input-status
(disabled)
(enabled)
@@ -72,13 +76,6 @@
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; forward declarations
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define-extern pc-pi-mapping-button (state pc-pad-proc))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; constants
+4 -3
View File
@@ -12,6 +12,7 @@
#include "common/goos/Reader.h"
#include "DataObjectGenerator.h"
#include "common/util/FileUtil.h"
#include "common/util/FontUtils.h"
#include "third-party/fmt/core.h"
namespace {
@@ -56,7 +57,7 @@ std::string get_string(const goos::Object& x) {
if (x.is_string()) {
return x.as_string()->data;
}
throw std::runtime_error(x.print() + " was supposed to be an string, but isn't");
throw std::runtime_error(x.print() + " was supposed to be a string, but isn't");
}
std::string uppercase(const std::string& in) {
@@ -89,7 +90,7 @@ std::vector<std::unordered_map<int, std::string>> parse(const goos::Object& data
for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
if (obj.is_pair()) {
auto head = obj.as_pair()->car;
auto& head = obj.as_pair()->car;
if (head.is_symbol() && head.as_symbol()->name == "language-count") {
if (languages_set) {
throw std::runtime_error("Languages has been set multiple times.");
@@ -126,7 +127,7 @@ std::vector<std::unordered_map<int, std::string>> parse(const goos::Object& data
throw std::runtime_error("Entry appears more than once");
}
map[id] = entry.as_string()->data;
map[id] = convert_to_jak1_encoding(entry.as_string()->data);
} else {
throw std::runtime_error("Each entry must be a string");
}
@@ -108,8 +108,8 @@
(spin-angle float :offset-assert 192)
(spin-speed float :offset-assert 196)
(interp float :offset-assert 200)
(duration uint64 :offset-assert 208)
(delay uint64 :offset-assert 216)
(duration int64 :offset-assert 208)
(delay int64 :offset-assert 216)
(color int8 :offset-assert 224)
)
:heap-base #x80
@@ -170,7 +170,7 @@
(if
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (-> self duration))
(-> self duration)
)
(go drop-plat-drop)
)
@@ -232,7 +232,7 @@
(when
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (-> self delay))
(-> self delay)
)
(let ((v1-14 (logand -3 (-> self draw status)))
(a0-5 (-> self draw))
@@ -514,7 +514,7 @@
;; Used lq/sq
(defbehavior
drop-plat-init-by-other drop-plat
((arg0 vector) (arg1 uint) (arg2 uint) (arg3 int))
((arg0 vector) (arg1 int) (arg2 int) (arg3 int))
(set! (-> self color) arg3)
(set! (-> self delay) arg1)
(set! (-> self duration) arg2)
@@ -564,8 +564,8 @@
(x-spacing float :offset-assert 256)
(z-spacing float :offset-assert 260)
(idle-distance float :offset-assert 264)
(duration uint64 :offset-assert 272)
(drop-time uint64 :offset-assert 280)
(duration int64 :offset-assert 272)
(drop-time int64 :offset-assert 280)
)
:heap-base #xb0
:method-count-assert 20
@@ -701,7 +701,10 @@
uint
sv-64
)
t0-0
(the-as
uint
t0-0
)
)
)
(->
@@ -723,7 +726,7 @@
)
)
)
(set! (-> self drop-time) (the-as uint (-> *display* base-frame-counter)))
(set! (-> self drop-time) (-> *display* base-frame-counter))
0
(none)
)
@@ -793,15 +796,12 @@
((= v1-0 'player-stepped)
(when
(>=
(-
(-> *display* base-frame-counter)
(the-as int (-> self drop-time))
)
(- (-> *display* base-frame-counter) (-> self drop-time))
60
)
(set!
(-> self drop-time)
(the-as uint (-> *display* base-frame-counter))
(-> *display* base-frame-counter)
)
(citb-drop-plat-drop-children
(the-as int (-> arg3 param 0))
@@ -824,7 +824,7 @@
(or
(>=
(- (-> *display* base-frame-counter) (-> self state-time))
(the-as int (+ (-> self duration) 600))
(+ (-> self duration) 600)
)
(or
(not *target*)
@@ -879,7 +879,7 @@
)
(set!
(-> obj duration)
(the-as uint (the int (* 300.0 (+ 2.0 (the float (-> obj z-count))))))
(the int (* 300.0 (+ 2.0 (the float (-> obj z-count)))))
)
(let ((f0-7 (res-lump-float arg0 'rotoffset)))
(quaternion-rotate-y! (-> obj root quat) (-> obj root quat) f0-7)