New Crowdin updates (#2775)

scripts/ci/lint-characters.py

@@ -2,11 +2,17 @@ import glob
 import json
 import re
 
-# TODO - add a way to make this auto replace bad characters with `?`
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--fix", action="store_true")
+parser.set_defaults(fix=False)
+args = parser.parse_args()
+
+# TODO - trim strings
 
 # fmt: off
 JAK1_ALLOWED_CHARACTERS = [
-    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
     "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
     "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
     "'", "!", "(", ")", "+", "-", ",", ".", "/", ":", "=", "<", ">", "*", "%", "?", "\"",
@@ -26,6 +32,12 @@ JAK1_ALLOWED_CODES = [
     "<TIL>",
     "<PAD_X>", "<PAD_TRIANGLE>", "<PAD_CIRCLE>", "<PAD_SQUARE>"
 ]
+
+JAK1_AUTO_REPLACEMENTS = {
+    "ª": "º",
+    "\n": "",
+    "’": "'",
+}
 # fmt: on
 
 invalid_characters_found = False
@@ -45,6 +57,23 @@ def char_allowed(char):
     return char in JAK1_ALLOWED_CHARACTERS
 
 
+def fix_character(char):
+    # First let's try upper-casing it, if that's allowed, let's use that instead
+    upper_case = char.upper()
+    if char_allowed(upper_case):
+        return upper_case
+    if char in JAK1_AUTO_REPLACEMENTS:
+        return JAK1_AUTO_REPLACEMENTS[char]
+    return char
+
+
+def replace_character(string, position, new_character):
+    string_list = list(string)
+    string_list[position] = new_character
+    new_string = "".join(string_list)
+    return new_string
+
+
 def lint_jak1_characters(text):
     invalid_characters_found = False
     pos = 0
@@ -54,30 +83,47 @@ def lint_jak1_characters(text):
             # Check to see if it's an allowed code
             code_end_pos = is_allowed_code(pos, text)
             if code_end_pos == -1:
-                print(
-                    "Character '{}' not allowed - Found in {}".format(character, text)
-                )
-                invalid_characters_found = True
+                # If we are fixing instances, attempt to do so
+                char_fixed = False
+                if args.fix:
+                    new_char = fix_character(character)
+                    if new_char != character:
+                        text = replace_character(text, pos, new_char)
+                        char_fixed = True
+                if not char_fixed:
+                    print(
+                        "Character '{}' not allowed - Found in {}".format(
+                            character, text
+                        )
+                    )
+                    invalid_characters_found = True
                 pos = pos + 1
             else:
                 # advance to the end of the code and continue checking
                 pos = code_end_pos
         else:
             pos = pos + 1
-    return invalid_characters_found
+    return invalid_characters_found, text
 
 
 # Iterate through the translations making sure there are no characters that are not allowed
 text_files = glob.glob("./game/assets/jak1/text/*.json")
 
 for text_file in text_files:
-    print("Checking {}...".format(text_file))
+    print("Checking {}".format(text_file))
     with open(text_file, encoding="utf-8") as f:
         file_data = json.load(f)
     for id, text in file_data.items():
-        invalid_chars_exist = lint_jak1_characters(text)
+        invalid_chars_exist, new_text = lint_jak1_characters(text)
+        if args.fix:
+            file_data[id] = new_text
         if invalid_chars_exist:
             invalid_characters_found = True
+    if args.fix:
+        # save the modified file back out
+        with open(text_file, "w", encoding="utf-8") as f:
+            json.dump(file_data, f, indent=2, ensure_ascii=False)
+            f.write("\n")
 
 subtitle_files = glob.glob("./game/assets/jak1/subtitle/*lines*.json")
 
@@ -87,20 +133,31 @@ for subtitle_file in subtitle_files:
         file_data = json.load(f)
     # Check Speakers
     for id, text in file_data["speakers"].items():
-        invalid_chars_exist = lint_jak1_characters(text)
+        invalid_chars_exist, new_text = lint_jak1_characters(text)
+        if args.fix and new_text != text:
+            file_data["speakers"][id] = new_text
         if invalid_chars_exist:
             invalid_characters_found = True
     # Check Lines
     for id, lines in file_data["cutscenes"].items():
-        for line in lines:
-            invalid_chars_exist = lint_jak1_characters(line)
+        for i, line in enumerate(lines):
+            invalid_chars_exist, new_text = lint_jak1_characters(line)
+            if args.fix and new_text != line:
+                lines[i] = new_text
             if invalid_chars_exist:
                 invalid_characters_found = True
     for id, lines in file_data["hints"].items():
-        for line in lines:
-            invalid_chars_exist = lint_jak1_characters(line)
+        for i, line in enumerate(lines):
+            invalid_chars_exist, new_text = lint_jak1_characters(line)
+            if args.fix and new_text != line:
+                lines[i] = new_text
             if invalid_chars_exist:
                 invalid_characters_found = True
+    if args.fix:
+        # save the modified file back out
+        with open(subtitle_file, "w", encoding="utf-8") as f:
+            json.dump(file_data, f, indent=2, ensure_ascii=False)
+            f.write("\n")
 
 if invalid_characters_found:
     print("Invalid characters were found, see above")