From b6ad40416df4f1604b0547c48b29bf50cd1b45c8 Mon Sep 17 00:00:00 2001 From: Cuyler36 Date: Wed, 13 Nov 2024 10:17:48 -0500 Subject: [PATCH] Add tools to interface with .rarc files, texture files, and message files --- tools/arc_tool.py | 90 ++++ tools/msg_tool.py | 766 ++++++++++++++++++++++++++++++++++ tools/pyjkernel/LICENSE | 674 ++++++++++++++++++++++++++++++ tools/pyjkernel/__init__.py | 5 + tools/pyjkernel/__main__.py | 6 + tools/pyjkernel/jkrarchive.py | 718 +++++++++++++++++++++++++++++++ tools/pyjkernel/jkrcomp.py | 187 +++++++++ tools/texture_tool.py | 562 +++++++++++++++++++++++++ 8 files changed, 3008 insertions(+) create mode 100644 tools/arc_tool.py create mode 100644 tools/msg_tool.py create mode 100644 tools/pyjkernel/LICENSE create mode 100644 tools/pyjkernel/__init__.py create mode 100644 tools/pyjkernel/__main__.py create mode 100644 tools/pyjkernel/jkrarchive.py create mode 100644 tools/pyjkernel/jkrcomp.py create mode 100644 tools/texture_tool.py diff --git a/tools/arc_tool.py b/tools/arc_tool.py new file mode 100644 index 00000000..50e36a26 --- /dev/null +++ b/tools/arc_tool.py @@ -0,0 +1,90 @@ +import pyjkernel +import os +import argparse + + +def unpack_dir(archive: pyjkernel.JKRArchive, dir: str, verbose=False): + if verbose: + print("Dumping dir: " + dir) + # create all files + for file in archive.list_files(dir): + if verbose: + print("Dumping file: " + file.name) + with open(os.path.join(dir, file.name), "wb") as f: + f.write(archive.get_file(dir + '/' + file.name).data) + + # create all subdirectories and recurse through them + for subdir in archive.list_folders(dir): + if not os.path.exists(dir + '/' + subdir): + os.mkdir(dir + '/' + subdir) + unpack_dir(archive, dir + '/' + subdir, verbose) + + +def unpack_archive(path: str, out_path: str, verbose=False): + archive = pyjkernel.from_archive_file(path, True) + orig_dir = os.path.abspath(os.curdir) + os.chdir(out_path) + if not os.path.exists(archive.root_name): + os.mkdir(archive.root_name) + unpack_dir(archive, archive.root_name, verbose) + os.chdir(orig_dir) + + +def pack_dir(archive: pyjkernel.JKRArchive, path: str, verbose=False): + local_path = os.path.dirname(path) + orig_dir = os.path.abspath(os.curdir) + if local_path != "": + os.chdir(local_path) + local_root = os.path.basename(os.path.normpath(path)) + + for root, dirs, files in os.walk(local_root): + root = root.replace('\\', '/') + files.sort(key=lambda item: (item.lower(), item)) + for dir in dirs: + archive.create_folder(root + '/' + dir) + + for file in files: + file = file.replace('\\', '/') + if verbose: + print("Packing file: " + root + '/' + file) + with open(root + '/' + file, "rb") as f: + archive.create_file( + root + '/' + file, + bytearray(f.read()), + pyjkernel.JKRPreloadType.ARAM, + ) + os.chdir(orig_dir) + + +def pack_archive(root_path: str, out_path: str, verbose=False): + root_name = os.path.basename(os.path.normpath(root_path)) + archive = pyjkernel.create_new_archive(root_name) + pack_dir(archive, root_path, verbose) + pyjkernel.write_archive_file( + archive, out_path, True, pyjkernel.jkrcomp.JKRCompression.NONE, 0 + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Pack or unpack JSystem JKernel archives." + ) + parser.add_argument( + "-v", help="Enable verbose logging.", required=False, action="store_true" + ) + parser.add_argument( + "path", help="The path of the folder to pack or archive file to unpack." + ) + parser.add_argument("out", help="The path of the destination folder or file.") + + args = parser.parse_args() + if os.path.isfile(args.path): + unpack_archive(args.path, args.out, args.v) + elif os.path.isdir(args.path): + pack_archive(args.path, args.out, args.v) + else: + raise Exception("path is not a valid file or directory!") + + +if __name__ == "__main__": + main() diff --git a/tools/msg_tool.py b/tools/msg_tool.py new file mode 100644 index 00000000..6404e9e9 --- /dev/null +++ b/tools/msg_tool.py @@ -0,0 +1,766 @@ +import argparse +import struct +import os +import re + +CHAR_MAP = [ + "¡", + "¿", + "Ä", + "À", + "Á", + "Â", + "Ã", + "Å", + "Ç", + "È", + "É", + "Ê", + "Ë", + "Ì", + "Í", + "Î", + "Ï", + "Ð", + "Ñ", + "Ò", + "Ó", + "Ô", + "Õ", + "Ö", + "Ø", + "Ù", + "Ú", + "Û", + "Ü", + "ß", + "Þ", + "à", + " ", + "!", + '"', + "á", + "â", + "%", + "&", + "'", + "(", + ")", + "~", + "♥", + ",", + "-", + ".", + "♪", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + "🌢", + "<", + "=", + ">", + "?", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "ã", + "💢", + "ä", + "å", + "_", + "ç", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "è", + "é", + "ê", + "ë", + "\u007f", + "�", + "ì", + "í", + "î", + "ï", + "•", + "ð", + "ñ", + "ò", + "ó", + "ô", + "õ", + "ö", + "⁰", + "ù", + "ú", + "ー", + "û", + "ü", + "ý", + "ÿ", + "þ", + "Ý", + "¦", + "§", + "ḏ", + "ṉ", + "‖", + "µ", + "³", + "²", + "¹", # note that a̱ and o̱ had to be changed because they're actually two characters in unicode. a̱ -> ḏ | o̱ -> ṉ + "¯", + "¬", + "Æ", + "æ", + "„", + "»", + "«", + "☀", + "☁", + "☂", + "🌬", + "☃", + "∋", + "∈", + "/", + "∞", + "○", + "🗙", + "□", + "△", + "+", + "⚡", + "♂", + "♀", + "🍀", + "★", + "💀", + "😮", + "😄", + "😣", + "😠", + "😃", + "×", + "➗", + "🔨", + "🎀", + "✉", + "💰", + "🐾", + "🐶", + "🐱", + "🐰", + "🐦", + "🐮", + "🐷", + "\n", + "🐟", + "🐞", + ";", + "#", + "\u00d2", + "\u00d3", + "⚷", + "\u00d5", + "\u00d6", + "\u00d7", + "\u00d8", + "\u00d9", + "\u00da", + "\u00db", + "\u00dc", + "Ỳ", + "ꟓ", + "\u00df", + "\u00e0", + "\u00e1", + "\u00e2", + "\u00e3", + "\u00e4", + "\u00e5", + "\u00e6", + "\u00e7", + "\u00e8", + "\u00e9", + "\u00ea", + "\u00eb", + "\u00ec", + "\u00ed", + "\u00ee", + "\u00ef", + "\u00f0", + "\u00f1", + "\u00f2", + "\u00f3", + "\u00f4", + "\u00f5", + "\u00f6", + "÷", + "\u00f8", + "\u00f9", + "\u00fa", + "\u00fb", + "\u00fc", + "\u00fd", + "\u00fe", + "\u00ff", +] + +CONT_SIZES = [ + 2, + 2, + 2, + 3, + 2, + 5, + 2, + 2, + 5, + 5, + 5, + 5, + 5, + 2, + 4, + 4, + 4, + 4, + 4, + 6, + 8, + 10, + 6, + 8, + 10, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 6, + 3, + 3, + 3, + 3, + 2, + 4, + 4, + 3, + 3, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 6, + 3, + 3, + 4, + 3, + 2, + 2, + 6, + 2, + 2, + 3, + 3, + 3, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 12, + 14, +] + +COMMANDS = [ + "MSGEND", + "MSGCONTINUE", + "MSGCLEAR", + "PAUSE", + "BTN", + "TEXTCOLOR", + "ABLECANCEL", + "UNABLECANCEL", + "DEMOPLR", + "DEMONPC0", + "DEMONPC1", + "DEMONPC2", + "DEMONPCQST", + "OPENCHOICE", + "SETFORCEMSG", + "SETNEXTMSG0", + "SETNEXTMSG1", + "SETNEXTMSG2", + "SETNEXTMSG3", + "SETNEXTMSGRND2", + "SETNEXTMSGRND3", + "SETNEXTMSGRND4", + "SETSELSTR2", + "SETSELSTR3", + "SETSELSTR4", + "FORCENEXT", + "STR_PLAYERNAME", + "STR_TALKNAME", + "STR_TAIL", + "STR_YEAR", + "STR_MONTH", + "STR_WEEK", + "STR_DAY", + "STR_HOUR", + "STR_MIN", + "STR_SEC", + "STR_FREE0", + "STR_FREE1", + "STR_FREE2", + "STR_FREE3", + "STR_FREE4", + "STR_FREE5", + "STR_FREE6", + "STR_FREE7", + "STR_FREE8", + "STR_FREE9", + "STR_DETERMINATION", + "STR_COUNTRYNAME", + "STR_RNDNUM", + "STR_ITEM0", + "STR_ITEM1", + "STR_ITEM2", + "STR_ITEM3", + "STR_ITEM4", + "STR_FREE10", + "STR_FREE11", + "STR_FREE12", + "STR_FREE13", + "STR_FREE14", + "STR_FREE15", + "STR_FREE16", + "STR_FREE17", + "STR_FREE18", + "STR_FREE19", + "STR_MAIL", + "LUCK_NEUTRAL", + "LUCK_RELATIONSHIP", + "LUCK_UNPOPULAR", + "LUCK_BAD", + "LUCK_MONEY", + "LUCK_GOODS", + "LUCK_6", + "LUCK_7", + "LUCK_8", + "LUCK_9", + "MSGCONTENTS_NORMAL", + "MSGCONTENTS_ANGRY", + "MSGCONTENTS_SAD", + "MSGCONTENTS_FUN", + "MSGCONTENTS_SLEEPY", + "COLORCHARS", + "SNDCUT", + "LINEOFS", + "LINETYPE", + "CHARSCALE", + "BTN2", + "BGMMAKE", + "BGMDELETE", + "MSGTIMEEND", + "SNDTRGSYS", + "LINESCALE", + "SNDNOPAGE", + "VOICETRUE", + "VOICEFALSE", + "SELNOB", + "GIVEOPEN", + "GIVECLOSE", + "MSGCONTENTS_GLOOMY", + "SELNOBCLOSE", + "SETNEXTMSGRNDSECTION", + "AGBDUMMY0", + "AGBDUMMY1", + "AGBDUMMY2", + "SPACE", + "AGBDUMMY3", + "AGBDUMMY4", + "MALEFEMALECHK", + "AGBDUMMY5", + "AGBDUMMY6", + "AGBDUMMY7", + "AGBDUMMY8", + "AGBDUMMY9", + "AGBDUMMY10", + "STR_ISLANDNAME", + "SETCURSORJUST", + "CLRCUSRORJUST", + "CUTARTICLE", + "CAPTIALIZE", + "STR_AMPM", + "SETNEXTMSG4", + "SETNEXTMSG5", + "SETSELSTR5", + "SETSELSTR6", +] + + +def decode_control_code(ba: bytearray, idx: int): + if ba[idx] != 0x7F: + raise ValueError("First character must be 0x7F") + cont_type = ba[idx + 1] + if cont_type >= len(CONT_SIZES): + raise ValueError(f"Invalid control code id {cont_type:02X}") + cont_size = CONT_SIZES[cont_type] + if len(ba) < idx + cont_size: + raise ValueError( + f"Bytearray is not large enough for control code {cont_type:02X}" + ) + + cmd = COMMANDS[cont_type] + if cmd is not None: + if cont_size > 2: + hex_values = " ".join( + "{:02X}".format(b) + for b in ba[(idx + 2) : (idx + CONT_SIZES[cont_type])] + ) + return f"<<{cmd} [{hex_values}]>>", cont_size + else: + return f"<<{cmd}>>", cont_size + else: + hex_values = " ".join( + "{:02X}".format(b) for b in ba[(idx + 2) : (idx + CONT_SIZES[cont_type])] + ) + return f"<>", cont_size + + +def decode_entry(ba: bytearray, start: int, end: int, idx: int): + parts = [f"[[ENTRY {idx} START]]\n"] # Use a list to collect string parts + i = start + while i < end: + char = ba[i] + if char == 0x7F: + cont_str, cont_size = decode_control_code(ba, i) + parts.append(cont_str) + i += cont_size + else: + parts.append(CHAR_MAP[ba[i]]) + i += 1 + parts.append("\n\n") + return "".join(parts) # Join the parts into a final string at the end + + +def decode_file(data_path: str, table_path: str, out_path: str): + idx = 0 + last_end = 0 + output_buffer = [] + + with open(data_path, "rb") as df, open(table_path, "rb") as tf, open( + out_path, "w" + ) as of: + while True: + bytes = tf.read(4) + if not bytes: + break + end = struct.unpack(">I", bytes)[0] + if end != 0: + size = end - last_end + last_end = end + data = bytearray(df.read(size)) + decoded_str = decode_entry(data, 0, size, idx) + output_buffer.append(decoded_str) + + idx += 1 + # Write buffer content to file to reduce write calls + if len(output_buffer) >= 8192: + of.write("".join(output_buffer)) + output_buffer.clear() + + # Write remaining buffer content to file + if output_buffer: + of.write("".join(output_buffer)) + + +# Function to convert a hex string to a list of integers +def convert_hex_string_to_ints(hex_str): + # Remove spaces and convert to upper case + clean_str = hex_str.replace(" ", "").upper() + # Convert every two characters to an integer + return [int(clean_str[i : i + 2], 16) for i in range(0, len(clean_str), 2)] + + +# pre-compiled regex patterns +cmd_pattern = re.compile(r"^([\w\s]+)") +arg_pattern = re.compile(r"\[([0-9A-Fa-f\s]*)\]") + + +def encode_control_code(cont_code_str: str, start_idx: int = 0, end_idx: int = None): + sliced_str = cont_code_str[start_idx:end_idx] # bad but necessary in python + cmd_match = cmd_pattern.match(sliced_str) + if not cmd_match: + raise ValueError("Missing command in control code!") + + cmd = cmd_match.group(1).strip() + args = arg_pattern.findall(sliced_str) + + cmd_idx = COMMANDS.index(cmd) + arg_list = [ + byte for hex_str in args for byte in convert_hex_string_to_ints(hex_str) + ] + return cmd_idx, arg_list + + +def encode_entry(entry: str): + ba = bytearray() + i = 0 + max = len(entry) + while i < max: + char = entry[i] + if char == "<" and i < max - 1 and entry[i + 1] == "<": + start = i + 2 + end = start + found = False + while end < max: + if entry[end] == ">" and end < max - 1 and entry[end + 1] == ">": + found = True + break + end += 1 + if found: + cmd_idx, arg_list = encode_control_code(entry, start, end) + cmd_size = CONT_SIZES[cmd_idx] + if len(arg_list) != cmd_size - 2: + raise ValueError( + f"Expected args of length {cmd_size - 2} for command {COMMANDS[cmd_idx]}, but got {len(arg_list)}" + ) + ba.append(0x7F) + ba.append(cmd_idx) + ba.extend(arg_list) + i = end + 2 + continue + + ba.append(CHAR_MAP.index(char)) + i += 1 + return ba + + +def encode_file( + file_path: str, + data_path: str, + table_path: str, + data_size: int = -1, + table_size: int = -1, +): + entries = {} + current_entry = None + recording = False + + with open(file_path, "r") as tf, open(data_path, "wb") as df, open( + table_path, "wb" + ) as tabf: + for line in tf: + stripped_line = line.strip() + + # Check for entry start + if stripped_line.startswith("[[ENTRY") and stripped_line.endswith("START]]"): + entry_index = stripped_line.split()[1] # Assuming the format [[ENTRY X START]] + current_entry = entry_index + entries[current_entry] = [] + recording = True + continue + + # Check for entry end + if ( + line.find("<>") != -1 + or line.find("<>") + ): + entries[current_entry].append(line) + recording = False + continue + + # Record lines if within an entry and not empty + if recording and line: + entries[current_entry].append(line) + + # end_ofs = 0 + for entry in entries: + entries[entry] = encode_entry("".join(entries[entry]).rstrip()) + df.write(entries[entry]) + tabf.write(struct.pack(">I", df.tell())) + + if data_size > 0: + data_remain = data_size - df.tell() + if data_remain > 0: + df.write(b"\x00" * data_remain) + + if table_size > 0: + table_remain = table_size - tabf.tell() + if table_remain > 0: + tabf.write(b"\x00" * table_remain) + + return entries + + +def main(): + parser = argparse.ArgumentParser( + description="Pack or dump Animal Crossing text files." + ) + parser.add_argument("-m", help="The mode to run. Valid arguments are un[pack].") + parser.add_argument("path", help="The path of the source file.") + parser.add_argument("out", help="The path of the destination file.") + parser.add_argument( + "--data_size", + help="Optional hexadecimal padded size for the data file.", + required=False, + ) + parser.add_argument( + "--table_size", + help="Optional hexadecimal padded size for the table file.", + required=False, + ) + + args = parser.parse_args() + if args.m.lower() == "pack": + # Create *_table.bin path + dir_name, file_name = os.path.split(args.out) + name, ext = os.path.splitext(file_name) + new_file_name = f"{name}_table{ext}" + + # encode + encode_file( + args.path, + args.out, + os.path.join(dir_name, new_file_name), + int(args.data_size, 16) if args.data_size is not None else -1, + int(args.table_size, 16) if args.table_size is not None else -1, + ) + elif args.m.lower() == "unpack": + # Search for *_table.bin + dir_name, file_name = os.path.split(args.path) + name, ext = os.path.splitext(file_name) + new_file_name = f"{name}_table{ext}" + table_path = os.path.join(dir_name, new_file_name) + + if not os.path.exists(table_path): + raise Exception( + f"Couldn't find a valid table path. Please ensure {new_file_name} exists!" + ) + + # decode + decode_file(args.path, table_path, args.out) + else: + raise Exception("Invalid mode! Please use -m un[pack]") + + +if __name__ == "__main__": + main() diff --git a/tools/pyjkernel/LICENSE b/tools/pyjkernel/LICENSE new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/tools/pyjkernel/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/tools/pyjkernel/__init__.py b/tools/pyjkernel/__init__.py new file mode 100644 index 00000000..8233b8ee --- /dev/null +++ b/tools/pyjkernel/__init__.py @@ -0,0 +1,5 @@ +__version__ = "0.2.1" +__author__ = "Aurum" + +from .jkrcomp import * +from .jkrarchive import * diff --git a/tools/pyjkernel/__main__.py b/tools/pyjkernel/__main__.py new file mode 100644 index 00000000..66c6108b --- /dev/null +++ b/tools/pyjkernel/__main__.py @@ -0,0 +1,6 @@ +def main(): + print("Not implemented yet.") + + +if __name__ == '__main__': + main() diff --git a/tools/pyjkernel/jkrarchive.py b/tools/pyjkernel/jkrarchive.py new file mode 100644 index 00000000..99af8711 --- /dev/null +++ b/tools/pyjkernel/jkrarchive.py @@ -0,0 +1,718 @@ +import enum +import struct + +from . import jkrcomp + +__all__ = [ + "JKRArchiveException", "JKRArchive", "JKRArchiveFile", "JKRPreloadType", "create_new_archive", + "from_archive_buffer", "from_archive_file", "write_archive_file", "write_archive_buffer" +] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Exception for JKRArchive-related actions. +# ---------------------------------------------------------------------------------------------------------------------- +class JKRArchiveException(Exception): + """ + Signals that an error occurred during any JKRArchive-related action. + """ + pass + + +# ---------------------------------------------------------------------------------------------------------------------- +# Declarations of folder node and directory entry structures. These low-level classes will not be visibly exported by +# the module. Instead, JKRArchive handles these structures automatically. The module offers a high-level file accessor +# for file directories instead. Some helper functions to read and write data can be found here as well. +# ---------------------------------------------------------------------------------------------------------------------- +def _file_name_to_hash_(file_name: str) -> int: + file_hash = 0 + for ch in file_name.encode("ascii"): + file_hash = (file_hash * 3) + ch + return file_hash & 0xFFFF + + +def _calc_node_identifier_(dir_name: str, is_root: bool) -> int: + # Root node uses "ROOT" as identifier + if is_root: + return 0x524F4F54 + + enc_upper = dir_name.upper().encode("ascii") + len_enc_name = len(enc_upper) + + identifier = 0 + for i in range(4): + identifier <<= 8 + if i >= len_enc_name: + identifier += 0x20 + else: + identifier += enc_upper[i] + + return identifier + + +class SNodeEntry: + __STRUCT_BE__ = struct.Struct(">2I2HI") + __STRUCT_LE__ = struct.Struct("<2I2HI") + + def __init__(self): + self._identifier_ = 0 # Updated by "SNodeEntry._pack_". Currently not used by the module. + self._off_name_ = 0 # Updated by "JKRArchive._pack_". + self._hash_ = 0 # Updated by "SNodeEntry._pack_". Currently not used by the module. + self._num_files_ = 0 # Updated by "SNodeEntry._pack_". + self._idx_files_start_ = 0 # Updated by "JKRArchive._fix_nodes_and_directories_". + + self._archive_ = None # The archive to which this node belongs. + self._name_ = "" # The node's folder name. + self._dirs_ = list() # The directories held by this node. + + def _unpack_(self, data, off: int, is_big_endian: bool): + strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__ + + self._identifier_, self._off_name_, self._hash_, self._num_files_, self._idx_files_start_ = strct.unpack_from(data, off) + + def _pack_(self, data, off: int, is_big_endian: bool): + strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__ + + self._identifier_ = _calc_node_identifier_(self._name_, self._archive_._root_ == self) + self._hash_ = _file_name_to_hash_(self._name_) + self._num_files_ = len(self._dirs_) + + strct.pack_into(data, off, self._identifier_, self._off_name_, self._hash_, self._num_files_, self._idx_files_start_) + + +class SDirEntry: + __STRUCT_BE__ = struct.Struct(">2H4I") + __STRUCT_LE__ = struct.Struct("<2H4I") + + def __init__(self): + self._index_ = 0xFFFF # Updated whenever a file gets created, deleted, etc. + self._hash_ = 0 # Updated by "SDirEntry._pack_". Currently not used by the module. + self._attributes_ = 0 # Updated by "SDirEntry._pack_". + self._off_name_ = 0 # Updated by "JKRArchive._pack_". + self._off_file_data_ = 0 # Updated by "JKRArchive._pack_" (files) / "._fix_node_and_directories_" (folders). + self._len_file_data_ = 0 # Updated by "SDirEntry._pack_". + self._unk10_ = 0 # Currently not used by the module. + + self._archive_ = None # The archive to which this directory belongs. + self._name_ = "" # The directory's name. + self._parent_ = None # The parent node to which this directory belongs. + self._node_ = None # The node that this directory represents (only for folders). + self._file_ = None # The file accessor that this directory represents (only for files). + + def _unpack_(self, data, off: int, is_big_endian: bool): + strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__ + + self._index_, self._hash_, self._attributes_, self._off_file_data_, self._len_file_data_, self._unk10_ = strct.unpack_from(data, off) + + self._off_name_ = self._attributes_ & 0x00FFFFFF + self._attributes_ = self._attributes_ >> 24 + + def _pack_(self, data, off: int, is_big_endian: bool): + strct = self.__STRUCT_BE__ if is_big_endian else self.__STRUCT_LE__ + + # File + if self._file_ is not None: + self._attributes_ = DirAttr.FILE + self._len_file_data_ = len(self._file_.data) + + if self._file_.compression == jkrcomp.JKRCompression.SZS: + self._attributes_ |= DirAttr.COMPRESSED | DirAttr.USE_SZS + elif self._file_.compression == jkrcomp.JKRCompression.SZP: + self._attributes_ |= DirAttr.COMPRESSED + + if self._file_.preload == JKRPreloadType.MRAM: + self._attributes_ |= DirAttr.PRELOAD_MRAM + elif self._file_.preload == JKRPreloadType.ARAM: + self._attributes_ |= DirAttr.PRELOAD_ARAM + elif self._file_.preload == JKRPreloadType.DVD: + self._attributes_ |= DirAttr.PRELOAD_DVD + # Folder + else: + self._attributes_ = DirAttr.FOLDER + self._index_ = 0xFFFF + self._len_file_data_ = 0x10 + + self._hash_ = _file_name_to_hash_(self._name_) + attr = (self._attributes_ << 24) | self._off_name_ + + strct.pack_into(data, off, self._index_, self._hash_, attr, self._off_file_data_, self._len_file_data_, self._unk10_) + + @property + def is_file(self): + return self._file_ is not None + + @property + def is_folder(self): + return self._file_ is None and self._name_ not in [".", ".."] + + @property + def is_shortcut(self): + return self._file_ is None and self._name_ in [".", ".."] + + +class DirAttr(enum.IntFlag): + FILE = 1 + FOLDER = 2 + COMPRESSED = 4 + _8 = 8 + PRELOAD_MRAM = 16 + PRELOAD_ARAM = 32 + PRELOAD_DVD = 64 + USE_SZS = 128 + + +def _read_string_(data, offset: int) -> str: + end = offset + while end < len(data) - 1 and data[end] != 0: + end += 1 + return data[offset:end + 1].decode("ascii").strip("\0") + + +def _get_aligned_size_(val: int) -> int: + a = val & 31 + if a: + return val + (32 - a) + return val + + +def _get_alignment_(data) -> bytes: + pad_len = len(data) & 31 + return bytes((32 - pad_len) if pad_len else 0) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Declarations of high-level file accessor and enumerations. The file accessor "wraps" a file directory from an archive. +# These will be visibly exported by the module. +# ---------------------------------------------------------------------------------------------------------------------- +class JKRPreloadType(enum.Enum): + MRAM = 0 + ARAM = 1 + DVD = 2 + + +class JKRArchiveFile: + def __init__(self): + self.data = None + self.preload = JKRPreloadType.MRAM + self._dir_ = None + + def __repr__(self): + return self._dir_._name_ + + @property + def name(self) -> str: + return self._dir_._name_ + + @property + def archive(self): + return self._dir_._archive_ + + @property + def index(self) -> int: + return self._dir_._index_ + + @property + def compression(self) -> jkrcomp.JKRCompression: + return jkrcomp.check_compression(self.data) + + +# ---------------------------------------------------------------------------------------------------------------------- +# JKRArchive implementation according to the RARC / ResourceArchive format +# ---------------------------------------------------------------------------------------------------------------------- +class JKRArchive: + __STRUCT_HEADER_BE__ = struct.Struct(">8I") + __STRUCT_HEADER_LE__ = struct.Struct("<8I") + __STRUCT_INFO_BE__ = struct.Struct(">6IH?") + __STRUCT_INFO_LE__ = struct.Struct("<6IH?") + __MAGIC__ = 0x52415243 # "RARC" hexspeak + + def __init__(self): + self._root_ = None # The archive's root node. + self._nodes_ = list() # All of the folder nodes. + self._dirs_ = list() # Collection of all directory entries. + self._next_file_id_ = 0 # The next file ID to be used. + self._sync_file_ids_ = True # Synchronizes file IDs with dir entry index. + + # Lookup maps for fast file and node access + self._lookup_files_ = dict() + self._lookup_nodes_ = dict() + + # Temporary data storage; used during saving process + self._mram_files_ = list() + self._aram_files_ = list() + self._dvd_files_ = list() + + # ------------------------------------------------------------------------------------------------------------------ + # Packing and unpacking + # ------------------------------------------------------------------------------------------------------------------ + def _unpack_(self, data, is_big_endian: bool): + # Attempt to decompress buffer first + data = jkrcomp.decompress(data) + + if struct.unpack_from(">I" if is_big_endian else " bytearray: + # Sort files into respective preload memory groups + for dir_entry in self._dirs_: + if dir_entry._file_ is not None: + if dir_entry._file_.preload == JKRPreloadType.MRAM: + self._mram_files_.append(dir_entry) + elif dir_entry._file_.preload == JKRPreloadType.ARAM: + self._aram_files_.append(dir_entry) + elif dir_entry._file_.preload == JKRPreloadType.DVD: + self._dvd_files_.append(dir_entry) + + # Gather information about the archive and prepare output buffer + num_nodes = len(self._nodes_) + off_nodes = 0x20 + num_dirs = len(self._dirs_) + off_dirs = _get_aligned_size_(off_nodes + num_nodes * 0x10) + total_file_size = _get_aligned_size_(0x20 + off_dirs + num_dirs * 0x14) + off_strings = total_file_size - 0x20 + + buffer = bytearray(total_file_size) + + # Collect strings; pool always starts out like this + string_pool = bytearray() + string_pool += f".\0..\0{self._root_._name_}\0".encode("ascii") + self._root_._off_name_ = 5 + + def collect_strings(folder_node: SNodeEntry): + nonlocal string_pool + + for dir_entry in folder_node._dirs_: + if dir_entry._name_ == ".": + dir_entry._off_name_ = 0 + elif dir_entry._name_ == "..": + dir_entry._off_name_ = 2 + else: + dir_entry._off_name_ = len(string_pool) + string_pool += (dir_entry._name_ + "\0").encode("ascii") + + if dir_entry._file_ is None and dir_entry._name_ not in [".", ".."]: + dir_entry._node_._off_name_ = dir_entry._off_name_ + collect_strings(dir_entry._node_) + + collect_strings(self._root_) + string_pool += _get_alignment_(string_pool) + buffer += string_pool + len_strings = len(string_pool) + del string_pool + + # Write folder nodes + off_tmp = 0x40 + + for folder_node in self._nodes_: + folder_node._pack_(buffer, off_tmp, is_big_endian) + off_tmp += 0x10 + + # Write file data + data_start = len(buffer) + + def write_file_data(files: list) -> int: + nonlocal buffer + categorized_data_start = len(buffer) + + for dir_entry in files: + dir_entry._off_file_data_ = len(buffer) - data_start + buffer += dir_entry._file_.data + buffer += _get_alignment_(dir_entry._file_.data) + + files.clear() + return len(buffer) - categorized_data_start + + mram_size = write_file_data(self._mram_files_) + aram_size = write_file_data(self._aram_files_) + dvd_size = write_file_data(self._dvd_files_) + + # Write dir entries + off_tmp = 0x20 + off_dirs + + for dir_entry in self._dirs_: + dir_entry._pack_(buffer, off_tmp, is_big_endian) + off_tmp += 0x14 + + # Write header and info block + strct_header = self.__STRUCT_HEADER_BE__ if is_big_endian else self.__STRUCT_HEADER_LE__ + strct_info = self.__STRUCT_INFO_BE__ if is_big_endian else self.__STRUCT_INFO_LE__ + total_file_size = len(buffer) + + strct_header.pack_into(buffer, 0, + self.__MAGIC__, total_file_size, 0x20, data_start - 0x20, + total_file_size - data_start, mram_size, aram_size, dvd_size) + strct_info.pack_into(buffer, 0x20, + num_nodes, off_nodes, num_dirs, off_dirs, + len_strings, off_strings, self._next_file_id_, self._sync_file_ids_) + + return buffer + + # ------------------------------------------------------------------------------------------------------------------ + # Helper functions for structure + # ------------------------------------------------------------------------------------------------------------------ + def _initialize_lookup_(self): + path = self._root_._name_.lower() + self._lookup_nodes_[path] = self._root_ + self._initialize_lookup_node_(self._root_, path) + + def _initialize_lookup_node_(self, folder_node: SNodeEntry, current_path: str): + for dir_entry in folder_node._dirs_: + if dir_entry.is_file: + path = current_path + "/" + dir_entry._name_.lower() + self._lookup_files_[path] = dir_entry._file_ + elif dir_entry.is_folder: + next_node = dir_entry._node_ + path = current_path + "/" + next_node._name_.lower() + self._lookup_nodes_[path] = next_node + self._initialize_lookup_node_(next_node, path) + + def _fix_nodes_and_directories_(self): + self._dirs_.clear() + self._fix_node_and_directories_(self._root_) + self._recalculate_file_indices_() + + def _fix_node_and_directories_(self, folder_node: SNodeEntry): + # Put the shortcut directories ("." and "..") at the end of the node's directory list + folders = list() + shortcuts = list() + + for dir_entry in list(folder_node._dirs_): + if dir_entry.is_shortcut: + shortcuts.append(dir_entry) + elif dir_entry.is_folder: + folders.append(dir_entry) + + for subdir in shortcuts: + node_index = self._nodes_.index(subdir._node_) if subdir._node_ is not None else 0xFFFFFFFF + subdir._off_file_data_ = node_index + + folder_node._dirs_.remove(subdir) + folder_node._dirs_.append(subdir) + + # Update the node's directory span + folder_node._idx_files_start_ = len(self._dirs_) + self._dirs_ += folder_node._dirs_ + + # Handle subfolders as well + for subdir in folders: + node_index = self._nodes_.index(subdir._node_) + subdir._off_file_data_ = node_index + + self._fix_node_and_directories_(subdir._node_) + + def _recalculate_file_indices_(self): + if self._sync_file_ids_: + self._next_file_id_ = len(self._dirs_) + + for i, dir_entry in enumerate(self._dirs_): + if dir_entry.is_file: + dir_entry._index_ = i + else: + file_id = 0 + + for dir_entry in self._dirs_: + if dir_entry.is_file: + dir_entry._index_ = file_id + file_id += 1 + + self._next_file_id_ = file_id + + def _create_dir_entry_(self, name: str, attr: int, node: SNodeEntry, parent_node: SNodeEntry) -> SDirEntry: + dir_entry = SDirEntry() + dir_entry._name_ = name + dir_entry._attributes_ = attr + dir_entry._node_ = node + dir_entry._parent_ = parent_node + dir_entry._archive_ = self + + parent_node._dirs_.append(dir_entry) + + return dir_entry + + def _create_root_(self, root_name: str): + if self._root_: + raise JKRArchiveException("Fatal! Root already exists!") + + root_node = SNodeEntry() + root_node._name_ = root_name + root_node._archive_ = self + self._root_ = root_node + self._nodes_.append(root_node) + + self._create_dir_entry_(".", DirAttr.FOLDER, root_node, root_node) + self._create_dir_entry_("..", DirAttr.FOLDER, None, root_node) + + self._lookup_nodes_[root_name.lower()] = root_node + + self._fix_nodes_and_directories_() + + # ------------------------------------------------------------------------------------------------------------------ + # High-level operations + # ------------------------------------------------------------------------------------------------------------------ + @property + def sync_file_ids(self) -> bool: + return self._sync_file_ids_ + + @sync_file_ids.setter + def sync_file_ids(self, val: bool): + self._sync_file_ids_ = val + self._recalculate_file_indices_() + + @property + def root_name(self) -> str: + return self._root_._name_ + + def directory_exists(self, file_path: str) -> bool: + key = file_path.lower() + return key in self._lookup_files_ or key in self._lookup_nodes_ + + def list_files(self, folder_path: str) -> list: + key = folder_path.lower() + if key not in self._lookup_nodes_: + raise JKRArchiveException(f"The folder {folder_path} does not exist!") + return list(de._file_ for de in filter(lambda de: de.is_file, self._lookup_nodes_[key]._dirs_)) + + def list_folders(self, folder_path: str) -> list: + key = folder_path.lower() + if key not in self._lookup_nodes_: + raise JKRArchiveException(f"The folder {folder_path} does not exist!") + return list(de._name_ for de in filter(lambda de: de.is_folder, self._lookup_nodes_[key]._dirs_)) + + def get_file(self, file_path: str) -> JKRArchiveFile: + key = file_path.lower() + if key not in self._lookup_files_: + raise JKRArchiveException(f"The file {file_path} does not exist!") + return self._lookup_files_[key] + + def create_folder(self, folder_path: str): + # Check if directory already exists + full_key = folder_path.lower() + + if full_key in self._lookup_nodes_ or full_key in self._lookup_files_: + raise JKRArchiveException(f"The directory {folder_path} already exists!") + + # Get parent folder first + split_path = folder_path.rsplit("/", 1) + + if len(split_path) == 1: + return None + + folder_path, folder_name = split_path + folder_path_key = folder_path.lower() + + if folder_path_key not in self._lookup_nodes_: + raise JKRArchiveException("Cannot create folder. Archive does not contain the folder " + folder_path) + + folder_node = self._lookup_nodes_[folder_path_key] + + # Create new node and directories + new_folder_node = SNodeEntry() + new_folder_node._name_ = folder_name + new_folder_node._archive_ = self + self._nodes_.append(new_folder_node) + + self._create_dir_entry_(folder_name, DirAttr.FOLDER, new_folder_node, folder_node) + self._create_dir_entry_(".", DirAttr.FOLDER, new_folder_node, new_folder_node) + self._create_dir_entry_("..", DirAttr.FOLDER, folder_node, new_folder_node) + + self._lookup_nodes_[full_key] = new_folder_node + + self._fix_nodes_and_directories_() + + def create_file(self, file_path: str, data=bytearray(), preload: JKRPreloadType = JKRPreloadType.MRAM) -> JKRArchiveFile: + # Check if directory already exists + full_key = file_path.lower() + + if full_key in self._lookup_nodes_ or full_key in self._lookup_files_: + raise JKRArchiveException(f"The directory {file_path} already exists!") + + # Get parent folder first + split_path = file_path.rsplit("/", 1) + + if len(split_path) == 1: + return None + + folder_path, file_name = split_path + folder_path_key = folder_path.lower() + + if folder_path_key not in self._lookup_nodes_: + raise JKRArchiveException("Cannot create file. Archive does not contain the folder " + folder_path) + + folder_node = self._lookup_nodes_[folder_path_key] + + # Create new directory and file entries + dir_entry = self._create_dir_entry_(file_name, DirAttr.FILE, None, folder_node) + new_file = JKRArchiveFile() + new_file.data = data + new_file.preload = preload + new_file._dir_ = dir_entry + dir_entry._file_ = new_file + + self._lookup_files_[full_key] = new_file + + self._fix_nodes_and_directories_() + return new_file + + def remove_file(self, file_path: str): + key = file_path.lower() + + if key not in self._lookup_files_: + raise JKRArchiveException(f"The file {file_path} does not exist!") + + file_access = self._lookup_files_[key] + dir_entry = file_access._dir_ + parent_node = dir_entry._parent_ + + # Detach file and clear access + parent_node._dirs_.remove(dir_entry) + dir_entry._parent_ = None + dir_entry._archive_ = None + dir_entry._file_ = None + file_access.data = None + file_access._dir_ = None + + self._lookup_files_.pop(key) + self._dirs_.remove(dir_entry) + self._recalculate_file_indices_() + + def remove_folder(self, folder_path: str): + raise NotImplementedError + + def __repr__(self): + return self._print_(self._root_, 0) + + def _print_(self, folder_node: SNodeEntry, depth: int): + indent = " " * depth + result = indent + folder_node._name_ + "\n" + + folders = list() + files = list() + + for dir_entry in folder_node._dirs_: + if dir_entry.is_file: + files.append(dir_entry._file_) + elif dir_entry.is_folder: + folders.append(dir_entry._node_) + + for folder_node in folders: + result += self._print_(folder_node, depth + 1) + + for file_access in files: + result += f" {indent}{file_access.name}\n" + + return result + + +# ---------------------------------------------------------------------------------------------------------------------- +# Helper I/O and creation functions +# ---------------------------------------------------------------------------------------------------------------------- +def create_new_archive(root_name: str, sync_file_ids: bool = True): + jkrarc = JKRArchive() + jkrarc._sync_file_ids_ = sync_file_ids + jkrarc._create_root_(root_name) + return jkrarc + + +def from_archive_buffer(buffer, big_endian: bool = True) -> JKRArchive: + jkrarc = JKRArchive() + jkrarc._unpack_(buffer, big_endian) + return jkrarc + + +def from_archive_file(file_path: str, big_endian: bool = True) -> JKRArchive: + jkrarc = JKRArchive() + with open(file_path, "rb") as f: + jkrarc._unpack_(f.read(), big_endian) + return jkrarc + + +def write_archive_buffer(jkrarc: JKRArchive, big_endian: bool = True, compression: jkrcomp.JKRCompression = jkrcomp.JKRCompression.NONE, level: int = 7): + return jkrcomp.compress(jkrarc._pack_(big_endian), compression, level) + + +def write_archive_file(jkrarc: JKRArchive, file_path: str, big_endian: bool = True, compression: jkrcomp.JKRCompression = jkrcomp.JKRCompression.NONE, level: int = 7): + buffer = jkrcomp.compress(jkrarc._pack_(big_endian), compression, level) + + with open(file_path, "wb") as f: + f.write(buffer) + f.flush() diff --git a/tools/pyjkernel/jkrcomp.py b/tools/pyjkernel/jkrcomp.py new file mode 100644 index 00000000..c290d59e --- /dev/null +++ b/tools/pyjkernel/jkrcomp.py @@ -0,0 +1,187 @@ +import enum +import struct + +__all__ = [ + "JKRCompression", "check_compression", "decompress", "compress", + "decompress_szs", "compress_szs", "decompress_szp", "compress_szp" +] + + +class JKRCompression(enum.Enum): + """ + A constant representing a JKernel compression format or no compression at all. + """ + NONE = 0 # Use no compression at all + SZP = 1 # Nintendo's older compression format used in some Nintendo 64 and GameCube games + SZS = 2 # Nintendo's newer compression format used since GameCube games + + +def check_compression(data) -> JKRCompression: + """ + Peeks at the input data's first four bytes to determine what JKernel compression format was used to compress the + data. A constant representing the respective compression format will be returned. + + :param data: the input buffer to be inspected. + :return: a constant representing a JKernel compression format or no compression at all. + """ + # Magic is Ya_0 + if data[0] == 0x59 and data[1] == 0x61 and data[3] == 0x30: + # Yaz0 -> SZS + if data[2] == 0x7A: + return JKRCompression.SZS + # Yay0 -> SZP + elif data[2] == 0x79: + return JKRCompression.SZP + + return JKRCompression.NONE + + +def decompress(data) -> bytes: + """ + Attempts to decompress the input data using JKernel decompression algorithms. If no JKernel compression format was + detected, the input buffer will be returned again. + + :param data: the buffer to be decompressed. + :return: a bytes object containing the decompressed data or the input buffer if no compressed data was found. + """ + # Magic is Ya_0 + if data[0] == 0x59 and data[1] == 0x61 and data[3] == 0x30: + # Yaz0 -> SZS + if data[2] == 0x7A: + return decompress_szs(data) + # Yay0 -> SZP + elif data[2] == 0x79: + return __decompress_szp__(data) + + return data + + +def compress(data, compression: JKRCompression, level: int = 7) -> bytes: + """ + Attempts to compress the input data using the specified JKernel compression format. If no JKernel compression format + was detected, the input buffer will be returned again. SZP compression is not implemented yet. Therefore, attempting + to compress a buffer with this algorithm yields a ``NotImplementedError``. + + :param data: the buffer to be compressed. + :param compression: the compression algorithm to be used. + :param level: the compression level (6 to 9; 6 is fastest and 9 is slowest). + :return: a bytes object containing the compressed data or the input buffer if no compression was specified. + """ + if compression == JKRCompression.SZS: + return compress_szs(data, level) + elif compression == JKRCompression.SZP: + return compress_szp(data, level) + + return data + + +def decompress_szs(data) -> bytes: + """ + Decompresses SZS-encoded input data and returns the decompressed bytes. This checks if the four magic bytes are + equal to the string "Yaz0" to ensure that the buffer contains SZS data. The input buffer will be returned in case + this check fails. Otherwise, the actual decompression will occur. + + :param data: the buffer to be decompressed. + :returns: a bytes object containing the decompressed data or the input buffer if no compressed data was found. + """ + if data[0] == 0x59 and data[1] == 0x61 and data[2] == 0x7A and data[3] == 0x30: + raise NotImplementedError("SZS decompression is not supported yet.") + + return data + + +def compress_szs(data, level: int = 7) -> bytes: + """ + Compresses the input data in the SZS compression format and returns the compressed bytes. + + :param data: the buffered data to be compressed. + :param level: the compression level (6 to 9; 6 is fastest and 9 is slowest). + :return: the compressed data. + """ + raise NotImplementedError("SZS compression is not supported yet.") + + +def decompress_szp(data) -> bytes: + """ + Decompresses SZP-encoded input data and returns the decompressed bytes. This checks if the four magic bytes are + equal to the string "Yay0" to ensure that the buffer contains SZP data. The input buffer will be returned in case + this check fails. Otherwise, the actual decompression will occur. + + :param data: the buffer to be decompressed. + :returns: a bytes object containing the decompressed data or the input buffer if no compressed data was found. + """ + if data[0] == 0x59 and data[1] == 0x61 and data[2] == 0x79 and data[3] == 0x30: + return __decompress_szp__(data) + + return data + + +def __decompress_szp__(data) -> bytes: + # Parse header and prepare output buffer + decompressed_size, off_copy_table, off_chunks = struct.unpack_from(">3I", data, 0x4) + decompressed = bytearray(decompressed_size) + + off_in = 16 # Compressed data comes after header + off_out = 0 + + block = 0 # The control block that describes how to decompress data, 32-bit + counter = 0 # Keeps track of the remaining bits to be checked for the current control block + + while off_out < decompressed_size: + # Get control block, which is a 32-bit word describing how to decompress data from the input buffer. Like SZS, + # the bits are read starting from the most significant bit. If the bit is set, we copy the next byte in the byte + # chunk table. Otherwise, we read information from the copy table to determine which decompressed bytes to copy + # into the output buffer. + if counter == 0: + block = struct.unpack_from(">I", data, off_in)[0] + counter = 32 + off_in += 4 + + # Is the most significant bit set? If so, copy a plain byte into the output buffer. + if block & 0x80000000: + decompressed[off_out] = data[off_chunks] + off_chunks += 1 + off_out += 1 + # Otherwise, read and copy decompressed data. + else: + # Read tokens + b1 = data[off_copy_table] + b2 = data[off_copy_table + 1] + off_copy_table += 2 + + # Get copy offset and size + dist = ((b1 & 0xF) << 8) | b2 + off_copy = off_out - dist - 1 + len_copy = b1 >> 4 + + # Copy 18+ bytes? + if len_copy == 0: + len_copy = data[off_chunks] + 18 + off_chunks += 1 + # Copy up to 17 bytes + else: + len_copy += 2 + + # Copy the actual data + for _ in range(len_copy): + decompressed[off_out] = decompressed[off_copy] + off_out += 1 + off_copy += 1 + + # Left-shift control block and decrement remaining bits to be checked + block <<= 1 + counter -= 1 + + return bytes(decompressed) + + +def compress_szp(data, level: int = 7) -> bytes: + """ + Compresses the input data in the SZP compression format and returns the compressed bytes. + Not implemented yet. + + :param data: the buffered data to be compressed. + :param level: the compression level. + :return: the compressed data. + """ + raise NotImplementedError("SZP compression is not supported yet.") diff --git a/tools/texture_tool.py b/tools/texture_tool.py new file mode 100644 index 00000000..1886ba7d --- /dev/null +++ b/tools/texture_tool.py @@ -0,0 +1,562 @@ +import argparse +import os +import subprocess +import sys +import numpy as np +from pathlib import Path +from PIL import Image + +current_path = sys.path.copy() +sys.path.append(str(Path(__file__).parent.parent)) + +sys.path = current_path + + +def is_windows() -> bool: + return os.name == "nt" + + +def unswizzle(input_list, width, height, pixels_per_block_w=8, pixels_per_block_h=8): + if width * height > len(input_list): + raise Exception( + f"There are not enough elements in input_list for the specified Width and Height!" + f"\nExpected a length of {width * height}, but got a length of {len(input_list)}!" + ) + + block_x_count = width // pixels_per_block_w + block_y_count = height // pixels_per_block_h + if block_y_count < 1: + block_y_count = 1 # Hack for small textures -- Not sure if this is correct. + + output_buffer = [None] * len(input_list) + pixel_index = 0 + + for y_block in range(block_y_count): + for x_block in range(block_x_count): + for y_pixel in range(pixels_per_block_h): + for x_pixel in range(pixels_per_block_w): + output_buffer_index = ( + (width * pixels_per_block_h * y_block) + + y_pixel * width + + x_block * pixels_per_block_w + + x_pixel + ) + output_buffer[output_buffer_index] = input_list[pixel_index] + pixel_index += 1 + + return output_buffer + + +def swizzle(input_list, width, height, pixels_per_block_w=8, pixels_per_block_h=8): + if width * height > len(input_list): + raise Exception( + f"There are not enough elements in input_list for the specified Width and Height!\n" + f"Width = {width} | Height = {height} | Width * Height = {width * height} | Input List Length = {len(input_list)}" + ) + + block_x_count = width // pixels_per_block_w + block_y_count = height // pixels_per_block_h + + output_buffer = [None] * len(input_list) + output_buffer_index = 0 + + for y_block in range(block_y_count): + for x_block in range(block_x_count): + for y_pixel in range(pixels_per_block_h): + for x_pixel in range(pixels_per_block_w): + pixel_index = ( + (width * pixels_per_block_h * y_block) + + y_pixel * width + + x_block * pixels_per_block_w + + x_pixel + ) + output_buffer[output_buffer_index] = input_list[pixel_index] + output_buffer_index += 1 + + return output_buffer + + +def read_rgb5a3_colors(input_bytes, n): + """ + Reads N unsigned 16-bit values from an input bytearray as big-endian RGB5A3 colors. + + Args: + - input_bytes (bytearray): The input bytearray. + - n (int): The number of 16-bit values to read. + + Returns: + - List of tuples representing RGB5A3 colors. + """ + if len(input_bytes) < n * 2: + raise ValueError("Input bytearray does not contain enough data for N colors") + + colors = [] + for i in range(n): + # Extract 16-bit value for each color, assuming big-endian order + color_value = int.from_bytes(input_bytes[i * 2 : i * 2 + 2], "big") + colors.append(color_value) + + return colors + + +def to_argb8(pixel): + if (pixel & 0x8000) == 0x8000: + # No Alpha Channel + a = 0xFF + + # Separate RGB from bits + r = (pixel & 0x7C00) >> 10 + g = (pixel & 0x03E0) >> 5 + b = pixel & 0x001F + + # Convert to RGB8 values + r = (r << (8 - 5)) | (r >> (10 - 8)) + g = (g << (8 - 5)) | (g >> (10 - 8)) + b = (b << (8 - 5)) | (b >> (10 - 8)) + else: + # An Alpha Channel Exists, 3 bits for Alpha Channel and 4 bits each for RGB + a = (pixel & 0x7000) >> 12 + r = (pixel & 0x0F00) >> 8 + g = (pixel & 0x00F0) >> 4 + b = pixel & 0x000F + + a = (a << (8 - 3)) | (a << (8 - 6)) | (a >> (9 - 8)) + r = (r << (8 - 4)) | r + g = (g << (8 - 4)) | g + b = (b << (8 - 4)) | b + + # Ensure the values are byte-sized (0-255) + a = a & 0xFF + r = r & 0xFF + g = g & 0xFF + b = b & 0xFF + + return r, g, b, a + + +def rgb5a3_to_argb8(colors): + """ + Converts an array of RGB5A3 colors to standard ARGB8 colors. + + Args: + - colors (List[int]): List of integers representing RGB5A3 colors. + + Returns: + - List of tuples representing ARGB8 colors. + """ + argb8_colors = [] + for color in colors: + argb8_colors.append((to_argb8(color))) + + return argb8_colors + + +def extract_texture_asset(byte_array, width, height, tex_count): + texture_data_size = (width * height) // 2 + entry_size = 2 * 16 + texture_data_size * tex_count + n_entries = len(byte_array) // entry_size + entries = [] + + for entry_index in range(n_entries): + offset = entry_index * entry_size + palette_bytes = byte_array[offset : offset + 2 * 16] + palette_colors = read_rgb5a3_colors(palette_bytes, 16) + + textures = [] + for texture_index in range(tex_count): + texture_offset = offset + 2 * 16 + (texture_index * texture_data_size) + texture_bytes = byte_array[ + texture_offset : texture_offset + texture_data_size + ] + textures.append(texture_bytes) + + entries.append({"palette": palette_colors, "textures": textures}) + + return entries + + +def generate_c_source_entries_with_all_textures(entries, width): + """ + Generates a list of C source strings from extracted palette and textures data. + Each string contains one palette and all textures formatted for a C source file, + where each entry is represented as a separate string in the list. + + Args: + - entries (List[Dict]): Extracted data containing 'palette' and 'textures' for each entry. + + Returns: + - List of strings, each representing the C source code for one entry including one palette and all textures. + """ + c_source_entries = [] + for i, entry in enumerate(entries): + c_source = "" + # Palette + c_source += f"// clang-format off\nunsigned short floor{i:02d}_pal[] = {{\n " + for j, color in enumerate(entry["palette"]): + c_source += f"0x{color:04X}," + if (j + 1) % 8 == 0 and (j + 1) < 16: # Newline after every 8 entries + c_source += "\n " + else: + c_source += " " + c_source = ( + c_source.rstrip(", \n") + "\n};\n// clang-format on\n\n" + ) # Remove trailing comma and add closing bracket + + # Textures + c_source += f"// clang-format off\nunsigned char floor{i:02d}_tex[] = {{\n // texture 0\n " + for texture_index, texture in enumerate(entry["textures"]): + for k, pixel in enumerate(texture): + c_source += f"0x{pixel:02X}," + if (k + 1) % (width // 2) == 0: # Newline after width/2 texture pixels + if ( + k + 1 != len(texture) + or texture_index < len(entry["textures"]) - 1 + ): + c_source += "\n " + else: + c_source += " " + if texture_index < len(entry["textures"]) - 1: + c_source += f"// texture {texture_index + 1}\n " # Add a newline after each texture except the last + c_source = ( + c_source.rstrip(", \n") + "\n};\n// clang-format on\n" + ) # Close texture array + + c_source_entries.append(c_source) + + return c_source_entries + + +def save_palette_as_hex(palette, directory_path): + """ + Saves an RGB5A3 palette list to a text file with hex values. + + Args: + - palette (List[int]): List of colors in RGB5A3 format. + - directory_path (Path): The directory where 'palette.txt' will be saved. + """ + # Ensure the directory exists + directory_path.mkdir(parents=True, exist_ok=True) + + # Define the file path + file_path = directory_path / "palette.txt" + + with file_path.open(mode="w") as file: + for color in palette: + # Write each color as a hexadecimal value + file.write(f"{color:04X}\n") + + +def save_textures_as_png(base_path, entries, width, height, base_name): + for i, entry in enumerate(entries): + folder_name = base_path / f"{base_name}{i:02d}" + folder_name.mkdir( + parents=True, exist_ok=True + ) # Create the directory, including any necessary parent directories + + # Save RGB5A3 palette to 'palette.txt' + save_palette_as_hex(entry["palette"], folder_name) + + # Convert to RGBA8 + palette = rgb5a3_to_argb8(entry["palette"]) + + for j, texture_data in enumerate(entry["textures"]): + # Construct an image from the 4-bit indexed data + img_data = bytearray( + len(texture_data) * 2 + ) # Each byte in texture_data represents two pixels + for idx, byte in enumerate(texture_data): + img_data[idx * 2] = byte >> 4 # High nibble for the first pixel + img_data[idx * 2 + 1] = byte & 0x0F # Low nibble for the second pixel + + # Map the indexed pixels to RGBA using the palette + rgba_img = [palette[pix] for pix in img_data] + + # Unswizzle the RGBA image data + unswizzled_data = unswizzle( + rgba_img, width, height + ) # Assume unswizzle is defined elsewhere + + # Create the final image and save it + img = Image.new("RGBA", (width, height)) + img.putdata(unswizzled_data) + img_file_path = folder_name / f"texture{j:02d}.png" + img.save(img_file_path) + + +def generate_includes(n, input_string): + includes = "" + for i in range(n): + includes += f'#include "./{input_string}{i:02d}.c"\n' + return includes + + +def extract_player_room_floor(byte_array: bytearray, out_dir: str): + entries = extract_texture_asset(byte_array, 64, 64, 4) + entry_sources = generate_c_source_entries_with_all_textures(entries, 64) + Path(out_dir).mkdir(parents=True, exist_ok=True) # make dirs + for i in range(len(entries)): + path = Path(out_dir) / ("floor%02d.c" % i) + with path.open("w") as f: + f.write(entry_sources[i]) + save_textures_as_png(Path(out_dir) / "tex", entries, 64, 64, "floor") + + # write aggregate file + with (Path(out_dir) / "player_room_floor.c").open("w") as f: + f.write(generate_includes(len(entries), "floor")) + + +def extract_player_room_wall(byte_array: bytearray, out_dir: str): + entries = extract_texture_asset(byte_array, 64, 64, 2) + entry_sources = generate_c_source_entries_with_all_textures(entries, 64) + Path(out_dir).mkdir(parents=True, exist_ok=True) # make dirs + for i in range(len(entries)): + path = Path(out_dir) / ("wall%02d.c" % i) + with path.open("w") as f: + f.write(entry_sources[i]) + save_textures_as_png(Path(out_dir) / "tex", entries, 64, 64, "wall") + + # write aggregate file + with (Path(out_dir) / "player_room_wall.c").open("w") as f: + f.write(generate_includes(len(entries), "wall")) + + +# packing + + +def rgba8_to_rgb5a3(r, g, b, a): + if a >= 224: # Treat as opaque + return (1 << 15) | ((r >> 3) << 10) | ((g >> 3) << 5) | (b >> 3) + else: # Use 3 bits for alpha + return ((a >> 5) << 12) | ((r >> 4) << 8) | ((g >> 4) << 4) | (b >> 4) + + +def find_closest_color(color, palette): + min_dist, index = float("inf"), -1 + for i, p in enumerate(palette): + # Directly compare the integer values + dist = (color - p) ** 2 + if dist < min_dist: + min_dist, index = dist, i + return index + + +def load_palette_from_hex(directory_path): + """ + Loads an RGB5A3 palette list from a text file containing hex values. + + Args: + - directory_path (Path): The directory where 'palette.txt' is located. + + Returns: + - List[int]: List of colors in RGB5A3 format. + """ + # Define the file path + file_path = directory_path / "palette.txt" + + palette = [] + with file_path.open(mode="r") as file: + for line in file: + # Convert each hexadecimal string back to an integer + color = int(line.strip(), 16) + palette.append(color) + + return palette + + +def process_png_image(image_path, palette): + img = Image.open(image_path).convert("RGBA") + pixels = np.array(img) + + # Create a texture map with indices + texture_map = [] + for row in pixels: + row_indices = [] + for r, g, b, a in row: + rgb5a3 = rgba8_to_rgb5a3(r, g, b, a) + closest_index = find_closest_color(rgb5a3, palette) + row_indices.append(closest_index) + texture_map.extend(row_indices) + + # Swizzle texture + swizzled_texture = swizzle(texture_map, img.width, img.height) + + # Pack texture into 4bpp texels + packed_texture_map = [0] * (len(swizzled_texture) // 2) + for i, p in enumerate(swizzled_texture): + if (i % 2) == 0: + packed_texture_map[i // 2] = (p & 0xF) << 4 + else: + packed_texture_map[i // 2] |= p & 0xF + + return packed_texture_map + + +def pack_player_room_floor(main_path: Path): + objects = [] + for dir in (main_path / "tex").iterdir(): + # load palette file + palette = load_palette_from_hex(dir) + + # load texture00.png-texture03.png + textures = [] + for i in range(0, 4): + textures.append(process_png_image(dir / f"texture{i:02d}.png", palette)) + + obj = { + "palette": palette, + "textures": textures, + "idx": int(dir.name.replace("floor", "")), + } + objects.append(obj) + + # sort list + objects.sort(key=lambda x: x["idx"]) + + # process texture data into C source + c_source = generate_c_source_entries_with_all_textures(objects, 64) + + # output C source to files + for i in range(len(objects)): + with (main_path / f"floor{i:02d}.c").open("w") as f: + f.write(c_source[i]) + + # build elf file and dump .data section + os.chdir(str(Path(__file__).parent.parent)) + + out_elf = main_path / f"{main_path.name}.o" + + if not is_windows(): + subprocess.run( + [ + "wibo", + "./build/compilers/1.3.2/mwcceppc.exe", + f"-I{str(main_path)}", + "-c", + main_path / f"{main_path.name}.c", + "-o", + out_elf, + ] + ) + else: + subprocess.run( + [ + "./build/compilers/1.3.2/mwcceppc.exe", + f"-I{str(main_path)}", + "-c", + main_path / f"{main_path.name}.c", + "-o", + out_elf, + ] + ) + + out_obj = main_path / f"{main_path.name}.bin" + subprocess.run( + ["powerpc-eabi-objcopy", "--dump-section", f".data={out_obj}", out_elf] + ) + + # restore current dir + os.chdir(str(Path(__file__).parent)) + + +def pack_player_room_wall(main_path: Path): + objects = [] + for dir in (main_path / "tex").iterdir(): + # load palette file + palette = load_palette_from_hex(dir) + + # load texture00.png-texture03.png + textures = [] + for i in range(0, 2): + textures.append(process_png_image(dir / f"texture{i:02d}.png", palette)) + + obj = { + "palette": palette, + "textures": textures, + "idx": int(dir.name.replace("wall", "")), + } + objects.append(obj) + + # sort list + objects.sort(key=lambda x: x["idx"]) + + # process texture data into C source + c_source = generate_c_source_entries_with_all_textures(objects, 64) + + # output C source to files + for i in range(len(objects)): + with (main_path / f"wall{i:02d}.c").open("w") as f: + f.write(c_source[i]) + + # build elf file and dump .data section + os.chdir(str(Path(__file__).parent.parent)) + + out_elf = main_path / f"{main_path.name}.o" + + if not is_windows(): + subprocess.run( + [ + "wibo", + "./build/compilers/1.3.2/mwcceppc.exe", + f"-I{str(main_path)}", + "-c", + main_path / f"{main_path.name}.c", + "-o", + out_elf, + ] + ) + else: + subprocess.run( + [ + "./build/compilers/1.3.2/mwcceppc.exe", + f"-I{str(main_path)}", + "-c", + main_path / f"{main_path.name}.c", + "-o", + out_elf, + ] + ) + + out_obj = main_path / f"{main_path.name}.bin" + subprocess.run( + ["powerpc-eabi-objcopy", "--dump-section", f".data={out_obj}", out_elf] + ) + + +def unpack(): + with open("src/data/bin2/data/player_room_floor.bin", "rb") as f: + extract_player_room_floor( + bytearray(f.read()), "src/data/item/player_room_floor" + ) + + with open("src/data/bin2/data/player_room_wall.bin", "rb") as f: + extract_player_room_wall(bytearray(f.read()), "src/data/item/player_room_wall") + + +def pack(): + cwd = os.getcwd() + + pack_player_room_floor(Path("src/data/item/player_room_floor")) + os.chdir(cwd) + + pack_player_room_wall(Path("src/data/item/player_room_wall")) + os.chdir(cwd) + + +def main(): + parser = argparse.ArgumentParser( + description="Pack or dump Animal Crossing player_room_[floor][wall].bin files." + ) + parser.add_argument("-m", help="The mode to run. Valid arguments are un[pack].") + + args = parser.parse_args() + + if args.m.lower() == "pack": + pack() + elif args.m.lower() == "unpack": + unpack() + else: + raise Exception("Invalid mode! Please use -m un[pack]") + + +if __name__ == "__main__": + main()