diff --git a/Makefile b/Makefile index 86106a8809..986ac8bc1c 100644 --- a/Makefile +++ b/Makefile @@ -156,11 +156,7 @@ tools: dirs $(ELF2DOL) $(YAZ0) assets: @mkdir -p game - $(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game native asset_config.json - -assets-fast: - @mkdir -p game - $(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game oead asset_config.json + $(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game asset_config.json docs: $(DOXYGEN) Doxyfile @@ -193,32 +189,16 @@ shiftedrels: shift $(RELS) game: shiftedrels @mkdir -p game - @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode native asset_config.json - -game-fast: shiftedrels - @mkdir -p game - @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode oead asset_config.json + @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode asset_config.json game-nocompile: @mkdir -p game - @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode native asset_config.json - -game-nocompile-fast: - @mkdir -p game - @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode oead asset_config.json + @$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode asset_config.json rungame-nocompile: game-nocompile @echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin! dolphin-emu $(BUILD_DIR)/game/sys/main.dol -rungame-nocompile-fast: game-nocompile-fast - @echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin! - dolphin-emu $(BUILD_DIR)/game/sys/main.dol - -rungame-fast: game-fast - @echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin! - dolphin-emu $(BUILD_DIR)/game/sys/main.dol - rungame: game @echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin! dolphin-emu $(BUILD_DIR)/game/sys/main.dol @@ -226,7 +206,6 @@ rungame: game iso: game @$(PYTHON) tools/packageISO.py $(BUILD_DIR)/game/ $(TARGET_ISO) - $(BUILD_DIR)/%.o: %.c $(BUILD_DIR)/%.d @mkdir -p $(@D) @echo building... $< diff --git a/tools/assets_config.py b/tools/assets_config.py index 67826018eb..11a7350d0c 100644 --- a/tools/assets_config.py +++ b/tools/assets_config.py @@ -4,6 +4,7 @@ import json from os import path CONFIG_DESCRIPTIONS = { + "oead_yaz0": "Use oead (pip install oead) for asset (de)compression (faster)", "decompress_assets": "Decompress Yaz0 compressed assets (appends .c to the filename)", "extract_arc": "Extract archive (.arc) files", "convert_stages": "Convert Stage Files (.dzs and .dzr) to json", @@ -13,6 +14,7 @@ CONFIG_DESCRIPTIONS = { } CONFIG_DEFAULTS = { + "oead_yaz0": False, "decompress_assets": True, "extract_arc": True, "convert_stages": True, diff --git a/tools/extract_game_assets.py b/tools/extract_game_assets.py index 4c607f4a83..68431c4cff 100644 --- a/tools/extract_game_assets.py +++ b/tools/extract_game_assets.py @@ -170,7 +170,7 @@ def writeFile(name, data): if ("exceptions" in extractDef and str(name) in extractDef["exceptions"]) or ("config_key" in extractDef and config[extractDef["config_key"]] == False): extractDef = None - if config["decompress_assets"] == None: + if config["decompress_assets"] == False: extractDef = None # If assets aren't being decompressed, just write the raw file if extractDef == None: @@ -245,16 +245,16 @@ def getDolInfo(disc): return dolOffset, dolSize -def extract(isoPath: Path, gamePath: Path, yaz0Encoder: str, config_file: str): - if yaz0Encoder == "oead": +def extract(isoPath: Path, gamePath: Path, config_file: str): + global config + config = assets_config.getConfig(config_file,update=True) + if config["oead_yaz0"]: try: from oead import yaz0 global yaz0DecompressFunction yaz0DecompressFunction = yaz0.decompress except: print("Extract: oead isn't installed, falling back to native yaz0") - global config - config = assets_config.getConfig(config_file,update=True) isoPath = isoPath.absolute() cwd = os.getcwd() os.chdir(gamePath) @@ -302,7 +302,7 @@ def extract(isoPath: Path, gamePath: Path, yaz0Encoder: str, config_file: str): def main(): - extract(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4]) + extract(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]) if __name__ == "__main__": diff --git a/tools/packageISO.py b/tools/packageISO.py index a074318e33..4b0f32680a 100644 --- a/tools/packageISO.py +++ b/tools/packageISO.py @@ -3,9 +3,90 @@ import os from pathlib import Path import struct +ISO_MAX_SIZE = 1459978240 + +# Directories towards the top will be at the end of the ISO +# The entries in the BOOT section have priority over the directories listed +TP_FILE_PRIORITY = [ + "rel/", + "./", # All uncaught files + "BOOT", # All files in the BOOT array + "res/Object/", + "res/Stage/", + "Audiores/Waves/", + "Audiores/Stream/", + "res/Particle/", + "Movie/", + "map/" +] + +# There are a few differences here than actually running the game from boot, +# Likely due to version differences. This is the order on GC US +TP_BOOT_PRIORITY = [ + "str/Final/Release/COPYDATE", + "RELS.arc", + "Audiores/Z2Sound.baa", + "Audiores/Seqs/Z2SoundSeqs.arc", + "res/Object/LogoUs.arc", + "res/Object/Always.arc", + "res/Object/Alink.arc", + "res/FieldMap/Field0.arc", + "res/Object/AlAnm.arc", + "res/Layout/fmapres.arc", + "res/Layout/dmapres.arc", + "res/Layout/clctres.arc", + "res/Layout/itemicon.arc", + "res/Layout/ringres.arc", + "res/Layout/playerName.arc", + "res/Layout/itmInfRes.arc", + "res/Layout/button.arc", + "res/CardIcon/cardicon.arc", + "res/Msgus/bmgres.arc", + "res/Layout/msgcom.arc", + "res/Layout/msgres00.arc", + "res/Layout/msgres01.arc", + "res/Layout/msgres02.arc", + "res/Layout/msgres03.arc", + "res/Layout/msgres04.arc", + "res/Layout/msgres05.arc", + "res/Layout/msgres06.arc", + "res/Layout/main2D.arc", + "res/Fontus/fontres.arc", + "res/Fontus/rubyres.arc", + "res/Particle/common.jpc", + "res/ItemTable/item_table.bin", + "res/ItemTable/enemy_table.bin", + "res/Stage/F_SP102/STG_00.arc", + "res/Object/Event.arc", + "res/Object/CamParam.arc", + "res/Particle/Pscene001.jpc", + "res/Msgus/bmgres8.arc", + "rel/Final/Release/d_a_title.rel", + "res/Stage/F_SP102/R00_00.arc", + "res/Object/Title.arc", + "res/Object/Demo38_01.arc", + "res/Layout/Title2D.arc", + "res/Object/Kmdl.arc", + "rel/Final/Release/d_a_obj_ihasi.rel", + "rel/Final/Release/d_a_horse.rel", + "res/Object/Obj_ihasi.arc", + "res/Object/CstaFB.arc", + "res/Object/@bg0056.arc", + "res/Object/@bg0010.arc", + "res/Object/HyShd.arc", + "res/Object/Horse.arc", + "res/Object/J_Umak.arc", + "res/Object/Midna.arc", + "Audiores/Stream/title_back.ast", + "res/Object/fileSel.arc", + "Audiores/Stream/menu_select.ast", +] + +def getPaddingByOffset(padding, offset): + return padding - (offset % padding) def addPaddingToFile(file, padding): - file.write(bytearray(padding - (file.tell() % padding))) + file.write(bytearray(getPaddingByOffset(padding,file.tell()))) def sortFileList(x): @@ -20,8 +101,8 @@ def sortFileList(x): def parseDir(dir, stringTable, currentEntryNum, parent): - os.chdir(dir) - entries = sorted(os.listdir("./"), key=sortFileList) + # print(dir) + entries = sorted(os.listdir(dir), key=sortFileList) table = [] for entry in entries: currentEntryNum = currentEntryNum + 1 @@ -35,20 +116,18 @@ def parseDir(dir, stringTable, currentEntryNum, parent): "path": None, } stringTable = stringTable + entry + "\0" - if os.path.isdir(entry): + if os.path.isdir(dir / entry): tableEntry["children"], stringTable, currentEntryNum = parseDir( - entry, stringTable, currentEntryNum, tableEntry + dir/entry, stringTable, currentEntryNum, tableEntry ) else: - tableEntry["size"] = os.path.getsize(entry) - tableEntry["path"] = os.path.abspath(entry) + tableEntry["size"] = os.path.getsize(dir/entry) + tableEntry["path"] = dir/entry table.append(tableEntry) - os.chdir("..") return table, stringTable, currentEntryNum - -def recurseWriteFst(files, fstBin, isoFile): +def recurseCreateFST(files,fst): for currentFile in files: if currentFile["children"]: # is a directory nextDirEntryNum = 0 @@ -62,34 +141,13 @@ def recurseWriteFst(files, fstBin, isoFile): break if currentFile["parent"]: parentEntryNum = currentFile["parent"]["entryNum"] - fstBin.append( - struct.pack( - ">BBHII", - 1, - (currentFile["stringTableOffset"] >> 16) & 0xFF, - currentFile["stringTableOffset"] & 0xFFFF, - parentEntryNum, - nextDirEntryNum, - ) - ) - recurseWriteFst(currentFile["children"], fstBin, isoFile) + fst.append({'type': 'dir', 'stringTableOffset': currentFile["stringTableOffset"], 'parentEntryNum': parentEntryNum, 'nextDirEntryNum': nextDirEntryNum}) + recurseCreateFST(currentFile["children"],fst) else: # is a file - fstBin.append( - struct.pack( - ">BBHII", - 0, - (currentFile["stringTableOffset"] >> 16) & 0xFF, - currentFile["stringTableOffset"] & 0xFFFF, - isoFile.tell(), - currentFile["size"], - ) - ) - isoFile.write(open(currentFile["path"], "rb").read()) - addPaddingToFile(isoFile, 0x100) - + fst.append({'type': 'file', 'path': currentFile["path"], 'stringTableOffset': currentFile["stringTableOffset"], 'offset': None, 'size': currentFile["size"], 'adjustedSize': currentFile["size"]+getPaddingByOffset(0x100,currentFile["size"])}) def packageISO(sourcePath, destPath): - print("Converting filesystem into ISO") + print(f"Packaging directory '{sourcePath}' into ISO '{destPath}'") ISOfile = open(destPath, "wb") # write sys files @@ -108,7 +166,9 @@ def packageISO(sourcePath, destPath): fstStartPos = ISOfile.tell() - files, stringTable, entryLength = parseDir(sourcePath / "files", "", 0, None) + cwd = os.getcwd() + os.chdir(sourcePath / "files") + files, stringTable, entryLength = parseDir(Path("./"), "", 0, None) entryLength = entryLength + 1 ISOfile.seek(0x424) @@ -116,26 +176,99 @@ def packageISO(sourcePath, destPath): struct.pack(">II", fstStartPos, (entryLength * 12) + len(stringTable)) ) + # print(files) + + fst = [] + recurseCreateFST(files,fst) + + pathToEntryDict = {f["path"]: i for i,f in enumerate(fst) if f["type"] == "file"} + + for priorityFile in TP_BOOT_PRIORITY: + entry = pathToEntryDict[Path(priorityFile)] + fst[entry]["bootPriority"] = True + fst[entry]["priorityIndex"] = TP_FILE_PRIORITY.index("BOOT") + + for file in fst: + if file["type"] == "dir" or ("bootPriority" in file and file["bootPriority"]): + continue + + priorityMatch = max([(i,str(file["path"]).rfind(s)) for i,s in enumerate(TP_FILE_PRIORITY)], key=lambda x: x[1]) + + priorityIndex = priorityMatch[0] + if priorityMatch[1] == -1: + priorityIndex = TP_FILE_PRIORITY.index("./") + + file["priorityIndex"] = priorityIndex + # print(file["path"]) + # print(priorityMatch) + # print(priorityIndex) + + fileOrderBins = [[j for j,f in enumerate(fst) if f["type"] == "file" and i == f["priorityIndex"]] for i in range(len(TP_FILE_PRIORITY))] + # print(fileOrderBins) + + fileOrder = [] + + current_offset = ISO_MAX_SIZE - 0x14 + for i,priority in enumerate(fileOrderBins): + if i == TP_FILE_PRIORITY.index("BOOT"): + priority = [pathToEntryDict[Path(j)] for j in TP_BOOT_PRIORITY] + for f in reversed(priority): + current_offset -= fst[f]["size"] + if os.path.splitext(fst[f]["path"])[1] != ".thp": + current_offset -= (current_offset % 4) + else: + current_offset -= (current_offset % 0x8000) + fst[f]["offset"] = current_offset + fileOrder.append(f) + # print(f"{current_offset:X} {fst[f]}") + + assert current_offset >= fstStartPos + (entryLength * 12) + len(stringTable), "ISO File is too large!" + + # Write FST and stringtable + + ISOfile.seek(fstStartPos) + + ISOfile.write(struct.pack(">BBHII", 1, 0, 0, 0, entryLength)) # ROOT + for f in fst: + if f["type"] == "dir": + ISOfile.write(struct.pack( + ">BBHII", + 1, + (f["stringTableOffset"] >> 16) & 0xFF, + f["stringTableOffset"] & 0xFFFF, + f["parentEntryNum"], + f["nextDirEntryNum"], + )) + else: + ISOfile.write(struct.pack( + ">BBHII", + 0, + (f["stringTableOffset"] >> 16) & 0xFF, + f["stringTableOffset"] & 0xFFFF, + f["offset"], + f["size"], + )) + ISOfile.write(bytearray(stringTable, "utf8")) + + for f in reversed(fileOrder): + file = fst[f] + ISOfile.seek(file["offset"]) + with open(file["path"],"rb") as of: + ISOfile.write(of.read()) + + # Fill the rest of the ISO in + ISOfile.write(bytearray(ISO_MAX_SIZE - (ISOfile.tell()))) + + os.chdir(cwd) + + # print(fst) + + # print(stringTable) # print(files) # print(hex(entryLength)) # print(files[0]["entryNum"]) - fstBin = [] - fstBin.append(struct.pack(">BBHII", 1, 0, 0, 0, entryLength)) - - ISOfile.seek(fstStartPos + (entryLength * 12) + len(stringTable)) - addPaddingToFile(ISOfile, 0x100) - recurseWriteFst(files, fstBin, ISOfile) - - ISOfile.write(bytearray(1459978240 - (ISOfile.tell()))) - - ISOfile.seek(fstStartPos) - for fileEntry in fstBin: - ISOfile.write(fileEntry) - ISOfile.seek((entryLength * 12) + fstStartPos) - ISOfile.write(bytearray(stringTable, "utf8")) - if __name__ == "__main__": packageISO(Path(sys.argv[1]), Path(sys.argv[2])) diff --git a/tools/package_game_assets.py b/tools/package_game_assets.py index daa7b753a5..463d224de3 100644 --- a/tools/package_game_assets.py +++ b/tools/package_game_assets.py @@ -21,34 +21,30 @@ def getMaxDateFromDir(path): return maxTime -convertDefinitions = [ - { - "sourceExtension": "arc", +convertDefinitions = { + "arc": { "destExtension": ".arc", "convertFunction": libarc.convert_dir_to_arc, "exceptions": ["game/files/res/Object/HomeBtn.c.arc/archive/dat/speakerse.arc"], + "sourceIsDir": True }, - { - "sourceExtension": "dzs.json", + "dzs.json": { "destExtension": ".dzs", "convertFunction": libstage.package_from_json, }, - { - "sourceExtension": "dzr.json", + "dzr.json": { "destExtension": ".dzr", "convertFunction": libstage.package_from_json, }, - { - "sourceExtension": "png", + "png": { "destExtension": ".bti", "convertFunction": libbti.png_to_bti }, - { - "sourceExtension": "bti.json", + "bti.json": { "destExtension": None, "convertFunction": None } -] +} yaz0CompressFunction = libyaz0.compress @@ -64,16 +60,15 @@ def convertEntry(file, path, destPath, returnData): data = None extractDef = None - for extractData in convertDefinitions: - if sourceExtension == extractData["sourceExtension"]: - if extractData["destExtension"] == None and extractData["convertFunction"] == None: - return - extractDef = extractData - if "exceptions" in extractData: - for exception in extractData["exceptions"]: - if str(path / file) == exception: - extractDef = None - break + if sourceExtension in convertDefinitions: + extractDef = convertDefinitions[sourceExtension] + + if extractDef["destExtension"] == None and extractDef["convertFunction"] == None: + extractDef = None + elif "exceptions" in extractDef and str(path/file) in extractDef["exceptions"]: + extractDef = None + elif "sourceIsDir" in extractDef and extractDef["sourceIsDir"] == True and os.path.isfile(path/file): + extractDef = None if extractDef != None: destFileName = destFileName.split(".")[0] + extractDef["destExtension"] @@ -348,8 +343,11 @@ def copyMapFiles(buildPath): for map in (buildPath/"dolzel2/rel/").rglob("*.map"): open(buildPath/"dolzel2/game/files/map/Final/Release/"/map.name,"w").write(postprocessMapFile(open(map,"r").read())) -def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file): - if yaz0Encoding == "oead": +def main(gamePath, buildPath, copyCode, config_file): + global config + config = assets_config.getConfig(config_file, update = True) + + if config["oead_yaz0"]: try: from oead import yaz0 global yaz0CompressFunction @@ -368,10 +366,7 @@ def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file): if not (gamePath / "files").exists() or not (gamePath / "sys").exists(): print("ISO is not extracted; extracting...") - extract_game_assets.extract(iso.absolute(),gamePath.absolute(),yaz0Encoding,config_file) - - global config - config = assets_config.getConfig(config_file, update = True) + extract_game_assets.extract(iso.absolute(),gamePath.absolute(),config_file) print("Copying game files...") if os.path.exists(buildPath / "dolzel2") == False: @@ -404,4 +399,4 @@ def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file): if __name__ == "__main__": - main(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4], sys.argv[5]) + main(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4]) diff --git a/tools/tp.py b/tools/tp.py index 80b686bfe6..9e84b4ec8d 100644 --- a/tools/tp.py +++ b/tools/tp.py @@ -145,16 +145,10 @@ elif platform.system() == "Darwin": default=DEFAULT_TOOLS_PATH, required=True, ) -@click.option( - "--yaz0-encoder", - type=str, - default="native", - required=False, -) @click.option("--force-download/--no-force-download") @click.option("--skip-iso/--no-skip-iso", default=False) @click.option("--use-default-config/--no-use-default-config", default=False) -def setup(debug: bool, game_path: Path, tools_path: Path, yaz0_encoder: str, force_download: bool, skip_iso: bool, use_default_config: bool): +def setup(debug: bool, game_path: Path, tools_path: Path, force_download: bool, skip_iso: bool, use_default_config: bool): """Setup project""" if debug: @@ -385,7 +379,7 @@ def setup(debug: bool, game_path: Path, tools_path: Path, yaz0_encoder: str, for try: import extract_game_assets - extract_game_assets.extract(iso, game_path, yaz0_encoder, configfile_name) + extract_game_assets.extract(iso, game_path, configfile_name) except ImportError as ex: _handle_import_error(ex) except Exception as e: