Tools: update packageISO and move yaz0 function to config (#2183)

This commit is contained in:
jdflyer 2024-07-22 11:33:16 -07:00 committed by GitHub
parent ab7d0b88c3
commit 70e278ee84
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 221 additions and 118 deletions

View File

@ -156,11 +156,7 @@ tools: dirs $(ELF2DOL) $(YAZ0)
assets:
@mkdir -p game
$(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game native asset_config.json
assets-fast:
@mkdir -p game
$(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game oead asset_config.json
$(PYTHON) tools/extract_game_assets.py $(IMAGENAME) game asset_config.json
docs:
$(DOXYGEN) Doxyfile
@ -193,32 +189,16 @@ shiftedrels: shift $(RELS)
game: shiftedrels
@mkdir -p game
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode native asset_config.json
game-fast: shiftedrels
@mkdir -p game
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode oead asset_config.json
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) copyCode asset_config.json
game-nocompile:
@mkdir -p game
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode native asset_config.json
game-nocompile-fast:
@mkdir -p game
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode oead asset_config.json
@$(PYTHON) tools/package_game_assets.py ./game $(BUILD_PATH) noCopyCode asset_config.json
rungame-nocompile: game-nocompile
@echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin!
dolphin-emu $(BUILD_DIR)/game/sys/main.dol
rungame-nocompile-fast: game-nocompile-fast
@echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin!
dolphin-emu $(BUILD_DIR)/game/sys/main.dol
rungame-fast: game-fast
@echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin!
dolphin-emu $(BUILD_DIR)/game/sys/main.dol
rungame: game
@echo If you are playing on a shifted game make sure Hyrule Field Speed hack is disabled in dolphin!
dolphin-emu $(BUILD_DIR)/game/sys/main.dol
@ -226,7 +206,6 @@ rungame: game
iso: game
@$(PYTHON) tools/packageISO.py $(BUILD_DIR)/game/ $(TARGET_ISO)
$(BUILD_DIR)/%.o: %.c $(BUILD_DIR)/%.d
@mkdir -p $(@D)
@echo building... $<

View File

@ -4,6 +4,7 @@ import json
from os import path
CONFIG_DESCRIPTIONS = {
"oead_yaz0": "Use oead (pip install oead) for asset (de)compression (faster)",
"decompress_assets": "Decompress Yaz0 compressed assets (appends .c to the filename)",
"extract_arc": "Extract archive (.arc) files",
"convert_stages": "Convert Stage Files (.dzs and .dzr) to json",
@ -13,6 +14,7 @@ CONFIG_DESCRIPTIONS = {
}
CONFIG_DEFAULTS = {
"oead_yaz0": False,
"decompress_assets": True,
"extract_arc": True,
"convert_stages": True,

View File

@ -170,7 +170,7 @@ def writeFile(name, data):
if ("exceptions" in extractDef and str(name) in extractDef["exceptions"]) or ("config_key" in extractDef and config[extractDef["config_key"]] == False):
extractDef = None
if config["decompress_assets"] == None:
if config["decompress_assets"] == False:
extractDef = None # If assets aren't being decompressed, just write the raw file
if extractDef == None:
@ -245,16 +245,16 @@ def getDolInfo(disc):
return dolOffset, dolSize
def extract(isoPath: Path, gamePath: Path, yaz0Encoder: str, config_file: str):
if yaz0Encoder == "oead":
def extract(isoPath: Path, gamePath: Path, config_file: str):
global config
config = assets_config.getConfig(config_file,update=True)
if config["oead_yaz0"]:
try:
from oead import yaz0
global yaz0DecompressFunction
yaz0DecompressFunction = yaz0.decompress
except:
print("Extract: oead isn't installed, falling back to native yaz0")
global config
config = assets_config.getConfig(config_file,update=True)
isoPath = isoPath.absolute()
cwd = os.getcwd()
os.chdir(gamePath)
@ -302,7 +302,7 @@ def extract(isoPath: Path, gamePath: Path, yaz0Encoder: str, config_file: str):
def main():
extract(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4])
extract(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3])
if __name__ == "__main__":

View File

@ -3,9 +3,90 @@ import os
from pathlib import Path
import struct
ISO_MAX_SIZE = 1459978240
# Directories towards the top will be at the end of the ISO
# The entries in the BOOT section have priority over the directories listed
TP_FILE_PRIORITY = [
"rel/",
"./", # All uncaught files
"BOOT", # All files in the BOOT array
"res/Object/",
"res/Stage/",
"Audiores/Waves/",
"Audiores/Stream/",
"res/Particle/",
"Movie/",
"map/"
]
# There are a few differences here than actually running the game from boot,
# Likely due to version differences. This is the order on GC US
TP_BOOT_PRIORITY = [
"str/Final/Release/COPYDATE",
"RELS.arc",
"Audiores/Z2Sound.baa",
"Audiores/Seqs/Z2SoundSeqs.arc",
"res/Object/LogoUs.arc",
"res/Object/Always.arc",
"res/Object/Alink.arc",
"res/FieldMap/Field0.arc",
"res/Object/AlAnm.arc",
"res/Layout/fmapres.arc",
"res/Layout/dmapres.arc",
"res/Layout/clctres.arc",
"res/Layout/itemicon.arc",
"res/Layout/ringres.arc",
"res/Layout/playerName.arc",
"res/Layout/itmInfRes.arc",
"res/Layout/button.arc",
"res/CardIcon/cardicon.arc",
"res/Msgus/bmgres.arc",
"res/Layout/msgcom.arc",
"res/Layout/msgres00.arc",
"res/Layout/msgres01.arc",
"res/Layout/msgres02.arc",
"res/Layout/msgres03.arc",
"res/Layout/msgres04.arc",
"res/Layout/msgres05.arc",
"res/Layout/msgres06.arc",
"res/Layout/main2D.arc",
"res/Fontus/fontres.arc",
"res/Fontus/rubyres.arc",
"res/Particle/common.jpc",
"res/ItemTable/item_table.bin",
"res/ItemTable/enemy_table.bin",
"res/Stage/F_SP102/STG_00.arc",
"res/Object/Event.arc",
"res/Object/CamParam.arc",
"res/Particle/Pscene001.jpc",
"res/Msgus/bmgres8.arc",
"rel/Final/Release/d_a_title.rel",
"res/Stage/F_SP102/R00_00.arc",
"res/Object/Title.arc",
"res/Object/Demo38_01.arc",
"res/Layout/Title2D.arc",
"res/Object/Kmdl.arc",
"rel/Final/Release/d_a_obj_ihasi.rel",
"rel/Final/Release/d_a_horse.rel",
"res/Object/Obj_ihasi.arc",
"res/Object/CstaFB.arc",
"res/Object/@bg0056.arc",
"res/Object/@bg0010.arc",
"res/Object/HyShd.arc",
"res/Object/Horse.arc",
"res/Object/J_Umak.arc",
"res/Object/Midna.arc",
"Audiores/Stream/title_back.ast",
"res/Object/fileSel.arc",
"Audiores/Stream/menu_select.ast",
]
def getPaddingByOffset(padding, offset):
return padding - (offset % padding)
def addPaddingToFile(file, padding):
file.write(bytearray(padding - (file.tell() % padding)))
file.write(bytearray(getPaddingByOffset(padding,file.tell())))
def sortFileList(x):
@ -20,8 +101,8 @@ def sortFileList(x):
def parseDir(dir, stringTable, currentEntryNum, parent):
os.chdir(dir)
entries = sorted(os.listdir("./"), key=sortFileList)
# print(dir)
entries = sorted(os.listdir(dir), key=sortFileList)
table = []
for entry in entries:
currentEntryNum = currentEntryNum + 1
@ -35,20 +116,18 @@ def parseDir(dir, stringTable, currentEntryNum, parent):
"path": None,
}
stringTable = stringTable + entry + "\0"
if os.path.isdir(entry):
if os.path.isdir(dir / entry):
tableEntry["children"], stringTable, currentEntryNum = parseDir(
entry, stringTable, currentEntryNum, tableEntry
dir/entry, stringTable, currentEntryNum, tableEntry
)
else:
tableEntry["size"] = os.path.getsize(entry)
tableEntry["path"] = os.path.abspath(entry)
tableEntry["size"] = os.path.getsize(dir/entry)
tableEntry["path"] = dir/entry
table.append(tableEntry)
os.chdir("..")
return table, stringTable, currentEntryNum
def recurseWriteFst(files, fstBin, isoFile):
def recurseCreateFST(files,fst):
for currentFile in files:
if currentFile["children"]: # is a directory
nextDirEntryNum = 0
@ -62,34 +141,13 @@ def recurseWriteFst(files, fstBin, isoFile):
break
if currentFile["parent"]:
parentEntryNum = currentFile["parent"]["entryNum"]
fstBin.append(
struct.pack(
">BBHII",
1,
(currentFile["stringTableOffset"] >> 16) & 0xFF,
currentFile["stringTableOffset"] & 0xFFFF,
parentEntryNum,
nextDirEntryNum,
)
)
recurseWriteFst(currentFile["children"], fstBin, isoFile)
fst.append({'type': 'dir', 'stringTableOffset': currentFile["stringTableOffset"], 'parentEntryNum': parentEntryNum, 'nextDirEntryNum': nextDirEntryNum})
recurseCreateFST(currentFile["children"],fst)
else: # is a file
fstBin.append(
struct.pack(
">BBHII",
0,
(currentFile["stringTableOffset"] >> 16) & 0xFF,
currentFile["stringTableOffset"] & 0xFFFF,
isoFile.tell(),
currentFile["size"],
)
)
isoFile.write(open(currentFile["path"], "rb").read())
addPaddingToFile(isoFile, 0x100)
fst.append({'type': 'file', 'path': currentFile["path"], 'stringTableOffset': currentFile["stringTableOffset"], 'offset': None, 'size': currentFile["size"], 'adjustedSize': currentFile["size"]+getPaddingByOffset(0x100,currentFile["size"])})
def packageISO(sourcePath, destPath):
print("Converting filesystem into ISO")
print(f"Packaging directory '{sourcePath}' into ISO '{destPath}'")
ISOfile = open(destPath, "wb")
# write sys files
@ -108,7 +166,9 @@ def packageISO(sourcePath, destPath):
fstStartPos = ISOfile.tell()
files, stringTable, entryLength = parseDir(sourcePath / "files", "", 0, None)
cwd = os.getcwd()
os.chdir(sourcePath / "files")
files, stringTable, entryLength = parseDir(Path("./"), "", 0, None)
entryLength = entryLength + 1
ISOfile.seek(0x424)
@ -116,26 +176,99 @@ def packageISO(sourcePath, destPath):
struct.pack(">II", fstStartPos, (entryLength * 12) + len(stringTable))
)
# print(files)
fst = []
recurseCreateFST(files,fst)
pathToEntryDict = {f["path"]: i for i,f in enumerate(fst) if f["type"] == "file"}
for priorityFile in TP_BOOT_PRIORITY:
entry = pathToEntryDict[Path(priorityFile)]
fst[entry]["bootPriority"] = True
fst[entry]["priorityIndex"] = TP_FILE_PRIORITY.index("BOOT")
for file in fst:
if file["type"] == "dir" or ("bootPriority" in file and file["bootPriority"]):
continue
priorityMatch = max([(i,str(file["path"]).rfind(s)) for i,s in enumerate(TP_FILE_PRIORITY)], key=lambda x: x[1])
priorityIndex = priorityMatch[0]
if priorityMatch[1] == -1:
priorityIndex = TP_FILE_PRIORITY.index("./")
file["priorityIndex"] = priorityIndex
# print(file["path"])
# print(priorityMatch)
# print(priorityIndex)
fileOrderBins = [[j for j,f in enumerate(fst) if f["type"] == "file" and i == f["priorityIndex"]] for i in range(len(TP_FILE_PRIORITY))]
# print(fileOrderBins)
fileOrder = []
current_offset = ISO_MAX_SIZE - 0x14
for i,priority in enumerate(fileOrderBins):
if i == TP_FILE_PRIORITY.index("BOOT"):
priority = [pathToEntryDict[Path(j)] for j in TP_BOOT_PRIORITY]
for f in reversed(priority):
current_offset -= fst[f]["size"]
if os.path.splitext(fst[f]["path"])[1] != ".thp":
current_offset -= (current_offset % 4)
else:
current_offset -= (current_offset % 0x8000)
fst[f]["offset"] = current_offset
fileOrder.append(f)
# print(f"{current_offset:X} {fst[f]}")
assert current_offset >= fstStartPos + (entryLength * 12) + len(stringTable), "ISO File is too large!"
# Write FST and stringtable
ISOfile.seek(fstStartPos)
ISOfile.write(struct.pack(">BBHII", 1, 0, 0, 0, entryLength)) # ROOT
for f in fst:
if f["type"] == "dir":
ISOfile.write(struct.pack(
">BBHII",
1,
(f["stringTableOffset"] >> 16) & 0xFF,
f["stringTableOffset"] & 0xFFFF,
f["parentEntryNum"],
f["nextDirEntryNum"],
))
else:
ISOfile.write(struct.pack(
">BBHII",
0,
(f["stringTableOffset"] >> 16) & 0xFF,
f["stringTableOffset"] & 0xFFFF,
f["offset"],
f["size"],
))
ISOfile.write(bytearray(stringTable, "utf8"))
for f in reversed(fileOrder):
file = fst[f]
ISOfile.seek(file["offset"])
with open(file["path"],"rb") as of:
ISOfile.write(of.read())
# Fill the rest of the ISO in
ISOfile.write(bytearray(ISO_MAX_SIZE - (ISOfile.tell())))
os.chdir(cwd)
# print(fst)
# print(stringTable)
# print(files)
# print(hex(entryLength))
# print(files[0]["entryNum"])
fstBin = []
fstBin.append(struct.pack(">BBHII", 1, 0, 0, 0, entryLength))
ISOfile.seek(fstStartPos + (entryLength * 12) + len(stringTable))
addPaddingToFile(ISOfile, 0x100)
recurseWriteFst(files, fstBin, ISOfile)
ISOfile.write(bytearray(1459978240 - (ISOfile.tell())))
ISOfile.seek(fstStartPos)
for fileEntry in fstBin:
ISOfile.write(fileEntry)
ISOfile.seek((entryLength * 12) + fstStartPos)
ISOfile.write(bytearray(stringTable, "utf8"))
if __name__ == "__main__":
packageISO(Path(sys.argv[1]), Path(sys.argv[2]))

View File

@ -21,34 +21,30 @@ def getMaxDateFromDir(path):
return maxTime
convertDefinitions = [
{
"sourceExtension": "arc",
convertDefinitions = {
"arc": {
"destExtension": ".arc",
"convertFunction": libarc.convert_dir_to_arc,
"exceptions": ["game/files/res/Object/HomeBtn.c.arc/archive/dat/speakerse.arc"],
"sourceIsDir": True
},
{
"sourceExtension": "dzs.json",
"dzs.json": {
"destExtension": ".dzs",
"convertFunction": libstage.package_from_json,
},
{
"sourceExtension": "dzr.json",
"dzr.json": {
"destExtension": ".dzr",
"convertFunction": libstage.package_from_json,
},
{
"sourceExtension": "png",
"png": {
"destExtension": ".bti",
"convertFunction": libbti.png_to_bti
},
{
"sourceExtension": "bti.json",
"bti.json": {
"destExtension": None,
"convertFunction": None
}
]
}
yaz0CompressFunction = libyaz0.compress
@ -64,16 +60,15 @@ def convertEntry(file, path, destPath, returnData):
data = None
extractDef = None
for extractData in convertDefinitions:
if sourceExtension == extractData["sourceExtension"]:
if extractData["destExtension"] == None and extractData["convertFunction"] == None:
return
extractDef = extractData
if "exceptions" in extractData:
for exception in extractData["exceptions"]:
if str(path / file) == exception:
extractDef = None
break
if sourceExtension in convertDefinitions:
extractDef = convertDefinitions[sourceExtension]
if extractDef["destExtension"] == None and extractDef["convertFunction"] == None:
extractDef = None
elif "exceptions" in extractDef and str(path/file) in extractDef["exceptions"]:
extractDef = None
elif "sourceIsDir" in extractDef and extractDef["sourceIsDir"] == True and os.path.isfile(path/file):
extractDef = None
if extractDef != None:
destFileName = destFileName.split(".")[0] + extractDef["destExtension"]
@ -348,8 +343,11 @@ def copyMapFiles(buildPath):
for map in (buildPath/"dolzel2/rel/").rglob("*.map"):
open(buildPath/"dolzel2/game/files/map/Final/Release/"/map.name,"w").write(postprocessMapFile(open(map,"r").read()))
def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file):
if yaz0Encoding == "oead":
def main(gamePath, buildPath, copyCode, config_file):
global config
config = assets_config.getConfig(config_file, update = True)
if config["oead_yaz0"]:
try:
from oead import yaz0
global yaz0CompressFunction
@ -368,10 +366,7 @@ def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file):
if not (gamePath / "files").exists() or not (gamePath / "sys").exists():
print("ISO is not extracted; extracting...")
extract_game_assets.extract(iso.absolute(),gamePath.absolute(),yaz0Encoding,config_file)
global config
config = assets_config.getConfig(config_file, update = True)
extract_game_assets.extract(iso.absolute(),gamePath.absolute(),config_file)
print("Copying game files...")
if os.path.exists(buildPath / "dolzel2") == False:
@ -404,4 +399,4 @@ def main(gamePath, buildPath, copyCode, yaz0Encoding, config_file):
if __name__ == "__main__":
main(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4], sys.argv[5])
main(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3], sys.argv[4])

View File

@ -145,16 +145,10 @@ elif platform.system() == "Darwin":
default=DEFAULT_TOOLS_PATH,
required=True,
)
@click.option(
"--yaz0-encoder",
type=str,
default="native",
required=False,
)
@click.option("--force-download/--no-force-download")
@click.option("--skip-iso/--no-skip-iso", default=False)
@click.option("--use-default-config/--no-use-default-config", default=False)
def setup(debug: bool, game_path: Path, tools_path: Path, yaz0_encoder: str, force_download: bool, skip_iso: bool, use_default_config: bool):
def setup(debug: bool, game_path: Path, tools_path: Path, force_download: bool, skip_iso: bool, use_default_config: bool):
"""Setup project"""
if debug:
@ -385,7 +379,7 @@ def setup(debug: bool, game_path: Path, tools_path: Path, yaz0_encoder: str, for
try:
import extract_game_assets
extract_game_assets.extract(iso, game_path, yaz0_encoder, configfile_name)
extract_game_assets.extract(iso, game_path, configfile_name)
except ImportError as ex:
_handle_import_error(ex)
except Exception as e: